How to raycast select a scaled OBB?
- by user3254944
I have the OBB picking code to select an OBB with code inspired from Real time Rendering 3 and opengl-tutorial.org. I can successfully select objects that have been moved or rotated. However, I cant correctly select an object that has been scaled. The bounding box scales right, but the I can only select the object in a thin strip on its center.
How do I fix the checkForHits() function to allow it to read the scaling that I passed to it in the raycast matrix?
void GLWidget::selectObjRaycast()
{
glm::vec2 mouse = (glm::vec2(mousePos.x(), mousePos.y()) / glm::vec2(this->width(), this->height())) * 2.0f - 1.0f;
mouse.y *= -1;
glm::mat4 toWorld = glm::inverse(ProjectionM * ViewM);
glm::vec4 from = toWorld * glm::vec4(mouse, -1.0f, 1.0f);
glm::vec4 to = toWorld * glm::vec4(mouse, 1.0f, 1.0f);
from /= from.w;
to /= to.w;
fromAABB = glm::vec3(from);
toAABB = glm::normalize(glm::vec3(to - from));
checkForHits();
}
void GLWidget::checkForHits()
{
for (int i = 0; i < myWin.myEtc->allObj.size(); ++i) //check for hits on each obj's bb
{
bool miss = 0;
float tMin = 0.0f;
float tMax = 100000.0f;
glm::vec3 bbPos(myWin.myEtc->allObj[i]->raycastM[3].x, myWin.myEtc->allObj[i]->raycastM[3].y, myWin.myEtc->allObj[i]->raycastM[3].z);
glm::vec3 delta = bbPos - fromAABB;
for (int j = 0; j < 3; ++j)
{
glm::vec3 axis(myWin.myEtc->allObj[i]->raycastM[j].x, myWin.myEtc->allObj[i]->raycastM[j].y, myWin.myEtc->allObj[i]->raycastM[j].z);
float e = glm::dot(axis, delta);
float f = glm::dot(toAABB, axis);
if (fabs(f) > 0.001f)
{
float t1 = (e + myWin.myEtc->allObj[i]->bbMin[j]) / f;
float t2 = (e + myWin.myEtc->allObj[i]->bbMax[j]) / f;
if (t1 > t2)
{
float w = t1;
t1 = t2;
t2 = w;
}
if (t2 < tMax)
tMax = t2;
if (t1 > tMin)
tMin = t1;
if (tMax < tMin)
miss = 1;
}
else
{
if (-e + myWin.myEtc->allObj[i]->bbMin[j] > 0.0f || -e + myWin.myEtc->allObj[i]->bbMax[j] < 0.0f)
miss = 1;
}
}
if (miss == 0)
{
intersection_distance = tMin;
myWin.myEtc->sel.push_back(myWin.myEtc->allObj[i]);
myWin.myEtc->allObj[i]->highlight = myWin.myGLHelp->highlight;
break;
}
}
}
void Object::render(glm::mat4 PV)
{
scaleM = glm::scale(glm::mat4(), s->val_3);
r_quat = glm::quat(glm::radians(r->val_3));
rotationM = glm::toMat4(r_quat);
translationM = glm::translate(glm::mat4(), t->val_3);
transLocal1M = glm::translate(glm::mat4(), -rsPivot->val_3);
transLocal2M = glm::translate(glm::mat4(), rsPivot->val_3);
raycastM = translationM * transLocal2M * rotationM * scaleM * transLocal1M; //
MVP = PV * translationM * transLocal2M * rotationM * scaleM * transLocal1M;
}
