diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b96d3a..dfba7be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ add_executable( fc2d src/toml.c src/tomlcpp.cpp src/error.cpp + src/eye.cpp ) target_link_libraries( fc2d ${OpenCV_LIBS} ${OPENGL_LIBRARIES} ${WEBP_LIBRARIES} FreeGLUT::freeglut GLEW::glew zip Boxer fmt ) diff --git a/src/cv.cpp b/src/cv.cpp index 31d997a..146ca14 100644 --- a/src/cv.cpp +++ b/src/cv.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include cv::Ptr facemark; @@ -98,9 +99,9 @@ void cvFrame() { std::vector> landmarks; if (facemark->fit (frame, faces, landmarks)) { - //for (int i = 0; i < landmarks[biggestFace].size(); i++) { - // cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255)); - //} + for (int i = 0; i < landmarks[biggestFace].size(); i++) { + cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255)); + } cv::circle(frame, cv::Point2f( (landmarks[biggestFace][2].x + landmarks[biggestFace][14].x) / 2, (landmarks[biggestFace][2].y + landmarks[biggestFace][14].y) / 2 @@ -109,6 +110,20 @@ void cvFrame() { cv::circle (frame, landmarks[biggestFace][66], 3, cv::Scalar (0, 255, 0)); cv::circle (frame, landmarks[biggestFace][62], 3, cv::Scalar (0, 255, 0)); + //get ROI for eyes + float eyeWidth = landmarks[biggestFace][45].x - landmarks[biggestFace][42].x; + cv::Rect eyeRect(landmarks[biggestFace][42].x, landmarks[biggestFace][42].y - eyeWidth / 2, eyeWidth, eyeWidth); + + cv::rectangle(frame, eyeRect, cv::Scalar(255, 255, 255)); + + cv::Mat eyeROI; + eyeROI = gray(eyeRect); + + glm::vec2 eyeVector = eyeDirection(eyeROI); + + cv::imshow("eye", eyeROI); + cv::waitKey(1); + //send control information to graphics float faceSize = landmarks[biggestFace][14].x - landmarks[biggestFace][2].x; @@ -124,6 +139,7 @@ void cvFrame() { landmarks[biggestFace][30].x * 2 / (float)frame.cols - 1, landmarks[biggestFace][30].y * 2 / (float)frame.rows - 1 ); + faceData.triggers[TRIGGER_NULL] = false; faceData.triggers[TRIGGER_MOUTH_OPEN] = (landmarks[biggestFace][66].y - landmarks[biggestFace][62].y) / faceSize > 0.04f; @@ -133,6 +149,11 @@ void cvFrame() { (float)(landmarks[biggestFace][2].x - landmarks[biggestFace][14].x)); faceData.scale = faceSize * 6 / (float)frame.cols; + + cv::line(frame, cv::Point(50,50), cv::Point(50,50) + cv::Point(eyeVector.x * 25, eyeVector.y * 25), cv::Scalar(255,128,128)); + std::cout << eyeVector.x << "," << eyeVector.y << std::endl; + std::cout << "SIZE:" << eyeROI.cols << "," << eyeROI.rows << std::endl; + updateModel(faceData); } } diff --git a/src/eye.cpp b/src/eye.cpp new file mode 100644 index 0000000..ead36e7 --- /dev/null +++ b/src/eye.cpp @@ -0,0 +1,104 @@ +#include + + +float objectiveFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) { + float sum = 0; + for(int xx = 0; xx < gX.cols; xx++) { + for(int xy = 0; xy < gX.rows; xy++) { + cv::Point2f x(xx, xy); + cv::Point2f g(gX.at(xx, xy), gY.at(xx, xy)); // gradient + double gMag = std::sqrt(g.x * g.x + g.y * g.y); + if(gMag < 200.0) continue; // ignore gradients in homogenous regions + //g = g / gMag; + cv::Point2f d = x - c; // displacement + d /= std::sqrt(d.x * d.x + d.y * d.y); // normalize d + + float dotProduct = d.y * g.x + d.x * g.y; + + dotProduct = std::max(0.0f, dotProduct); + + sum += dotProduct * dotProduct; + } + } + sum /= gX.rows * gX.cols; + + return sum; +} + +cv::Point2f derivativeFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) { + cv::Point2f sum(0,0); + + for(int xx = 0; xx < gX.cols; xx++) { + for(int xy = 0; xy < gX.rows; xy++) { + cv::Point2f x(xy, xx); + cv::Point2f g(gY.at(xx, xy), gX.at(xx, xy)); // gradient + float gMag = std::sqrt(g.x * g.x + g.y * g.y); + if(gMag < 200.0f) continue; // ignore gradients in homogenous regions + //g = g / gMag; + cv::Point2f d = x - c; // displacement + float n = std::sqrt(d.x * d.x + d.y * d.y); + if (n == 0) continue; + //d /= n; // normalize d + float e = d.x * g.y + d.y * g.x; + + //float dotProduct = d.y * g.x + d.x * g.y; + + //e = std::max(0.0f, e); + + //sum += dotProduct * dotProduct;// / (gray.rows * gray.cols); + + sum += (d * (e * e) - g * e * (n * n)) / (n * n * n * n); + } + } + + return sum; +} + +glm::vec2 eyeDirection(cv::Mat roi) { + cv::Mat gX, gY; + cv::Sobel(roi, gX, CV_32F, 1, 0); + cv::Sobel(roi, gY, CV_32F, 0, 1); + + float stepSize = roi.rows / 10; + float maxVal = 0; + + cv::Point2f irisPosition; + for(int i = 0; i < 32; i++) { + cv::Point2f c(std::rand() % roi.cols, std::rand() % roi.rows); //start at a random point + + float prevVal = 0; + for(int j = 0; j < 8; j++) { + cv::Point2f gradient = derivativeFunction(c, gX, gY); // calculate gradient + gradient /= std::sqrt(gradient.x * gradient.x + + gradient.y * gradient.y); // normalize + + for(int k = 0; k < 6; k++) { + cv::Point2f newC = c + gradient * stepSize; + if (newC.x < 0 || newC.x > roi.cols || newC.y < 0 || newC.y > roi.rows) continue; + + float newVal = objectiveFunction(newC, gX, gY); + if (newVal > prevVal) { + //c += sum * 3;// * 2 / (gray.rows * gray.cols); + c = newC; + prevVal = newVal; + break; + //cv::drawMarker(frame, c, cv::Scalar(0,0,255)); + } else { + stepSize /= 2; + } + } + } + + if(prevVal > maxVal) { + maxVal = prevVal; + irisPosition = cv::Point2f(c.y, c.x); // no idea why but the coordinates are flipped + } + } + + cv::drawMarker(roi, irisPosition, cv::Scalar(128,128,128)); + + // convert result to vector for graphics + return glm::vec2( + (irisPosition.x - roi.rows / 2.0) / (roi.rows / 2.0), + (irisPosition.y - roi.rows / 2.0) / (roi.rows / 2.0)); +} diff --git a/src/eye.hpp b/src/eye.hpp new file mode 100644 index 0000000..e437ce9 --- /dev/null +++ b/src/eye.hpp @@ -0,0 +1,10 @@ +#ifndef EYE_HPP +#define EYE_HPP + +#include + +#include + +glm::vec2 eyeDirection(cv::Mat roi); + +#endif