CV: Add eye look direction
This commit is contained in:
parent
6038cb7870
commit
60c5254a47
|
@ -61,6 +61,7 @@ add_executable( fc2d
|
|||
src/toml.c
|
||||
src/tomlcpp.cpp
|
||||
src/error.cpp
|
||||
src/eye.cpp
|
||||
)
|
||||
target_link_libraries( fc2d ${OpenCV_LIBS} ${OPENGL_LIBRARIES} ${WEBP_LIBRARIES}
|
||||
FreeGLUT::freeglut GLEW::glew zip Boxer fmt )
|
||||
|
|
27
src/cv.cpp
27
src/cv.cpp
|
@ -7,6 +7,7 @@
|
|||
#include <paths.hpp>
|
||||
#include <args.hpp>
|
||||
#include <cv.hpp>
|
||||
#include <eye.hpp>
|
||||
#include <modelpart.hpp>
|
||||
|
||||
cv::Ptr<cv::face::Facemark> facemark;
|
||||
|
@ -98,9 +99,9 @@ void cvFrame() {
|
|||
std::vector<std::vector<cv::Point2f>> landmarks;
|
||||
|
||||
if (facemark->fit (frame, faces, landmarks)) {
|
||||
//for (int i = 0; i < landmarks[biggestFace].size(); i++) {
|
||||
// cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
|
||||
//}
|
||||
for (int i = 0; i < landmarks[biggestFace].size(); i++) {
|
||||
cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
|
||||
}
|
||||
cv::circle(frame, cv::Point2f(
|
||||
(landmarks[biggestFace][2].x + landmarks[biggestFace][14].x) / 2,
|
||||
(landmarks[biggestFace][2].y + landmarks[biggestFace][14].y) / 2
|
||||
|
@ -109,6 +110,20 @@ void cvFrame() {
|
|||
cv::circle (frame, landmarks[biggestFace][66], 3, cv::Scalar (0, 255, 0));
|
||||
cv::circle (frame, landmarks[biggestFace][62], 3, cv::Scalar (0, 255, 0));
|
||||
|
||||
//get ROI for eyes
|
||||
float eyeWidth = landmarks[biggestFace][45].x - landmarks[biggestFace][42].x;
|
||||
cv::Rect eyeRect(landmarks[biggestFace][42].x, landmarks[biggestFace][42].y - eyeWidth / 2, eyeWidth, eyeWidth);
|
||||
|
||||
cv::rectangle(frame, eyeRect, cv::Scalar(255, 255, 255));
|
||||
|
||||
cv::Mat eyeROI;
|
||||
eyeROI = gray(eyeRect);
|
||||
|
||||
glm::vec2 eyeVector = eyeDirection(eyeROI);
|
||||
|
||||
cv::imshow("eye", eyeROI);
|
||||
cv::waitKey(1);
|
||||
|
||||
//send control information to graphics
|
||||
float faceSize = landmarks[biggestFace][14].x - landmarks[biggestFace][2].x;
|
||||
|
||||
|
@ -124,6 +139,7 @@ void cvFrame() {
|
|||
landmarks[biggestFace][30].x * 2 / (float)frame.cols - 1,
|
||||
landmarks[biggestFace][30].y * 2 / (float)frame.rows - 1
|
||||
);
|
||||
|
||||
faceData.triggers[TRIGGER_NULL] = false;
|
||||
faceData.triggers[TRIGGER_MOUTH_OPEN] =
|
||||
(landmarks[biggestFace][66].y - landmarks[biggestFace][62].y) / faceSize > 0.04f;
|
||||
|
@ -133,6 +149,11 @@ void cvFrame() {
|
|||
(float)(landmarks[biggestFace][2].x - landmarks[biggestFace][14].x));
|
||||
faceData.scale = faceSize * 6 / (float)frame.cols;
|
||||
|
||||
|
||||
cv::line(frame, cv::Point(50,50), cv::Point(50,50) + cv::Point(eyeVector.x * 25, eyeVector.y * 25), cv::Scalar(255,128,128));
|
||||
std::cout << eyeVector.x << "," << eyeVector.y << std::endl;
|
||||
std::cout << "SIZE:" << eyeROI.cols << "," << eyeROI.rows << std::endl;
|
||||
|
||||
updateModel(faceData);
|
||||
}
|
||||
}
|
||||
|
|
104
src/eye.cpp
Normal file
104
src/eye.cpp
Normal file
|
@ -0,0 +1,104 @@
|
|||
#include <eye.hpp>
|
||||
|
||||
|
||||
float objectiveFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
|
||||
float sum = 0;
|
||||
for(int xx = 0; xx < gX.cols; xx++) {
|
||||
for(int xy = 0; xy < gX.rows; xy++) {
|
||||
cv::Point2f x(xx, xy);
|
||||
cv::Point2f g(gX.at<float>(xx, xy), gY.at<float>(xx, xy)); // gradient
|
||||
double gMag = std::sqrt(g.x * g.x + g.y * g.y);
|
||||
if(gMag < 200.0) continue; // ignore gradients in homogenous regions
|
||||
//g = g / gMag;
|
||||
cv::Point2f d = x - c; // displacement
|
||||
d /= std::sqrt(d.x * d.x + d.y * d.y); // normalize d
|
||||
|
||||
float dotProduct = d.y * g.x + d.x * g.y;
|
||||
|
||||
dotProduct = std::max(0.0f, dotProduct);
|
||||
|
||||
sum += dotProduct * dotProduct;
|
||||
}
|
||||
}
|
||||
sum /= gX.rows * gX.cols;
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
cv::Point2f derivativeFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
|
||||
cv::Point2f sum(0,0);
|
||||
|
||||
for(int xx = 0; xx < gX.cols; xx++) {
|
||||
for(int xy = 0; xy < gX.rows; xy++) {
|
||||
cv::Point2f x(xy, xx);
|
||||
cv::Point2f g(gY.at<float>(xx, xy), gX.at<float>(xx, xy)); // gradient
|
||||
float gMag = std::sqrt(g.x * g.x + g.y * g.y);
|
||||
if(gMag < 200.0f) continue; // ignore gradients in homogenous regions
|
||||
//g = g / gMag;
|
||||
cv::Point2f d = x - c; // displacement
|
||||
float n = std::sqrt(d.x * d.x + d.y * d.y);
|
||||
if (n == 0) continue;
|
||||
//d /= n; // normalize d
|
||||
float e = d.x * g.y + d.y * g.x;
|
||||
|
||||
//float dotProduct = d.y * g.x + d.x * g.y;
|
||||
|
||||
//e = std::max(0.0f, e);
|
||||
|
||||
//sum += dotProduct * dotProduct;// / (gray.rows * gray.cols);
|
||||
|
||||
sum += (d * (e * e) - g * e * (n * n)) / (n * n * n * n);
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
glm::vec2 eyeDirection(cv::Mat roi) {
|
||||
cv::Mat gX, gY;
|
||||
cv::Sobel(roi, gX, CV_32F, 1, 0);
|
||||
cv::Sobel(roi, gY, CV_32F, 0, 1);
|
||||
|
||||
float stepSize = roi.rows / 10;
|
||||
float maxVal = 0;
|
||||
|
||||
cv::Point2f irisPosition;
|
||||
for(int i = 0; i < 32; i++) {
|
||||
cv::Point2f c(std::rand() % roi.cols, std::rand() % roi.rows); //start at a random point
|
||||
|
||||
float prevVal = 0;
|
||||
for(int j = 0; j < 8; j++) {
|
||||
cv::Point2f gradient = derivativeFunction(c, gX, gY); // calculate gradient
|
||||
gradient /= std::sqrt(gradient.x * gradient.x +
|
||||
gradient.y * gradient.y); // normalize
|
||||
|
||||
for(int k = 0; k < 6; k++) {
|
||||
cv::Point2f newC = c + gradient * stepSize;
|
||||
if (newC.x < 0 || newC.x > roi.cols || newC.y < 0 || newC.y > roi.rows) continue;
|
||||
|
||||
float newVal = objectiveFunction(newC, gX, gY);
|
||||
if (newVal > prevVal) {
|
||||
//c += sum * 3;// * 2 / (gray.rows * gray.cols);
|
||||
c = newC;
|
||||
prevVal = newVal;
|
||||
break;
|
||||
//cv::drawMarker(frame, c, cv::Scalar(0,0,255));
|
||||
} else {
|
||||
stepSize /= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(prevVal > maxVal) {
|
||||
maxVal = prevVal;
|
||||
irisPosition = cv::Point2f(c.y, c.x); // no idea why but the coordinates are flipped
|
||||
}
|
||||
}
|
||||
|
||||
cv::drawMarker(roi, irisPosition, cv::Scalar(128,128,128));
|
||||
|
||||
// convert result to vector for graphics
|
||||
return glm::vec2(
|
||||
(irisPosition.x - roi.rows / 2.0) / (roi.rows / 2.0),
|
||||
(irisPosition.y - roi.rows / 2.0) / (roi.rows / 2.0));
|
||||
}
|
10
src/eye.hpp
Normal file
10
src/eye.hpp
Normal file
|
@ -0,0 +1,10 @@
|
|||
#ifndef EYE_HPP
|
||||
#define EYE_HPP
|
||||
|
||||
#include <opencv2/opencv.hpp>
|
||||
|
||||
#include<glm/vec2.hpp>
|
||||
|
||||
glm::vec2 eyeDirection(cv::Mat roi);
|
||||
|
||||
#endif
|
Loading…
Reference in a new issue