CV: Add eye look direction

CMakeLists.txt

@@ -61,6 +61,7 @@ add_executable( fc2d
 	src/toml.c
 	src/tomlcpp.cpp
 	src/error.cpp
+	src/eye.cpp
 )
 target_link_libraries( fc2d ${OpenCV_LIBS} ${OPENGL_LIBRARIES} ${WEBP_LIBRARIES}
 	FreeGLUT::freeglut GLEW::glew zip Boxer fmt )

src/cv.cpp

@@ -7,6 +7,7 @@
 #include <paths.hpp>
 #include <args.hpp>
 #include <cv.hpp>
+#include <eye.hpp>
 #include <modelpart.hpp>
 
 cv::Ptr<cv::face::Facemark> facemark;
@@ -98,9 +99,9 @@ void cvFrame() {
 	std::vector<std::vector<cv::Point2f>> landmarks;
 
 	if (facemark->fit (frame, faces, landmarks)) {
-		//for (int i = 0; i < landmarks[biggestFace].size(); i++) {
-		//	cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
-		//}
+		for (int i = 0; i < landmarks[biggestFace].size(); i++) {
+			cv::circle (frame, landmarks[biggestFace][i], 2, cv::Scalar (255, 255, 255));
+		}
 		cv::circle(frame, cv::Point2f(
 			(landmarks[biggestFace][2].x + landmarks[biggestFace][14].x) / 2,
 			(landmarks[biggestFace][2].y + landmarks[biggestFace][14].y) / 2
@@ -109,6 +110,20 @@ void cvFrame() {
 		cv::circle (frame, landmarks[biggestFace][66], 3, cv::Scalar (0, 255, 0));
 		cv::circle (frame, landmarks[biggestFace][62], 3, cv::Scalar (0, 255, 0));
 
+		//get ROI for eyes
+		float eyeWidth = landmarks[biggestFace][45].x - landmarks[biggestFace][42].x;
+		cv::Rect eyeRect(landmarks[biggestFace][42].x, landmarks[biggestFace][42].y - eyeWidth / 2, eyeWidth, eyeWidth);
+
+		cv::rectangle(frame, eyeRect, cv::Scalar(255, 255, 255));
+
+		cv::Mat eyeROI;
+		eyeROI = gray(eyeRect);
+
+		glm::vec2 eyeVector = eyeDirection(eyeROI);
+
+		cv::imshow("eye", eyeROI);
+		cv::waitKey(1);
+
 		//send control information to graphics
 		float faceSize = landmarks[biggestFace][14].x - landmarks[biggestFace][2].x;
 
@@ -124,6 +139,7 @@ void cvFrame() {
 			landmarks[biggestFace][30].x * 2 / (float)frame.cols - 1,
 			landmarks[biggestFace][30].y * 2 / (float)frame.rows - 1
 			);
+
 		faceData.triggers[TRIGGER_NULL] = false;
 		faceData.triggers[TRIGGER_MOUTH_OPEN] = 
 		(landmarks[biggestFace][66].y - landmarks[biggestFace][62].y) / faceSize > 0.04f;
@@ -133,6 +149,11 @@ void cvFrame() {
 			(float)(landmarks[biggestFace][2].x - landmarks[biggestFace][14].x));
 		faceData.scale = faceSize * 6 / (float)frame.cols;
 
+
+		cv::line(frame, cv::Point(50,50), cv::Point(50,50) + cv::Point(eyeVector.x * 25, eyeVector.y * 25), cv::Scalar(255,128,128));
+		std::cout << eyeVector.x << "," << eyeVector.y << std::endl;
+		std::cout << "SIZE:" << eyeROI.cols << "," << eyeROI.rows << std::endl;
+
 		updateModel(faceData);
 	}
 }

src/eye.cpp

@@ -0,0 +1,104 @@
+#include <eye.hpp>
+
+
+float objectiveFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
+	float sum = 0;
+	for(int xx = 0; xx < gX.cols; xx++) {
+		for(int xy = 0; xy < gX.rows; xy++) {
+			cv::Point2f x(xx, xy);
+			cv::Point2f g(gX.at<float>(xx, xy), gY.at<float>(xx, xy)); // gradient
+			double gMag = std::sqrt(g.x * g.x + g.y * g.y);
+			if(gMag < 200.0) continue;      // ignore gradients in homogenous regions
+			//g = g / gMag;
+			cv::Point2f d = x - c;  // displacement
+			d /= std::sqrt(d.x * d.x + d.y * d.y);  // normalize d
+
+			float dotProduct = d.y * g.x + d.x * g.y;
+
+			dotProduct = std::max(0.0f, dotProduct);
+
+			sum += dotProduct * dotProduct;
+		}
+	}
+	sum /= gX.rows * gX.cols;
+
+	return sum;
+}
+
+cv::Point2f derivativeFunction(cv::Point2f c, cv::Mat gX, cv::Mat gY) {
+	cv::Point2f sum(0,0);
+
+	for(int xx = 0; xx < gX.cols; xx++) {
+		for(int xy = 0; xy < gX.rows; xy++) {
+			cv::Point2f x(xy, xx);
+			cv::Point2f g(gY.at<float>(xx, xy), gX.at<float>(xx, xy)); // gradient
+			float gMag = std::sqrt(g.x * g.x + g.y * g.y);
+			if(gMag < 200.0f) continue;	// ignore gradients in homogenous regions
+			//g = g / gMag;
+			cv::Point2f d = x - c;	// displacement
+			float n = std::sqrt(d.x * d.x + d.y * d.y);
+			if (n == 0) continue;
+			//d /= n;	// normalize d
+			float e = d.x * g.y + d.y * g.x;
+
+			//float dotProduct = d.y * g.x + d.x * g.y;
+
+			//e = std::max(0.0f, e);
+
+			//sum += dotProduct * dotProduct;// / (gray.rows * gray.cols);
+
+			sum += (d * (e * e) - g * e * (n * n)) / (n * n * n * n);
+		}
+	}
+
+	return sum;
+}
+
+glm::vec2 eyeDirection(cv::Mat roi) {
+	cv::Mat gX, gY;
+	cv::Sobel(roi, gX, CV_32F, 1, 0);
+	cv::Sobel(roi, gY, CV_32F, 0, 1);
+
+	float stepSize = roi.rows / 10;
+	float maxVal = 0;
+
+	cv::Point2f irisPosition;
+	for(int i = 0; i < 32; i++) {
+		cv::Point2f c(std::rand() % roi.cols, std::rand() % roi.rows); //start at a random point
+		
+		float prevVal = 0;
+		for(int j = 0; j < 8; j++) {
+			cv::Point2f gradient = derivativeFunction(c, gX, gY);	// calculate gradient
+			gradient /= std::sqrt(gradient.x * gradient.x +
+					gradient.y * gradient.y);		// normalize
+
+			for(int k = 0; k < 6; k++) {
+				cv::Point2f newC = c + gradient * stepSize;
+				if (newC.x < 0 || newC.x > roi.cols || newC.y < 0 || newC.y > roi.rows) continue;
+
+				float newVal = objectiveFunction(newC, gX, gY);
+				if (newVal > prevVal) {
+					//c += sum * 3;// * 2 / (gray.rows * gray.cols);
+					c = newC;
+					prevVal = newVal;
+					break;
+					//cv::drawMarker(frame, c, cv::Scalar(0,0,255));
+				} else {
+					stepSize /= 2;
+				}
+			}
+		}
+
+		if(prevVal > maxVal) {
+			maxVal = prevVal;
+			irisPosition = cv::Point2f(c.y, c.x); // no idea why but the coordinates are flipped
+		}
+	}
+
+	cv::drawMarker(roi, irisPosition, cv::Scalar(128,128,128));
+
+	// convert result to vector for graphics
+	return glm::vec2(
+			(irisPosition.x - roi.rows / 2.0) / (roi.rows / 2.0),
+			(irisPosition.y - roi.rows / 2.0) / (roi.rows / 2.0));
+}

src/eye.hpp

@@ -0,0 +1,10 @@
+#ifndef EYE_HPP
+#define EYE_HPP
+
+#include <opencv2/opencv.hpp>
+
+#include<glm/vec2.hpp>
+
+glm::vec2 eyeDirection(cv::Mat roi);
+
+#endif