add eye tracking script

Image-Processing/Eye_Tracking/README.md

@@ -0,0 +1,20 @@
+# Eye Tracking
+
+![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)
+
+
+This is a Python (2 and 3) library that provides a **webcam-based eye tracking system**. It gives you the exact position of the pupils and the gaze direction, in real time.
+
+
+
+## Installation
+
+Install these dependencies (NumPy, OpenCV, Dlib):
+
+
+Run the demo:
+
+```
+python eye_tracking.py
+```
+

Image-Processing/Eye_Tracking/eye_tracking.py

@@ -0,0 +1,41 @@
+"""
+Demonstration of the GazeTracking library.
+Check the README.md for complete documentation.
+"""
+
+import cv2
+from gaze_tracking import GazeTracking
+
+gaze = GazeTracking()
+webcam = cv2.VideoCapture(0)
+
+while True:
+    # We get a new frame from the webcam
+    _, frame = webcam.read()
+
+    # We send this frame to GazeTracking to analyze it
+    gaze.refresh(frame)
+
+    frame = gaze.annotated_frame()
+    text = ""
+
+    if gaze.is_blinking():
+        text = "Blinking"
+    elif gaze.is_right():
+        text = "Looking right"
+    elif gaze.is_left():
+        text = "Looking left"
+    elif gaze.is_center():
+        text = "Looking center"
+
+    cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2)
+
+    left_pupil = gaze.pupil_left_coords()
+    right_pupil = gaze.pupil_right_coords()
+    cv2.putText(frame, "Left pupil:  " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1)
+    cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1)
+
+    cv2.imshow("Demo", frame)
+
+    if cv2.waitKey(1) == 27:
+        break

Image-Processing/Eye_Tracking/gaze_tracking/__init__.py

@@ -0,0 +1 @@
+from .gaze_tracking import GazeTracking

Image-Processing/Eye_Tracking/gaze_tracking/calibration.py

@@ -0,0 +1,77 @@
+from __future__ import division
+import cv2
+from .pupil import Pupil
+
+
+class Calibration(object):
+    """
+    This class calibrates the pupil detection algorithm by finding the
+    best binarization threshold value for the person and the webcam.
+    """
+
+    def __init__(self):
+        self.nb_frames = 20
+        self.thresholds_left = []
+        self.thresholds_right = []
+
+    def is_complete(self):
+        """Returns true if the calibration is completed"""
+        return len(self.thresholds_left) >= self.nb_frames and len(self.thresholds_right) >= self.nb_frames
+
+    def threshold(self, side):
+        """Returns the threshold value for the given eye.
+
+        Argument:
+            side: Indicates whether it's the left eye (0) or the right eye (1)
+        """
+        if side == 0:
+            return int(sum(self.thresholds_left) / len(self.thresholds_left))
+        elif side == 1:
+            return int(sum(self.thresholds_right) / len(self.thresholds_right))
+
+    @staticmethod
+    def iris_size(frame):
+        """Returns the percentage of space that the iris takes up on
+        the surface of the eye.
+
+        Argument:
+            frame (numpy.ndarray): Binarized iris frame
+        """
+        frame = frame[5:-5, 5:-5]
+        height, width = frame.shape[:2]
+        nb_pixels = height * width
+        nb_blacks = nb_pixels - cv2.countNonZero(frame)
+        return nb_blacks / nb_pixels
+
+    @staticmethod
+    def find_best_threshold(eye_frame):
+        """Calculates the optimal threshold to binarize the
+        frame for the given eye.
+
+        Argument:
+            eye_frame (numpy.ndarray): Frame of the eye to be analyzed
+        """
+        average_iris_size = 0.48
+        trials = {}
+
+        for threshold in range(5, 100, 5):
+            iris_frame = Pupil.image_processing(eye_frame, threshold)
+            trials[threshold] = Calibration.iris_size(iris_frame)
+
+        best_threshold, iris_size = min(trials.items(), key=(lambda p: abs(p[1] - average_iris_size)))
+        return best_threshold
+
+    def evaluate(self, eye_frame, side):
+        """Improves calibration by taking into consideration the
+        given image.
+
+        Arguments:
+            eye_frame (numpy.ndarray): Frame of the eye
+            side: Indicates whether it's the left eye (0) or the right eye (1)
+        """
+        threshold = self.find_best_threshold(eye_frame)
+
+        if side == 0:
+            self.thresholds_left.append(threshold)
+        elif side == 1:
+            self.thresholds_right.append(threshold)

Image-Processing/Eye_Tracking/gaze_tracking/eye.py

@@ -0,0 +1,117 @@
+import math
+import numpy as np
+import cv2
+from .pupil import Pupil
+
+
+class Eye(object):
+    """
+    This class creates a new frame to isolate the eye and
+    initiates the pupil detection.
+    """
+
+    LEFT_EYE_POINTS = [36, 37, 38, 39, 40, 41]
+    RIGHT_EYE_POINTS = [42, 43, 44, 45, 46, 47]
+
+    def __init__(self, original_frame, landmarks, side, calibration):
+        self.frame = None
+        self.origin = None
+        self.center = None
+        self.pupil = None
+
+        self._analyze(original_frame, landmarks, side, calibration)
+
+    @staticmethod
+    def _middle_point(p1, p2):
+        """Returns the middle point (x,y) between two points
+
+        Arguments:
+            p1 (dlib.point): First point
+            p2 (dlib.point): Second point
+        """
+        x = int((p1.x + p2.x) / 2)
+        y = int((p1.y + p2.y) / 2)
+        return (x, y)
+
+    def _isolate(self, frame, landmarks, points):
+        """Isolate an eye, to have a frame without other part of the face.
+
+        Arguments:
+            frame (numpy.ndarray): Frame containing the face
+            landmarks (dlib.full_object_detection): Facial landmarks for the face region
+            points (list): Points of an eye (from the 68 Multi-PIE landmarks)
+        """
+        region = np.array([(landmarks.part(point).x, landmarks.part(point).y) for point in points])
+        region = region.astype(np.int32)
+
+        # Applying a mask to get only the eye
+        height, width = frame.shape[:2]
+        black_frame = np.zeros((height, width), np.uint8)
+        mask = np.full((height, width), 255, np.uint8)
+        cv2.fillPoly(mask, [region], (0, 0, 0))
+        eye = cv2.bitwise_not(black_frame, frame.copy(), mask=mask)
+
+        # Cropping on the eye
+        margin = 5
+        min_x = np.min(region[:, 0]) - margin
+        max_x = np.max(region[:, 0]) + margin
+        min_y = np.min(region[:, 1]) - margin
+        max_y = np.max(region[:, 1]) + margin
+
+        self.frame = eye[min_y:max_y, min_x:max_x]
+        self.origin = (min_x, min_y)
+
+        height, width = self.frame.shape[:2]
+        self.center = (width / 2, height / 2)
+
+    def _blinking_ratio(self, landmarks, points):
+        """Calculates a ratio that can indicate whether an eye is closed or not.
+        It's the division of the width of the eye, by its height.
+
+        Arguments:
+            landmarks (dlib.full_object_detection): Facial landmarks for the face region
+            points (list): Points of an eye (from the 68 Multi-PIE landmarks)
+
+        Returns:
+            The computed ratio
+        """
+        left = (landmarks.part(points[0]).x, landmarks.part(points[0]).y)
+        right = (landmarks.part(points[3]).x, landmarks.part(points[3]).y)
+        top = self._middle_point(landmarks.part(points[1]), landmarks.part(points[2]))
+        bottom = self._middle_point(landmarks.part(points[5]), landmarks.part(points[4]))
+
+        eye_width = math.hypot((left[0] - right[0]), (left[1] - right[1]))
+        eye_height = math.hypot((top[0] - bottom[0]), (top[1] - bottom[1]))
+
+        try:
+            ratio = eye_width / eye_height
+        except ZeroDivisionError:
+            ratio = None
+
+        return ratio
+
+    def _analyze(self, original_frame, landmarks, side, calibration):
+        """Detects and isolates the eye in a new frame, sends data to the calibration
+        and initializes Pupil object.
+
+        Arguments:
+            original_frame (numpy.ndarray): Frame passed by the user
+            landmarks (dlib.full_object_detection): Facial landmarks for the face region
+            side: Indicates whether it's the left eye (0) or the right eye (1)
+            calibration (calibration.Calibration): Manages the binarization threshold value
+        """
+        if side == 0:
+            points = self.LEFT_EYE_POINTS
+        elif side == 1:
+            points = self.RIGHT_EYE_POINTS
+        else:
+            return
+
+        self.blinking = self._blinking_ratio(landmarks, points)
+        self._isolate(original_frame, landmarks, points)
+
+        if not calibration.is_complete():
+            calibration.evaluate(self.frame, side)
+
+        threshold = calibration.threshold(side)
+        self.pupil = Pupil(self.frame, threshold)

Image-Processing/Eye_Tracking/gaze_tracking/gaze_tracking.py

@@ -0,0 +1,133 @@
+from __future__ import division
+import os
+import cv2
+import dlib
+from .eye import Eye
+from .calibration import Calibration
+
+
+class GazeTracking(object):
+    """
+    This class tracks the user's gaze.
+    It provides useful information like the position of the eyes
+    and pupils and allows to know if the eyes are open or closed
+    """
+
+    def __init__(self):
+        self.frame = None
+        self.eye_left = None
+        self.eye_right = None
+        self.calibration = Calibration()
+
+        # _face_detector is used to detect faces
+        self._face_detector = dlib.get_frontal_face_detector()
+
+        # _predictor is used to get facial landmarks of a given face
+        cwd = os.path.abspath(os.path.dirname(__file__))
+        model_path = os.path.abspath(os.path.join(cwd, "trained_models/shape_predictor_68_face_landmarks.dat"))
+        self._predictor = dlib.shape_predictor(model_path)
+
+    @property
+    def pupils_located(self):
+        """Check that the pupils have been located"""
+        try:
+            int(self.eye_left.pupil.x)
+            int(self.eye_left.pupil.y)
+            int(self.eye_right.pupil.x)
+            int(self.eye_right.pupil.y)
+            return True
+        except Exception:
+            return False
+
+    def _analyze(self):
+        """Detects the face and initialize Eye objects"""
+        frame = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
+        faces = self._face_detector(frame)
+
+        try:
+            landmarks = self._predictor(frame, faces[0])
+            self.eye_left = Eye(frame, landmarks, 0, self.calibration)
+            self.eye_right = Eye(frame, landmarks, 1, self.calibration)
+
+        except IndexError:
+            self.eye_left = None
+            self.eye_right = None
+
+    def refresh(self, frame):
+        """Refreshes the frame and analyzes it.
+
+        Arguments:
+            frame (numpy.ndarray): The frame to analyze
+        """
+        self.frame = frame
+        self._analyze()
+
+    def pupil_left_coords(self):
+        """Returns the coordinates of the left pupil"""
+        if self.pupils_located:
+            x = self.eye_left.origin[0] + self.eye_left.pupil.x
+            y = self.eye_left.origin[1] + self.eye_left.pupil.y
+            return (x, y)
+
+    def pupil_right_coords(self):
+        """Returns the coordinates of the right pupil"""
+        if self.pupils_located:
+            x = self.eye_right.origin[0] + self.eye_right.pupil.x
+            y = self.eye_right.origin[1] + self.eye_right.pupil.y
+            return (x, y)
+
+    def horizontal_ratio(self):
+        """Returns a number between 0.0 and 1.0 that indicates the
+        horizontal direction of the gaze. The extreme right is 0.0,
+        the center is 0.5 and the extreme left is 1.0
+        """
+        if self.pupils_located:
+            pupil_left = self.eye_left.pupil.x / (self.eye_left.center[0] * 2 - 10)
+            pupil_right = self.eye_right.pupil.x / (self.eye_right.center[0] * 2 - 10)
+            return (pupil_left + pupil_right) / 2
+
+    def vertical_ratio(self):
+        """Returns a number between 0.0 and 1.0 that indicates the
+        vertical direction of the gaze. The extreme top is 0.0,
+        the center is 0.5 and the extreme bottom is 1.0
+        """
+        if self.pupils_located:
+            pupil_left = self.eye_left.pupil.y / (self.eye_left.center[1] * 2 - 10)
+            pupil_right = self.eye_right.pupil.y / (self.eye_right.center[1] * 2 - 10)
+            return (pupil_left + pupil_right) / 2
+
+    def is_right(self):
+        """Returns true if the user is looking to the right"""
+        if self.pupils_located:
+            return self.horizontal_ratio() <= 0.35
+
+    def is_left(self):
+        """Returns true if the user is looking to the left"""
+        if self.pupils_located:
+            return self.horizontal_ratio() >= 0.65
+
+    def is_center(self):
+        """Returns true if the user is looking to the center"""
+        if self.pupils_located:
+            return self.is_right() is not True and self.is_left() is not True
+
+    def is_blinking(self):
+        """Returns true if the user closes his eyes"""
+        if self.pupils_located:
+            blinking_ratio = (self.eye_left.blinking + self.eye_right.blinking) / 2
+            return blinking_ratio > 3.8
+
+    def annotated_frame(self):
+        """Returns the main frame with pupils highlighted"""
+        frame = self.frame.copy()
+
+        if self.pupils_located:
+            color = (0, 255, 0)
+            x_left, y_left = self.pupil_left_coords()
+            x_right, y_right = self.pupil_right_coords()
+            cv2.line(frame, (x_left - 5, y_left), (x_left + 5, y_left), color)
+            cv2.line(frame, (x_left, y_left - 5), (x_left, y_left + 5), color)
+            cv2.line(frame, (x_right - 5, y_right), (x_right + 5, y_right), color)
+            cv2.line(frame, (x_right, y_right - 5), (x_right, y_right + 5), color)
+
+        return frame