Hand detect ex

bindings/python/examples/gesture_rec/install_deps.sh

@@ -0,0 +1,11 @@
+
+#!/bin/bash
+
+sudo apt-get install -y python3-tk
+sudo apt-get install -y python3-matplotlib
+sudo apt-get install -y python3-skimage
+sudo apt-get install -y python3-pil.imagetk
+sudo apt-get install -y python3-pil
+cp notebook.py /home/analog/Desktop
+cp process.py /home/analog/Desktop
+

bindings/python/examples/gesture_rec/notebook.py

@@ -0,0 +1,149 @@
+from tkinter import *
+from tkinter import ttk
+import aditofpython as tof
+import numpy as np
+import cv2 as cv
+from enum import Enum
+from process import ProcessTab
+from PIL import Image, ImageTk
+
+
+class ModesEnum(Enum):
+    MODE_NEAR = 0
+    MODE_MEDIUM = 1
+    MODE_FAR = 2
+
+
+smallSignalThreshold = 100
+
+
+class GestureDemo(Frame):
+
+    def __init__(self, isapp=True, name='gesturedemo'):
+        Frame.__init__(self, name=name)
+        self.pack(expand=Y, fill=BOTH)
+        self.master.title('Gesture Demo')
+        self.isapp = isapp
+        self._create_widgets()
+
+    def _create_widgets(self):
+        self._create_main_panel()
+
+    def _create_main_panel(self):
+
+        mainPanel = Frame(self, name='demo')
+        mainPanel.pack(side=TOP, fill=BOTH, expand=Y)
+
+        # create the notebook
+        nb = ttk.Notebook(mainPanel, name='gesturedemo')
+
+        nb.pack(fill=BOTH, expand=Y, padx=2, pady=3)
+        self._create_video_tab(nb)
+        self._create_process_tab(nb)
+
+    # =============================================================================
+    def _create_video_tab(self, nb):
+        # frame to hold contentx
+        frame = Frame(nb, name='video')
+
+        # widgets to be displayed on Video tab
+        msg = ["Capture an image for processing"]
+        lbl = Label(frame, justify=LEFT, anchor=N,
+                    text=''.join(msg))
+        lbl_frame = LabelFrame(frame, bg='red')
+        self.lbl_vid = Label(lbl_frame, bg='blue')
+        btn = Button(frame, text='Init. Dev', underline=0,
+                     command=lambda: self._init_dev())
+        btn_start = Button(frame, text='Start Video', underline=0,
+                           command=lambda: self._start_video())
+        btn_capture = Button(frame, text="Snap!", command=lambda: self._capture_img())
+
+        # position and set resize behaviour
+        lbl.grid(row=0, column=0)
+        lbl_frame.grid(row=0, column=1, columnspan=4)
+        self.lbl_vid.grid(row=0, column=1, columnspan=4)
+        btn.grid(row=1, column=0, pady=(2, 4))
+        btn_start.grid(row=2, column=0, pady=(2, 4))
+        btn_capture.grid(row=3, column=0, pady=(2, 4))
+        nb.add(frame, text='Video', padding=2)
+
+    # =============================================================================
+    def _init_dev(self):
+        system = tof.System()
+        print(system)
+
+        self.cameras = []
+
+        status = system.getCameraList(self.cameras)
+        if not status:
+            print("system.getCameraList failed with status: ", status)
+
+        status = self.cameras[0].initialize()
+        if not status:
+            print("cameras[0].initialize() failed with status: ", status)
+
+        modes = []
+        status = self.cameras[0].getAvailableModes(modes)
+        if not status:
+            print("system.getAvailableModes() failed with status: ", status)
+
+        types = []
+        status = self.cameras[0].getAvailableFrameTypes(types)
+        if not status:
+            print("system.getAvailableFrameTypes() failed with status: ", status)
+
+        status = self.cameras[0].setFrameType(types[0])
+        if not status:
+            print("cameras[0].setFrameType() failed with status:", status)
+
+        status = self.cameras[0].setMode(modes[ModesEnum.MODE_NEAR.value])
+        if not status:
+            print("cameras[0].setMode() failed with status: ", status)
+
+    # =============================================================================
+    def _start_video(self):
+        camDetails = tof.CameraDetails()
+        status = self.cameras[0].getDetails(camDetails)
+        if not status:
+            print("system.getDetails() failed with status: ", status)
+
+        # Enable noise reduction for better results
+        smallSignalThreshold = 100
+        self.cameras[0].setControl("noise_reduction_threshold", str(smallSignalThreshold))
+
+        camera_range = camDetails.depthParameters.maxDepth
+        distance_scale = 255.0 / camera_range
+        tof_frame = tof.Frame()
+        while True:
+            # Capture frame-by-frame
+            status = self.cameras[0].requestFrame(tof_frame)
+            if not status:
+                print("cameras[0].requestFrame() failed with status: ", status)
+
+            depth_map = np.array(tof_frame.getData(tof.FrameDataType.Depth), dtype="uint16", copy=False)
+            # Creation of the Depth image
+            depth_map = depth_map[0: 480, 0:640]
+            depth_map = cv.flip(depth_map, 1)
+            depth_map = distance_scale * depth_map
+            depth_map = np.uint8(depth_map)
+            depth_map = cv.applyColorMap(depth_map, cv.COLORMAP_RAINBOW)
+            depth_map = cv.cvtColor(depth_map, cv.COLOR_BGR2RGB)
+            self.img_obj = Image.fromarray(depth_map)
+            img = ImageTk.PhotoImage(self.img_obj.resize((640, 480)))
+            self.lbl_vid['image'] = img
+            self.update()
+
+    # =============================================================================
+    def _capture_img(self):
+        file_name = "test.png"
+        self.img_obj.resize((640, 480)).save(file_name)
+
+    # =============================================================================
+    def _create_process_tab(self, nb):
+        # add the process tab to main panel
+        frame = ProcessTab(nb)
+        nb.add(frame, text='Process', underline=0)
+
+
+if __name__ == '__main__':
+    GestureDemo().mainloop()

bindings/python/examples/gesture_rec/process.py

@@ -0,0 +1,134 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+from tkinter import *
+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
+from skimage.color import rgb2hsv
+from skimage import measure
+from scipy.spatial import ConvexHull, convex_hull_plot_2d
+from scipy.ndimage import distance_transform_edt
+from scipy import ndimage
+
+hue_threshold = 0.3
+
+
+class ProcessTab(Frame):
+
+    def __init__(self, parent, *args, **kwargs):
+        Frame.__init__(self, parent, *args, **kwargs)
+        self._create_widgets()
+
+    def _create_widgets(self):
+        self._add_canvases()
+        btn_load = Button(self, text='Load',
+                          command=lambda: self._load_img())
+        self.resultVar = StringVar()
+        self.resultVar.set("How many fingers?")
+        btn_process = Button(self, text='Process',
+                             command=lambda v=self.resultVar: self._display_result(v))
+        lbl_result = Label(self, textvariable=self.resultVar, name='result')
+        btn_load.grid(row=0, column=2, pady=(2, 4))
+        btn_process.grid(row=1, column=2, pady=(2, 4))
+        lbl_result.grid(row=2, column=2)
+
+    def _add_canvases(self):
+        # input image canvas
+        fig_input, self.ax_input = plt.subplots(figsize=(4, 3))
+        self.ax_input.set_title("Depth image")
+        self.input_canvas = FigureCanvasTkAgg(fig_input, master=self)
+        self.input_canvas.get_tk_widget().grid(row=1, column=0)
+        # result image canvas
+        fig_result, self.ax_result = plt.subplots(figsize=(4, 3))
+        self.ax_result.set_title("Detected Hand and Fingers")
+        self.result_canvas = FigureCanvasTkAgg(fig_result, master=self)
+        self.result_canvas.get_tk_widget().grid(row=1, column=1)
+        # process plots canvas
+        fig, (self.ax1, self.ax2) = plt.subplots(ncols=2, figsize=(8, 3))
+        self.ax1.set_title("Histogram of the Hue channel with threshold")
+        self.ax1.set_ylim([0, 3000])
+        self.ax1.axvline(x=hue_threshold, color='r', linestyle='dashed', linewidth=2)
+        self.ax2.set_title("Hue-thresholded image")
+        self.ax2.axis('off')
+        self.canvas = FigureCanvasTkAgg(fig, master=self)
+        self.canvas.get_tk_widget().grid(row=2, column=0, columnspan=2)
+
+    # =============================================================================
+    def _load_img(self):
+        self.rgb_img = mpimg.imread('test.png')
+        self.ax_input.imshow(self.rgb_img)
+        self.input_canvas.draw()
+
+    # =============================================================================
+    def _display_result(self, v):
+        self._hue_ch_hist()
+        self._detect_hand()
+        self._count_fingers()
+
+    # =============================================================================
+    def _hue_ch_hist(self):
+        # Remove the background
+        hsv_img = rgb2hsv(self.rgb_img)
+        hue_img = hsv_img[:, :, 0]
+        self.binary_img = (hue_img < hue_threshold) * 1
+        self.ax1.hist(hue_img.ravel(), 512)
+        self.ax2.imshow(self.binary_img, cmap='gray')
+        self.canvas.draw()
+
+    # =============================================================================
+    def _detect_hand(self):
+        # Find all the objects in the image and select the largest one, which is the hand
+        labels = measure.label(self.binary_img)
+        props = measure.regionprops(labels)
+        props.reverse()
+        max_area = props[0]
+        points = props[0].filled_image
+        points_hull = np.where(points == 1)
+        self.cord = list(zip(points_hull[0], points_hull[1]))
+        self.dist_map = distance_transform_edt(points)
+        self.hand_center = ndimage.center_of_mass(self.dist_map)
+        self.radius = 1.75 * np.max(self.dist_map)
+
+    # =============================================================================
+    def _count_fingers(self):
+        # Find the convex hull = contour of the hand with extremities
+        hull = ConvexHull(self.cord)
+        cord_arr = np.array(self.cord)
+        vertices = cord_arr[hull.vertices]
+        vertices = vertices[1:len(vertices) - 2]
+
+        dist = vertices[0:len(vertices) - 1] - vertices[1:len(vertices)]
+        cdist = np.sqrt(dist[:, 0] ** 2 + dist[:, 1] ** 2)
+        cdist = (cdist < 15) * 1
+        cdist = cdist[0:len(cdist) - 1] - cdist[1:len(cdist)]
+        if cdist[0] != 0:
+            cdist[0] = -1
+        if cdist[len(cdist) - 1] != 0:
+            cdist[len(cdist) - 1] = -1
+        dist_idx = np.where(cdist == -1)
+        dist_idx = np.array(dist_idx) + 1
+
+        # Compute the finger tips distances to the center of the hand and detect the gesture
+        finger_dist = np.array(vertices[dist_idx]) - self.hand_center
+        for fd in finger_dist:
+            finger_cdist = np.sqrt(fd[:, 0] ** 2 + fd[:, 1] ** 2)
+        self.fingers = np.where(finger_cdist > self.radius)
+        self.detect_gesture()
+
+        self.ax_result.set_title("Detected hand and fingers")
+        self.ax_result.imshow(self.dist_map, cmap='gray')
+        self.ax_result.plot(self.hand_center[1], self.hand_center[0], "o", mec='g', color='none', markersize=self.radius)
+        self.ax_result.plot(vertices[dist_idx, 1], vertices[dist_idx, 0], 'o', mec='r', color='none', lw=1, markersize=10)
+        self.ax_result.plot(vertices[:, 1], vertices[:, 0], 'x', mec='g', color='none', lw=1, markersize=4)
+        self.result_canvas.draw()
+
+    # =============================================================================
+    def detect_gesture(self):
+
+        if len(self.fingers[0]) == 5:
+            self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers.Paper")
+        elif self.fingers == 2 or self.fingers == 3:
+            self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Scissors")
+        elif self.fingers == 0:
+            self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Rock")
+        else:
+            self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Unknown gesture")