diff --git a/bindings/python/examples/gesture_rec/install_deps.sh b/bindings/python/examples/gesture_rec/install_deps.sh new file mode 100644 index 00000000..85cf971d --- /dev/null +++ b/bindings/python/examples/gesture_rec/install_deps.sh @@ -0,0 +1,11 @@ + +#!/bin/bash + +sudo apt-get install -y python3-tk +sudo apt-get install -y python3-matplotlib +sudo apt-get install -y python3-skimage +sudo apt-get install -y python3-pil.imagetk +sudo apt-get install -y python3-pil +cp notebook.py /home/analog/Desktop +cp process.py /home/analog/Desktop + diff --git a/bindings/python/examples/gesture_rec/notebook.py b/bindings/python/examples/gesture_rec/notebook.py new file mode 100644 index 00000000..47e68eb1 --- /dev/null +++ b/bindings/python/examples/gesture_rec/notebook.py @@ -0,0 +1,149 @@ +from tkinter import * +from tkinter import ttk +import aditofpython as tof +import numpy as np +import cv2 as cv +from enum import Enum +from process import ProcessTab +from PIL import Image, ImageTk + + +class ModesEnum(Enum): + MODE_NEAR = 0 + MODE_MEDIUM = 1 + MODE_FAR = 2 + + +smallSignalThreshold = 100 + + +class GestureDemo(Frame): + + def __init__(self, isapp=True, name='gesturedemo'): + Frame.__init__(self, name=name) + self.pack(expand=Y, fill=BOTH) + self.master.title('Gesture Demo') + self.isapp = isapp + self._create_widgets() + + def _create_widgets(self): + self._create_main_panel() + + def _create_main_panel(self): + + mainPanel = Frame(self, name='demo') + mainPanel.pack(side=TOP, fill=BOTH, expand=Y) + + # create the notebook + nb = ttk.Notebook(mainPanel, name='gesturedemo') + + nb.pack(fill=BOTH, expand=Y, padx=2, pady=3) + self._create_video_tab(nb) + self._create_process_tab(nb) + + # ============================================================================= + def _create_video_tab(self, nb): + # frame to hold contentx + frame = Frame(nb, name='video') + + # widgets to be displayed on Video tab + msg = ["Capture an image for processing"] + lbl = Label(frame, justify=LEFT, anchor=N, + text=''.join(msg)) + lbl_frame = LabelFrame(frame, bg='red') + self.lbl_vid = Label(lbl_frame, bg='blue') + btn = Button(frame, text='Init. Dev', underline=0, + command=lambda: self._init_dev()) + btn_start = Button(frame, text='Start Video', underline=0, + command=lambda: self._start_video()) + btn_capture = Button(frame, text="Snap!", command=lambda: self._capture_img()) + + # position and set resize behaviour + lbl.grid(row=0, column=0) + lbl_frame.grid(row=0, column=1, columnspan=4) + self.lbl_vid.grid(row=0, column=1, columnspan=4) + btn.grid(row=1, column=0, pady=(2, 4)) + btn_start.grid(row=2, column=0, pady=(2, 4)) + btn_capture.grid(row=3, column=0, pady=(2, 4)) + nb.add(frame, text='Video', padding=2) + + # ============================================================================= + def _init_dev(self): + system = tof.System() + print(system) + + self.cameras = [] + + status = system.getCameraList(self.cameras) + if not status: + print("system.getCameraList failed with status: ", status) + + status = self.cameras[0].initialize() + if not status: + print("cameras[0].initialize() failed with status: ", status) + + modes = [] + status = self.cameras[0].getAvailableModes(modes) + if not status: + print("system.getAvailableModes() failed with status: ", status) + + types = [] + status = self.cameras[0].getAvailableFrameTypes(types) + if not status: + print("system.getAvailableFrameTypes() failed with status: ", status) + + status = self.cameras[0].setFrameType(types[0]) + if not status: + print("cameras[0].setFrameType() failed with status:", status) + + status = self.cameras[0].setMode(modes[ModesEnum.MODE_NEAR.value]) + if not status: + print("cameras[0].setMode() failed with status: ", status) + + # ============================================================================= + def _start_video(self): + camDetails = tof.CameraDetails() + status = self.cameras[0].getDetails(camDetails) + if not status: + print("system.getDetails() failed with status: ", status) + + # Enable noise reduction for better results + smallSignalThreshold = 100 + self.cameras[0].setControl("noise_reduction_threshold", str(smallSignalThreshold)) + + camera_range = camDetails.depthParameters.maxDepth + distance_scale = 255.0 / camera_range + tof_frame = tof.Frame() + while True: + # Capture frame-by-frame + status = self.cameras[0].requestFrame(tof_frame) + if not status: + print("cameras[0].requestFrame() failed with status: ", status) + + depth_map = np.array(tof_frame.getData(tof.FrameDataType.Depth), dtype="uint16", copy=False) + # Creation of the Depth image + depth_map = depth_map[0: 480, 0:640] + depth_map = cv.flip(depth_map, 1) + depth_map = distance_scale * depth_map + depth_map = np.uint8(depth_map) + depth_map = cv.applyColorMap(depth_map, cv.COLORMAP_RAINBOW) + depth_map = cv.cvtColor(depth_map, cv.COLOR_BGR2RGB) + self.img_obj = Image.fromarray(depth_map) + img = ImageTk.PhotoImage(self.img_obj.resize((640, 480))) + self.lbl_vid['image'] = img + self.update() + + # ============================================================================= + def _capture_img(self): + file_name = "test.png" + self.img_obj.resize((640, 480)).save(file_name) + + # ============================================================================= + def _create_process_tab(self, nb): + # add the process tab to main panel + frame = ProcessTab(nb) + nb.add(frame, text='Process', underline=0) + + +if __name__ == '__main__': + GestureDemo().mainloop() diff --git a/bindings/python/examples/gesture_rec/process.py b/bindings/python/examples/gesture_rec/process.py new file mode 100644 index 00000000..2b8f8fe3 --- /dev/null +++ b/bindings/python/examples/gesture_rec/process.py @@ -0,0 +1,134 @@ +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.image as mpimg +from tkinter import * +from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg +from skimage.color import rgb2hsv +from skimage import measure +from scipy.spatial import ConvexHull, convex_hull_plot_2d +from scipy.ndimage import distance_transform_edt +from scipy import ndimage + +hue_threshold = 0.3 + + +class ProcessTab(Frame): + + def __init__(self, parent, *args, **kwargs): + Frame.__init__(self, parent, *args, **kwargs) + self._create_widgets() + + def _create_widgets(self): + self._add_canvases() + btn_load = Button(self, text='Load', + command=lambda: self._load_img()) + self.resultVar = StringVar() + self.resultVar.set("How many fingers?") + btn_process = Button(self, text='Process', + command=lambda v=self.resultVar: self._display_result(v)) + lbl_result = Label(self, textvariable=self.resultVar, name='result') + btn_load.grid(row=0, column=2, pady=(2, 4)) + btn_process.grid(row=1, column=2, pady=(2, 4)) + lbl_result.grid(row=2, column=2) + + def _add_canvases(self): + # input image canvas + fig_input, self.ax_input = plt.subplots(figsize=(4, 3)) + self.ax_input.set_title("Depth image") + self.input_canvas = FigureCanvasTkAgg(fig_input, master=self) + self.input_canvas.get_tk_widget().grid(row=1, column=0) + # result image canvas + fig_result, self.ax_result = plt.subplots(figsize=(4, 3)) + self.ax_result.set_title("Detected Hand and Fingers") + self.result_canvas = FigureCanvasTkAgg(fig_result, master=self) + self.result_canvas.get_tk_widget().grid(row=1, column=1) + # process plots canvas + fig, (self.ax1, self.ax2) = plt.subplots(ncols=2, figsize=(8, 3)) + self.ax1.set_title("Histogram of the Hue channel with threshold") + self.ax1.set_ylim([0, 3000]) + self.ax1.axvline(x=hue_threshold, color='r', linestyle='dashed', linewidth=2) + self.ax2.set_title("Hue-thresholded image") + self.ax2.axis('off') + self.canvas = FigureCanvasTkAgg(fig, master=self) + self.canvas.get_tk_widget().grid(row=2, column=0, columnspan=2) + + # ============================================================================= + def _load_img(self): + self.rgb_img = mpimg.imread('test.png') + self.ax_input.imshow(self.rgb_img) + self.input_canvas.draw() + + # ============================================================================= + def _display_result(self, v): + self._hue_ch_hist() + self._detect_hand() + self._count_fingers() + + # ============================================================================= + def _hue_ch_hist(self): + # Remove the background + hsv_img = rgb2hsv(self.rgb_img) + hue_img = hsv_img[:, :, 0] + self.binary_img = (hue_img < hue_threshold) * 1 + self.ax1.hist(hue_img.ravel(), 512) + self.ax2.imshow(self.binary_img, cmap='gray') + self.canvas.draw() + + # ============================================================================= + def _detect_hand(self): + # Find all the objects in the image and select the largest one, which is the hand + labels = measure.label(self.binary_img) + props = measure.regionprops(labels) + props.reverse() + max_area = props[0] + points = props[0].filled_image + points_hull = np.where(points == 1) + self.cord = list(zip(points_hull[0], points_hull[1])) + self.dist_map = distance_transform_edt(points) + self.hand_center = ndimage.center_of_mass(self.dist_map) + self.radius = 1.75 * np.max(self.dist_map) + + # ============================================================================= + def _count_fingers(self): + # Find the convex hull = contour of the hand with extremities + hull = ConvexHull(self.cord) + cord_arr = np.array(self.cord) + vertices = cord_arr[hull.vertices] + vertices = vertices[1:len(vertices) - 2] + + dist = vertices[0:len(vertices) - 1] - vertices[1:len(vertices)] + cdist = np.sqrt(dist[:, 0] ** 2 + dist[:, 1] ** 2) + cdist = (cdist < 15) * 1 + cdist = cdist[0:len(cdist) - 1] - cdist[1:len(cdist)] + if cdist[0] != 0: + cdist[0] = -1 + if cdist[len(cdist) - 1] != 0: + cdist[len(cdist) - 1] = -1 + dist_idx = np.where(cdist == -1) + dist_idx = np.array(dist_idx) + 1 + + # Compute the finger tips distances to the center of the hand and detect the gesture + finger_dist = np.array(vertices[dist_idx]) - self.hand_center + for fd in finger_dist: + finger_cdist = np.sqrt(fd[:, 0] ** 2 + fd[:, 1] ** 2) + self.fingers = np.where(finger_cdist > self.radius) + self.detect_gesture() + + self.ax_result.set_title("Detected hand and fingers") + self.ax_result.imshow(self.dist_map, cmap='gray') + self.ax_result.plot(self.hand_center[1], self.hand_center[0], "o", mec='g', color='none', markersize=self.radius) + self.ax_result.plot(vertices[dist_idx, 1], vertices[dist_idx, 0], 'o', mec='r', color='none', lw=1, markersize=10) + self.ax_result.plot(vertices[:, 1], vertices[:, 0], 'x', mec='g', color='none', lw=1, markersize=4) + self.result_canvas.draw() + + # ============================================================================= + def detect_gesture(self): + + if len(self.fingers[0]) == 5: + self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers.Paper") + elif self.fingers == 2 or self.fingers == 3: + self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Scissors") + elif self.fingers == 0: + self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Rock") + else: + self.resultVar.set("Found " + str(len(self.fingers[0])) + " extended fingers. Unknown gesture")