Merge pull request #17 from bartleboeuf/matchtemplate-concurrency

Speed up search by adding concurrency to findMatches methods

Author: LauLauThom

CHANGELOG.md

@@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).  
 
+### [1.6.4] - 2023-03-03
+
+### Changed 
+- Improve speed by adding concurrency in the findMatches method, using half the number of cpu cores available.
+- Mention installation in editable mode in README
+
 ### [1.6.3] - 2021-11-24  
 
 ### Changed 

MTM/__init__.py

@@ -1,13 +1,16 @@
 """Main code for Multi-Template-Matching (MTM)."""
+import os
+import warnings
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 import cv2
-import numpy  as np
+import numpy as np
 import pandas as pd
-import warnings
+from scipy.signal import find_peaks
 from skimage.feature import peak_local_max
-from scipy.signal    import find_peaks
-from .version import __version__
 
 from .NMS import NMS
+from .version import __version__
 
 __all__ = ['NMS']
 
@@ -33,7 +36,7 @@ def _findLocalMax_(corrMap, score_threshold=0.6):
         peaks = [[i,0] for i in peaks[0]]
 
 
-    else: # Correlatin map is 2D
+    else: # Correlation map is 2D
         peaks = peak_local_max(corrMap, threshold_abs=score_threshold, exclude_border=False).tolist()
 
     return peaks
@@ -116,82 +119,111 @@ def findMatches(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=floa
     -------
     - Pandas DataFrame with 1 row per hit and column "TemplateName"(string), "BBox":(X, Y, Width, Height), "Score":float
     """
-    if N_object != float("inf") and type(N_object) != int:
+    if N_object != float("inf") and not isinstance(N_object, int):
         raise TypeError("N_object must be an integer")
 
     ## Crop image to search region if provided
     if searchBox is not None:
         xOffset, yOffset, searchWidth, searchHeight = searchBox
         image = image[yOffset : yOffset+searchHeight, xOffset : xOffset+searchWidth]
-        
     else:
         xOffset=yOffset=0
-    
+
     # Check that the template are all smaller are equal to the image (original, or cropped if there is a search region)
     for index, tempTuple in enumerate(listTemplates):
-        
+
         if not isinstance(tempTuple, tuple) or len(tempTuple)==1:
             raise ValueError("listTemplates should be a list of tuples as ('name','array') or ('name', 'array', 'mask')")
-            
+
         templateSmallerThanImage = all(templateDim <= imageDim for templateDim, imageDim in zip(tempTuple[1].shape, image.shape))
-        
+
         if not templateSmallerThanImage :
             fitIn = "searchBox" if (searchBox is not None) else "image"
             raise ValueError("Template '{}' at index {} in the list of templates is larger than {}.".format(tempTuple[0], index, fitIn) )
-        
+
     listHit = []
-    for tempTuple in listTemplates:
+    ## Use multi-threading to iterate through all templates, using half the number of cpu cores available.
+    with ThreadPoolExecutor(max_workers=round(os.cpu_count()*.5)) as executor:
+        futures = [executor.submit(_multi_compute, tempTuple, image, method, N_object, score_threshold, xOffset, yOffset, listHit) for tempTuple in listTemplates]
+        for future in as_completed(futures):
+            _ = future.result()
 
-        templateName, template = tempTuple[:2]
-        mask = None
+    if listHit:
+        return pd.DataFrame(listHit) # All possible hits before Non-Maxima Supression
+    else:
+        return pd.DataFrame(columns=["TemplateName", "BBox", "Score"])
 
-        if len(tempTuple)>=3: # ie a mask is also provided
-            if method in (0,3):
-                mask = tempTuple[2]
-            else:
-                warnings.warn("Template matching method not supporting the use of Mask. Use 0/TM_SQDIFF or 3/TM_CCORR_NORMED.")
 
-        #print('\nSearch with template : ',templateName)
-        corrMap = computeScoreMap(template, image, method, mask=mask)
+def _multi_compute(tempTuple, image, method, N_object, score_threshold, xOffset, yOffset, listHit):
+    """
+    Find all possible template locations satisfying the score threshold provided a template to search and an image.
+    Add the hits in the list of hits.
+
+    Parameters
+    ----------
+    - tempTuple : a tuple (LabelString, template, mask (optional))
+                      template to search in each image, associated to a label
+                      labelstring : string
+                      template    : numpy array (grayscale or RGB)
+                      mask (optional): numpy array, should have the same dimensions and type than the template
 
-        ## Find possible location of the object
-        if N_object==1: # Detect global Min/Max
-            minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(corrMap)
+    - image  : Grayscale or RGB numpy array
+               image in which to perform the search, it should be the same bitDepth and number of channels than the templates
 
-            if method in (0,1):
-                peaks = [minLoc[::-1]] # opposite sorting than in the multiple detection
+    - method : int
+                one of OpenCV template matching method (0 to 5), default 5=0-mean cross-correlation
 
-            else:
-                peaks = [maxLoc[::-1]]
+    - N_object: int or float("inf")
+                expected number of objects in the image, default to infinity if unknown
 
+    - score_threshold: float in range [0,1]
+                if N_object>1, returns local minima/maxima respectively below/above the score_threshold
 
-        else:# Detect local max or min
-            if method in (0,1): # Difference => look for local minima
-                peaks = _findLocalMin_(corrMap, score_threshold)
+    - xOffset : int 
+                optional the x offset if the search area is provided
 
-            else:
-                peaks = _findLocalMax_(corrMap, score_threshold)
+    - yOffset : int
+                optional the y offset if the search area is provided
 
+    - listHit : the list of hits which we want to add the discovered hit
+                expected array of hits
+    """
+    templateName, template = tempTuple[:2]
+    mask = None
 
-        #print('Initially found',len(peaks),'hit with this template')
+    if len(tempTuple)>=3: # ie a mask is also provided
+        if method in (0,3):
+            mask = tempTuple[2]
+        else:
+            warnings.warn("Template matching method not supporting the use of Mask. Use 0/TM_SQDIFF or 3/TM_CCORR_NORMED.")
 
+    #print('\nSearch with template : ',templateName)
+    corrMap = computeScoreMap(template, image, method, mask=mask)
 
-        # Once every peak was detected for this given template
-        ## Create a dictionnary for each hit with {'TemplateName':, 'BBox': (x,y,Width, Height), 'Score':coeff}
+    ## Find possible location of the object
+    if N_object==1: # Detect global Min/Max
+        _, _, minLoc, maxLoc = cv2.minMaxLoc(corrMap)
+        if method in (0,1):
+            peaks = [minLoc[::-1]] # opposite sorting than in the multiple detection
+        else:
+            peaks = [maxLoc[::-1]]
+    else:# Detect local max or min
+        if method in (0,1): # Difference => look for local minima
+            peaks = _findLocalMin_(corrMap, score_threshold)
+        else:
+            peaks = _findLocalMax_(corrMap, score_threshold)
 
-        height, width = template.shape[0:2] # slicing make sure it works for RGB too
+    #print('Initially found',len(peaks),'hit with this template')
 
-        for peak in peaks :
-            coeff  = corrMap[tuple(peak)]
-            newHit = {'TemplateName':templateName, 'BBox': ( int(peak[1])+xOffset, int(peak[0])+yOffset, width, height ) , 'Score':coeff}
+    # Once every peak was detected for this given template
+    ## Create a dictionnary for each hit with {'TemplateName':, 'BBox': (x,y,Width, Height), 'Score':coeff}
 
-            # append to list of potential hit before Non maxima suppression
-            listHit.append(newHit)
+    height, width = template.shape[0:2] # slicing make sure it works for RGB too
 
-    if listHit:
-        return pd.DataFrame(listHit) # All possible hits before Non-Maxima Supression
-    else:
-        return pd.DataFrame(columns=["TemplateName", "BBox", "Score"]) # empty df with correct column header
+    for peak in peaks :
+        # append to list of potential hit before Non maxima suppression
+        # no need to lock the list, append is thread-safe
+        listHit.append({'TemplateName':templateName, 'BBox': ( int(peak[1])+xOffset, int(peak[0])+yOffset, width, height ) , 'Score':corrMap[tuple(peak)]}) # empty df with correct column header
 
 
 def matchTemplates(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=float("inf"), score_threshold=0.5, maxOverlap=0.25, searchBox=None):
@@ -239,7 +271,7 @@ def matchTemplates(listTemplates, image, method=cv2.TM_CCOEFF_NORMED, N_object=f
     tableHit = findMatches(listTemplates, image, method, N_object, score_threshold, searchBox)
 
     if method == 0: raise ValueError("The method TM_SQDIFF is not supported. Use TM_SQDIFF_NORMED instead.")
-    sortAscending = True if method==1 else False
+    sortAscending = (method==1)
 
     return NMS(tableHit, score_threshold, sortAscending, N_object, maxOverlap)
 
@@ -275,7 +307,7 @@ def drawBoxesOnRGB(image, tableHit, boxThickness=2, boxColor=(255, 255, 00), sho
     if image.ndim == 2: outImage = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) # convert to RGB to be able to show detections as color box on grayscale image
     else:               outImage = image.copy()
 
-    for index, row in tableHit.iterrows():
+    for _, row in tableHit.iterrows():
         x,y,w,h = row['BBox']
         cv2.rectangle(outImage, (x, y), (x+w, y+h), color=boxColor, thickness=boxThickness)
         if showLabel: cv2.putText(outImage, text=row['TemplateName'], org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=labelScale, color=labelColor, lineType=cv2.LINE_AA)
@@ -315,9 +347,9 @@ def drawBoxesOnGray(image, tableHit, boxThickness=2, boxColor=255, showLabel=Fal
     if image.ndim == 3: outImage = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) # convert to RGB to be able to show detections as color box on grayscale image
     else:               outImage = image.copy()
 
-    for index, row in tableHit.iterrows():
+    for _, row in tableHit.iterrows():
         x,y,w,h = row['BBox']
         cv2.rectangle(outImage, (x, y), (x+w, y+h), color=boxColor, thickness=boxThickness)
         if showLabel: cv2.putText(outImage, text=row['TemplateName'], org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=labelScale, color=labelColor, lineType=cv2.LINE_AA)
 
-    return outImage
+    return outImage

MTM/version.py

@@ -2,4 +2,4 @@
 # 1) we don't load dependencies by storing it in __init__.py
 # 2) we can import it in setup.py for the same reason
 # 3) we can import it into your module module
-__version__ = '1.6.3'
+__version__ = '1.6.4'

README.md

@@ -8,14 +8,23 @@ The main function `MTM.matchTemplates` returns the best predicted locations prov
 
 The branch opencl contains some test using the UMat object to run on GPU, but it is actually slow, which can be expected for small dataset as the transfer of the data between the CPU and GPU is slow.
 
-__** News **__ : You might be interested to test the newer python implementation which is more object-oriented and only relying on scikit-image and shapely.*
+# News
+- 03/03/2023 : Version 1.6.4 contributed by @bartleboeuf comes with speed enhancement thanks to parallelizing of the individual template searches.
+Thanks for this first PR !!
+- 10/11/2021 : You might be interested to test the newer python implementation which is more object-oriented and only relying on scikit-image and shapely.*
 https://github.com/multi-template-matching/mtm-python-oop 
 
 # Installation
 Using pip in a python environment, `pip install Multi-Template-Matching`  
 Once installed, `import MTM`should work.  
 Example jupyter notebooks can be downloaded from the tutorial folder of the github repository and executed in the newly configured python environement.  
 
+## Install in dev mode
+If you want to contribute or experiment with the source code, you can install the package "from source", by first downloading or cloning the repo.    
+Then opening a command prompt in the repo's root directory (the one containing this README) and calling `pip install -e .` (mind the final dot).    
+- the `-e` flag stands for editable and make sure that any change to the source code will be directly reflected when you import the package in your script  
+- the . just tell pip to look for the package to install in the current directory
+
 # Documentation
 The [wiki](https://github.com/multi-template-matching/MultiTemplateMatching-Python/wiki) section of the repo contains a mini API documentation with description of the key functions of the package.   
 The [website](https://multi-template-matching.github.io/Multi-Template-Matching/) of the project contains some more general documentation.