stats_engine: Fix metrics

Previously, there was a chance that a match would be counted twice
towards the actually added/removed lines of a patch against a git
repository.

Now, we blacklist the indices of lines we already found in a file.

Author: raymanfx

src/stats_engine.py

@@ -153,23 +153,30 @@ def collect_stats(kernel_repo, cve_patch):
             # keep track of all added lines
             all_added.extend(added)
 
+        # list of blacklisted indices of lines we already found
+        already_found = []
         for i in range(0, len(fp_contents)):
             line = fp_contents[i]
 
-            for (above, added, below) in patch_added_lines[file]:
+            for j in range(0, len(patch_added_lines[file])):
+                (above, added, below) = patch_added_lines[file][j]
+                # check if we already found these lines
+                if j in already_found:
+                    continue
+
                 # check if the added block is present
                 block_end_index = i
                 if added[0] in line:
-                    added_lines = added[1:]
-                    fp_lines = fp_contents[i+1:i+len(added_lines)+1]
+                    added_lines = added
+                    fp_lines = fp_contents[i:i+len(added_lines)]
 
                     if set(added_lines) != set(fp_lines):
                         continue
 
-                    # check the surrounding lines (two above, two below)
+                    # check the surrounding lines
                     fp_above = []
                     idx = i - 1
-                    while len(fp_above) < 2:
+                    while len(fp_above) < len(above):
                         above_candidate = fp_contents[idx]
                         if above_candidate not in all_added:
                             fp_above.append(above_candidate)
@@ -179,7 +186,7 @@ def collect_stats(kernel_repo, cve_patch):
 
                     fp_below = []
                     idx = block_end_index + 1
-                    while len(fp_below) < 2:
+                    while len(fp_below) < len(below):
                         if idx == len(fp_contents):
                             break
                         below_candidate = fp_contents[idx]
@@ -190,6 +197,7 @@ def collect_stats(kernel_repo, cve_patch):
                         if (set(above) == set(fp_above)
                                 and set(below) == set(fp_below)):
                             actually_added_lines += len(added)
+                            already_found.append(j)
 
     for file in patch_removed_lines:
         file_path = kernel_repo + '/' + file
@@ -204,16 +212,25 @@ def collect_stats(kernel_repo, cve_patch):
             print("[E] Failed to read file: " + file_path + ", skipping!")
             continue
 
+        # list of blacklisted indices of lines we already found
+        already_found = []
         for i in range(0, len(fp_contents)):
-            lines = fp_contents[i:i+2]
-            for (above, removed, below) in patch_removed_lines[file]:
+
+            for j in range(0, len(patch_removed_lines[file])):
+                (above, removed, below) = patch_removed_lines[file][j]
+                # check if we already found these lines
+                if j in already_found:
+                    continue
+
+                lines = fp_contents[i:i+len(above)]
+
                 # make sure the removed block is missing
                 if set(above) != set(lines):
                     continue
 
                 fp_below = []
-                idx = i + 2
-                while len(fp_below) < 2:
+                idx = i + len(above)
+                while len(fp_below) < len(below):
                     if idx == len(fp_contents):
                         break
                     below_candidate = fp_contents[idx]
@@ -223,6 +240,7 @@ def collect_stats(kernel_repo, cve_patch):
 
                 if set(below) == set(fp_below):
                     actually_removed_lines += len(removed)
+                    already_found.append(j)
 
     # if the patch does not add/remove lines, we return None
     if total_added_lines == 0:
@@ -268,23 +286,22 @@ def main():
         print("[E] invalid patchfile: " + patchfile)
         return
 
-    patch_added_count = 0
-    patch_removed_count = 0
     patch_added_lines = find_changed_lines(patchfile, '+')
+    total_added_lines = 0
+    for file in patch_added_lines:
+        triplets = patch_added_lines[file]
+        for (above, changed, below) in triplets:
+            total_added_lines += len(changed)
+
     patch_removed_lines = find_changed_lines(patchfile, '-')
+    total_removed_lines = 0
+    for file in patch_removed_lines:
+        triplets = patch_removed_lines[file]
+        for (above, changed, below) in triplets:
+            total_removed_lines += len(changed)
 
-    # collect metrics
-    for changed_file in patch_added_lines:
-        for triplet in patch_added_lines[changed_file]:
-            # triplet[1] holds the actually changed lines
-            patch_added_count += len(triplet[1])
-    for changed_file in patch_removed_lines:
-        for triplet in patch_removed_lines[changed_file]:
-            # triplet[1] holds the actually changed lines
-            patch_removed_count += len(triplet[1])
-
-    print(patchfile + ": " + str(patch_added_count) + " insertions(+)" +
-          ", " + str(patch_removed_count) + " deletions(-)")
+    print(patchfile + ": " + str(total_added_lines) + " insertions(+)" +
+          ", " + str(total_removed_lines) + " deletions(-)")
 
     if kernel_repo:
         ((add, total_add), (rem, total_rem)) = collect_stats(kernel_repo, patchfile)