Merge pull request #134 from datafold/inf_threshold

Fixed tests; bisection_threshold can now be inf

Author: erezsh

data_diff/diff_tables.py

@@ -4,9 +4,10 @@
 from abc import ABC, abstractmethod
 import time
 import os
+from numbers import Number
 from operator import attrgetter, methodcaller
 from collections import defaultdict
-from typing import List, Tuple, Iterator, Optional, Type
+from typing import List, Tuple, Iterator, Optional
 import logging
 from concurrent.futures import ThreadPoolExecutor
 
@@ -263,14 +264,14 @@ class TableDiffer:
 
     Parameters:
         bisection_factor (int): Into how many segments to bisect per iteration.
-        bisection_threshold (int): When should we stop bisecting and compare locally (in row count).
+        bisection_threshold (Number): When should we stop bisecting and compare locally (in row count).
         threaded (bool): Enable/disable threaded diffing. Needed to take advantage of database threads.
         max_threadpool_size (int): Maximum size of each threadpool. ``None`` means auto. Only relevant when `threaded` is ``True``.
                                    There may be many pools, so number of actual threads can be a lot higher.
     """
 
     bisection_factor: int = DEFAULT_BISECTION_FACTOR
-    bisection_threshold: int = DEFAULT_BISECTION_THRESHOLD
+    bisection_threshold: Number = DEFAULT_BISECTION_THRESHOLD       # Accepts inf for tests
     threaded: bool = True
     max_threadpool_size: Optional[int] = 1
 

tests/test_database_types.py

@@ -589,7 +589,7 @@ def test_types(self, source_db, target_db, source_type, target_type, type_catego
         # reasonable amount of rows each. These will then be downloaded in
         # parallel, using the existing implementation.
         dl_factor = max(int(N_SAMPLES / 100_000), 2) if BENCHMARK else 2
-        dl_threshold = int(N_SAMPLES / dl_factor) + 1 if BENCHMARK else N_SAMPLES + 1
+        dl_threshold = int(N_SAMPLES / dl_factor) + 1 if BENCHMARK else math.inf
         dl_threads = 1
         differ = TableDiffer(
             bisection_threshold=dl_threshold, bisection_factor=dl_factor, max_threadpool_size=dl_threads
@@ -599,10 +599,7 @@ def test_types(self, source_db, target_db, source_type, target_type, type_catego
         download_duration = time.time() - start
         expected = []
         self.assertEqual(expected, diff)
-        if type_category == "uuid":
-            pass  # UUIDs aren't serial, so they mess with the first max_rows estimation.
-        else:
-            self.assertEqual(len(sample_values), differ.stats.get("rows_downloaded", 0))
+        self.assertEqual(len(sample_values), differ.stats.get("rows_downloaded", 0))
 
         result = {
             "test": self._testMethodName,