remove heavy weight dependencies and compute text cosine similarity in pure python

Author: bckohan

.github/workflows/lint.yml

@@ -22,7 +22,6 @@ jobs:
         django-version:
           - '3.2' # LTS April 2024
           - '4.2' # LTS April 2026
-          - '5.0' # April 2025
           - '5.1' # December 2025
         exclude:
           - python-version: '3.8'
@@ -33,10 +32,6 @@ jobs:
             django-version: '3.2'
           - python-version: '3.10'
             django-version: '3.2'
-          - python-version: '3.8'
-            django-version: '5.0'
-          - python-version: '3.10'
-            django-version: '5.0'
           - python-version: '3.8'
             django-version: '5.1'
           - python-version: '3.10'

.github/workflows/test.yml

@@ -57,9 +57,6 @@ jobs:
       with:
         virtualenvs-create: true
         virtualenvs-in-project: true
-    - name: Install libopenblas-dev
-      if: matrix.python-version == '3.13'
-      run: sudo apt-get install libopenblas-dev
     - name: Install Release Dependencies
       run: |
         poetry config virtualenvs.in-project true

pyproject.toml

@@ -94,19 +94,7 @@ doc8 = ">=1.1.1"
 aiohttp = ">=3.9.1"
 readme-renderer = {extras = ["md"], version = ">=42"}
 sphinxcontrib-typer = {extras = ["html", "pdf", "png"], version = ">=0.5.0", markers="python_version >= '3.9'"}
-scikit-learn = [
-    { version = ">=1.5", markers = "python_version > '3.8'" },
-    { version = ">=1.0", markers = "python_version <= '3.8'" },
-]
 pytest-env = ">=1.0.0"
-numpy = [
-    { version = ">=1.26", markers = "python_version > '3.8'" },
-    { version = "<=1.24", markers = "python_version <= '3.8'" },
-]
-scipy = [
-    { version = ">=1.11", markers = "python_version > '3.8'" },
-    { version = "<=1.10", markers = "python_version <= '3.8'" },
-]
 django-stubs = ">=4.2.7"
 pexpect = ">=4.9.0"
 pyright = ">=1.1.357"

tests/utils.py

@@ -5,11 +5,12 @@
 import sys
 from pathlib import Path
 from typing import Tuple
-
+import re
+import math
+import re
+from collections import Counter
 import pexpect
 from django.core.management.color import no_style
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
 
 try:
     import rich
@@ -22,6 +23,26 @@
 TESTS_DIR = Path(__file__).parent
 DJANGO_PARAMETER_LOG_FILE = TESTS_DIR / "dj_params.json"
 manage_py = TESTS_DIR.parent / "manage.py"
+WORD = re.compile(r"\w+")
+
+
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+
+    sum1 = sum([vec1[x] ** 2 for x in list(vec1.keys())])
+    sum2 = sum([vec2[x] ** 2 for x in list(vec2.keys())])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+    if not denominator:
+        return 0.0
+    else:
+        return float(numerator) / denominator
+
+
+def text_to_vector(text):
+    words = WORD.findall(text)
+    return Counter(words)
 
 
 def similarity(text1, text2):
@@ -31,10 +52,10 @@ def similarity(text1, text2):
 
     We use this to lazily evaluate the output of --help to our
     renderings.
-    """
-    vectorizer = TfidfVectorizer()
-    tfidf_matrix = vectorizer.fit_transform([text1, text2])
-    return cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
+    #"""
+    vector1 = text_to_vector(text1)
+    vector2 = text_to_vector(text2)
+    return get_cosine(vector1, vector2)
 
 
 def get_named_arguments(function):