better tests (#4)

- Introduces more tests
- Introduces the virtual file system test helper
- Adds convenience methods to the following classes:
    - Position: __eq__ and __repr__
    - Querier: __repr__

Author: nohehf

.github/workflows/ci.yaml

@@ -25,19 +25,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.11.9"
-
-      - name: Setup venv
-        run: python -m venv .venv
-
-      - name: Install Python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install maturin
-
+          
       - name: Run tests
-        run: |
-          . .venv/bin/activate
-          maturin develop
-          which python
-          which maturin
-          python tests/test.py
+        run: make setup test

Makefile

@@ -1,17 +1,21 @@
+# Run all commands in a single shell, so that the virtual environment is activated for all commands.
+.ONESHELL:
+
 .PHONY: setup
 setup:
 	python -m venv .venv
 	. .venv/bin/activate
 	pip install maturin
+	pip install pytest
 
 .PHONY: develop
 develop:
+	. .venv/bin/activate
 	maturin develop
 
-
 test: develop
-    ## TODO: Add actual tests with pytest
-	python tests/test.py
+	. .venv/bin/activate
+	pytest
 
 .PHONY: validate-tags release
 

py.typed

@@ -30,6 +30,7 @@ pub struct Position {
 #[pyclass]
 pub struct Querier {
     db_reader: SQLiteReader,
+    db_path: String,
 }
 
 #[pymethods]
@@ -38,7 +39,8 @@ impl Querier {
     pub fn new(db_path: String) -> Self {
         println!("Opening database: {}", db_path);
         Querier {
-            db_reader: SQLiteReader::open(db_path).unwrap(),
+            db_reader: SQLiteReader::open(db_path.clone()).unwrap(),
+            db_path: db_path,
         }
     }
 
@@ -54,6 +56,10 @@ impl Querier {
 
         Ok(positions)
     }
+
+    fn __repr__(&self) -> String {
+        format!("Querier(db_path=\"{}\")", self.db_path)
+    }
 }
 
 // TODO(@nohehf): Indexer class
@@ -64,6 +70,17 @@ impl Position {
     fn new(path: String, line: usize, column: usize) -> Self {
         Position { path, line, column }
     }
+
+    fn __eq__(&self, other: &Position) -> bool {
+        self.path == other.path && self.line == other.line && self.column == other.column
+    }
+
+    fn __repr__(&self) -> String {
+        format!(
+            "Position(path=\"{}\", line={}, column={})",
+            self.path, self.line, self.column
+        )
+    }
 }
 
 impl Display for Position {

src/classes.rs

@@ -12,9 +12,12 @@ class Position:
     column: int
 
     def __init__(self, path: str, line: int, column: int) -> None: ...
+    def __eq__(self, other: object) -> bool: ...
+    def __repr__(self) -> str: ...
 
 class Querier:
     def __init__(self, db_path: str) -> None: ...
     def definitions(self, reference: Position) -> list[Position]: ...
+    def __repr__(self) -> str: ...
 
 def index(paths: list[str], db_path: str, language: Language) -> None: ...

stack_graphs_python.pyi

@@ -0,0 +1,36 @@
+import os
+from stack_graphs_python import index, Querier, Position, Language
+
+
+# index ./js_sample directory
+# This test is the same as the one in js_ok_test.py, without using the virtual file system helper
+def test_from_dir():
+    # convert ./js_sample directory to absolute path
+    dir = os.path.abspath("./tests/js_sample")
+    db_path = os.path.abspath("./db.sqlite")
+
+    print("Indexing directory: ", dir)
+    print("Database path: ", db_path)
+
+    index([dir], db_path, language=Language.JavaScript)
+
+    source_reference = Position(path=dir + "/index.js", line=2, column=12)
+
+    print("Querying definition for: ", source_reference.path)
+
+    querier = Querier(db_path)
+
+    results = querier.definitions(source_reference)
+
+    print("Results: ", results)
+
+    for result in results:
+        print("Path: ", result.path)
+        print("Line: ", result.line)
+        print("Column: ", result.column)
+        print("\n")
+
+    assert len(results) == 2
+    assert results[0].path.endswith("index.js")
+    assert results[0].line == 0
+    assert results[0].column == 9

tests/from_dir_test.py

@@ -0,0 +1,150 @@
+import os
+import re
+import tempfile
+import shutil
+from typing import Iterator
+from stack_graphs_python import Position
+import contextlib
+
+
+POSITION_MARKER_REGEX = r"( *)\^{([a-zA-Z0-9_]+)}\n"
+FILE_SEPARATOR_REGEX = r"\n;---([a-zA-Z0-9\_./]+)---\n"
+
+
+def _remove_position_markers(string: str) -> str:
+    return re.subn(POSITION_MARKER_REGEX, "", string)[0]
+
+
+def _split_files(string: str) -> list[tuple[str, str]]:
+    """
+    Split a string into a list of tuples where each tuple contains the file path and the file content
+    """
+    files = []
+    matches = [m for m in re.finditer(FILE_SEPARATOR_REGEX, string)]
+    # Get the file path from match group 0
+    # Get the file content from the end of the previous match to the start of the current match
+    for i in range(1, len(matches) + 1):
+        end = len(string) if i == len(matches) else matches[i].start()
+        file_path = matches[i - 1].group(1)
+        file_content = string[matches[i - 1].end() : end]
+        files.append((file_path, file_content))
+
+    return files
+
+
+def _get_positions_in_file(file_path: str, contents: str) -> dict[str, Position]:
+    positions = {}
+    while match := re.search(POSITION_MARKER_REGEX, contents):
+        identifier = match.group(2)
+        line = contents.count("\n", 0, match.start()) - 1
+        # count the number of spaces
+        column = len(match.group(1))
+        positions[identifier] = Position(file_path, line, column)
+        contents = contents[: match.start()] + contents[match.end() :]
+
+    return positions
+
+
+@contextlib.contextmanager
+def string_to_virtual_repo(
+    string: str,
+) -> Iterator[tuple[str, dict[str, Position]]]:
+    """
+    # Usage
+    ```py
+    string = \"""
+    ;---main.py---
+    from module import module
+         ^{pos1}
+
+    ;---module.py---
+    module = "module.py"
+    ^{pos2}
+    \"""
+
+    with string_to_virtual_repo(string) as (repo_path, positions):
+        ...
+    ```
+
+    # Description
+
+    This is a utility to create a virtual repository from a string
+    It allows to quickly write tests, as creating those files and getting the correct positions is cumbersome
+
+    It parses a string into a list of files and positions in the files
+    Files separators are defined as ';---{filepath}---'. It should be preceded by a newline and followed by a newline
+    Positions queries are defined as '^{identifier}' where identifier is a string, prefixed by any number of spaces
+    Positions queries are stripped from the file content in the temporary filesystem
+
+    For example this string:
+    ```py
+    string = \"""
+    ;---main.py---
+    from module import module
+        ^{pos1}
+    import lib
+    from lib.file import file
+
+    print(module)
+    print(lib.lib)
+            ^{pos2}
+    print(file)
+
+    ;---module.py---
+    module = "module.py"
+            ^{pos3}
+
+    ;---lib/__init__.py---
+    lib = "lib/__init__.py"
+
+    ;---lib/file.py---
+    file = "lib/file.py"
+    \"""
+    ```
+
+    When parsed via:
+    ```py
+    with string_to_virtual_repo(string) as (repo_path, positions):
+        ...
+    ```
+
+    Will create the following temporary filesystem:
+    ```sh
+    /
+    ├── main.py
+    ├── module.py
+    └── lib
+        ├── __init__.py
+        └── file.py
+    ```
+
+    And output the following positions:
+    ```py
+    {
+        "pos1": Position("main.py", 0, 6),
+        "pos2": Position("main.py", 5, 11),
+        "pos3": Position("module.py", 0, 10),
+    }
+    ```
+    """
+    files = _split_files(string)
+    temp_dir = tempfile.mkdtemp()
+    temp_dir = os.path.abspath(tempfile.mkdtemp())
+    positions_map = {}
+    for file_path, file_content in files:
+        abs_path = os.path.join(temp_dir, file_path)
+        positions_map.update(_get_positions_in_file(abs_path, file_content))
+
+    try:
+        for file_path, file_content in files:
+            file_content = _remove_position_markers(file_content)
+            full_path = os.path.join(temp_dir, file_path)
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+            with open(full_path, "w") as f:
+                f.write(file_content)
+        yield temp_dir, positions_map
+    except Exception as e:
+        print(e)
+    finally:
+        print("Removing temp dir")
+        shutil.rmtree(temp_dir)