Merge branch 'main' of github.com:codellm-devkit/python-sdk

Author: rahlk

cldk/analysis/java/codeanalyzer/codeanalyzer.py

@@ -67,7 +67,10 @@ def __init__(
         self.eager_analysis = eager_analysis
         self.analysis_level = analysis_level
         self.target_files = target_files
-        self.application = self._init_codeanalyzer(analysis_level=1 if analysis_level == AnalysisLevel.symbol_table else 2)
+        if self.source_code is None:
+            self.application = self._init_codeanalyzer(analysis_level=1 if analysis_level == AnalysisLevel.symbol_table else 2)
+        else:
+            self.application = self._codeanalyzer_single_file()
         # Attributes related the Java code analysis...
         if analysis_level == AnalysisLevel.call_graph:
             self.call_graph: nx.DiGraph = self._generate_call_graph(using_symbol_table=False)

cldk/analysis/java/codeanalyzer/jar/codeanalyzer-2.3.3.jar

@@ -28,6 +28,7 @@
 from cldk.analysis.java import JavaAnalysis
 from cldk.analysis.commons.treesitter import TreesitterJava
 from cldk.analysis.python.python_analysis import PythonAnalysis
+from cldk.analysis.python.python_analysis import PythonAnalysis
 from cldk.utils.exceptions import CldkInitializationException
 from cldk.utils.sanitization.java import TreesitterSanitizer
 
@@ -123,9 +124,6 @@ def analysis(
             return PythonAnalysis(
                 project_dir=project_path,
                 source_code=source_code,
-                analysis_backend_path=analysis_backend_path,
-                analysis_json_path=analysis_json_path,
-                eager_analysis=eager,
             )
         elif self.language == "c":
             return CAnalysis(project_dir=project_path)

tests/resources/java/codeanalyzer_jars/2.3.0/codeanalyzer-2.3.0.jar renamed to cldk/analysis/java/codeanalyzer/jar/codeanalyzer-2.3.4.jar

@@ -270,6 +270,7 @@ class JCallable(BaseModel):
         code (str): The code block of the callable.
         start_line (int): The starting line number of the callable in the source file.
         end_line (int): The ending line number of the callable in the source file.
+        code_start_line (int): The starting line number of the code block of a callable in the source file.
         referenced_types (List[str]): The types referenced within the callable.
         accessed_fields (List[str]): Fields accessed in the callable.
         call_sites (List[JCallSite]): Call sites in the callable.
@@ -293,6 +294,7 @@ class JCallable(BaseModel):
     code: str
     start_line: int
     end_line: int
+    code_start_line: int
     referenced_types: List[str]
     accessed_fields: List[str]
     call_sites: List[JCallSite]
@@ -363,11 +365,15 @@ class JCompilationUnit(BaseModel):
     """Represents a compilation unit in Java.
 
     Attributes:
+        file_path (str): The path to the source file.
+        package_name (str): The name of the package for the comppilation unit.
         comments (List[JComment]): A list of comments in the compilation unit.
         imports (List[str]): A list of import statements in the compilation unit.
         type_declarations (Dict[str, JType]): A dictionary mapping type names to their corresponding JType representations.
     """
 
+    file_path: str
+    package_name: str
     comments: List[JComment]
     imports: List[str]
     type_declarations: Dict[str, JType]
@@ -430,11 +436,10 @@ def validate_source(cls, value) -> JMethodDetail:
         j_callable: JCallable = _CALLABLES_LOOKUP_TABLE.get(
             (type_declaration, signature),
             JCallable(
-                comments=[],
                 signature=signature,
                 is_implicit=True,
                 is_constructor="<init>" in value["callable_declaration"],
-                comment="",
+                comments=[],
                 annotations=[],
                 modifiers=[],
                 thrown_exceptions=[],
@@ -443,12 +448,15 @@ def validate_source(cls, value) -> JMethodDetail:
                     JCallableParameter(name=None, type=t, annotations=[], modifiers=[], start_column=-1, end_column=-1, start_line=-1, end_line=-1)
                     for t in value["callable_declaration"].split("(")[1].split(")")[0].split(",")
                 ],
+                return_type=None,
                 code="",
                 start_line=-1,
                 end_line=-1,
+                code_start_line=-1,
                 referenced_types=[],
                 accessed_fields=[],
                 call_sites=[],
+                is_entrypoint=False,
                 variable_declarations=[],
                 crud_operations=[],
                 crud_queries=[],

cldk/core.py

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cldk"
-version = "1.0.1"
+version = "1.0.4"
 description = "The official Python SDK for Codellm-Devkit."
 authors = ["Rahul Krishna <[email protected]>", "Rangeet Pan <[email protected]>", "Saurabh Sinhas <[email protected]>",
             "Raju Pavuluri <[email protected]>"]
@@ -120,4 +120,4 @@ directory = "coverage_html_report"
 sample-c-application = "tests/resources/c/application/"
 sample-application = "tests/resources/java/application/"
 sample-application-analysis-json = "tests/resources/java/analysis_json/"
-codeanalyzer-jar-path = "tests/resources/java/codeanalyzer_jars/"
+codeanalyzer-jar-path = "cldk/analysis/java/codeanalyzer/jar/"

cldk/models/java/models.py

@@ -50,6 +50,24 @@ def test_get_symbol_table_is_not_null(test_fixture, analysis_json):
         )
         assert analysis.get_symbol_table() is not None
 
+def test_get_symbol_table_source_code(java_code):
+    """Should return a symbol table for source analysis with expected class/method count"""
+
+    # Initialize the CLDK object with the project directory, language, and analysis_backend
+    cldk = CLDK(language="java")
+    analysis = cldk.analysis(
+        source_code=java_code,
+        analysis_backend_path=None,
+        eager=True,
+        analysis_level=AnalysisLevel.symbol_table,
+    )
+
+    # assert on expected class name and method count in the symbol table
+    expected_class_name = "com.acme.modres.WeatherServlet"
+    assert analysis.get_symbol_table() is not None
+    assert len(analysis.get_symbol_table().keys()) == 1
+    assert expected_class_name in analysis.get_methods().keys()
+    assert len(analysis.get_methods().get(expected_class_name).keys()) == 9
 
 def test_get_imports(test_fixture, analysis_json):
     """Should return NotImplemented for get_imports()"""
@@ -632,11 +650,11 @@ def test_get_methods_in_class(test_fixture, analysis_json):
             eager_analysis=False,
         )
 
-        # Test that there are 30 methods in the Log class
+        # Test that there are 29 methods in the Log class
         methods = java_analysis.get_methods_in_class("com.ibm.websphere.samples.daytrader.util.Log")
         assert methods is not None
         assert isinstance(methods, Dict)
-        assert len(methods) == 30
+        assert len(methods) == 29
         for method in methods:
             assert isinstance(methods[method], JCallable)
 

pyproject.toml

@@ -67,7 +67,7 @@ def divide(self, a, b):
 
 def test_get_methods():
     """Should return all of the methods"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     all_methods = python_analysis.get_methods()
     assert all_methods is not None
@@ -79,7 +79,7 @@ def test_get_methods():
 
 def test_get_functions():
     """Should return all of the functions"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     all_functions = python_analysis.get_functions()
     assert all_functions is not None
@@ -91,7 +91,7 @@ def test_get_functions():
 
 def test_get_all_modules(tmp_path):
     """Should return all of the modules"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=tmp_path, source_code=None, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=tmp_path, source_code=None)
 
     # set up some temporary modules
     temp_file_path = os.path.join(tmp_path, "hello.py")
@@ -111,7 +111,7 @@ def test_get_all_modules(tmp_path):
 
 def test_get_method_details():
     """Should return the method details"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     method_details = python_analysis.get_method_details("add(self, a, b)")
     assert method_details is not None
@@ -121,7 +121,7 @@ def test_get_method_details():
 
 def test_is_parsable():
     """Should be able to parse the code"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     code = "def is_parsable(self, code: str) -> bool: return True"
     is_parsable = python_analysis.is_parsable(code)
@@ -134,7 +134,7 @@ def test_is_parsable():
 
 def test_get_raw_ast():
     """Should return the raw AST"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     raw_ast = python_analysis.get_raw_ast(PYTHON_CODE)
     assert raw_ast is not None
@@ -144,7 +144,7 @@ def test_get_raw_ast():
 
 def test_get_imports():
     """Should return all of the imports"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     all_imports = python_analysis.get_imports()
     assert all_imports is not None
@@ -156,7 +156,7 @@ def test_get_imports():
 
 def test_get_variables():
     """Should return all of the variables"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_variables()
@@ -165,7 +165,7 @@ def test_get_variables():
 
 def test_get_classes():
     """Should return all of the classes"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     all_classes = python_analysis.get_classes()
     assert all_classes is not None
@@ -178,7 +178,7 @@ def test_get_classes():
 
 def test_get_classes_by_criteria():
     """Should return all of the classes that match the criteria"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_classes_by_criteria()
@@ -187,7 +187,7 @@ def test_get_classes_by_criteria():
 
 def test_get_sub_classes():
     """Should return all of the subclasses"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_sub_classes()
@@ -196,7 +196,7 @@ def test_get_sub_classes():
 
 def test_get_nested_classes():
     """Should return all of the nested classes"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_nested_classes()
@@ -205,7 +205,7 @@ def test_get_nested_classes():
 
 def test_get_constructors():
     """Should return all of the constructors"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_constructors()
@@ -214,7 +214,7 @@ def test_get_constructors():
 
 def test_get_methods_in_class():
     """Should return all of the methods in the class"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_methods_in_class()
@@ -223,7 +223,7 @@ def test_get_methods_in_class():
 
 def test_get_fields():
     """Should return all of the fields in the class"""
-    python_analysis = PythonAnalysis(eager_analysis=True, project_dir=None, source_code=PYTHON_CODE, analysis_backend_path=None, analysis_json_path=None)
+    python_analysis = PythonAnalysis(project_dir=None, source_code=PYTHON_CODE)
 
     with pytest.raises(NotImplementedError) as except_info:
         python_analysis.get_fields()

tests/analysis/java/test_java_analysis.py

@@ -75,7 +75,7 @@ def codeanalyzer_jar_path():
     # Load the configuration
     config = toml.load(pyproject_path)
 
-    return Path(config["tool"]["cldk"]["testing"]["codeanalyzer-jar-path"]) / "2.3.0"
+    return Path(config["tool"]["cldk"]["testing"]["codeanalyzer-jar-path"])
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -190,3 +190,24 @@ def test_fixture_binutils():
     for directory in Path(test_data_path).iterdir():
         if directory.exists() and directory.is_dir():
             shutil.rmtree(directory)
+
+@pytest.fixture(scope="session", autouse=True)
+def java_code() -> str:
+    """
+    Returns sample Java source code for analysis.
+
+    Yields:
+        str : Java code to be analyzed.
+    """
+    # ----------------------------------[ SETUP ]----------------------------------
+    # Path to your pyproject.toml
+    pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
+
+    # Load the configuration
+    config = toml.load(pyproject_path)
+
+    # Access the test data path
+    test_data_path = config["tool"]["cldk"]["testing"]["sample-application"]
+    javafile = Path(test_data_path).absolute() / ("WeatherServlet.java")
+    with open(javafile) as f:
+        return f.read()