diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0dda56a2..bf292a3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,14 @@ jobs: - name: Build Package run: poetry build + - name: Read Changelog Entry + id: changelog_reader + uses: mindsers/changelog-reader-action@v2 + with: + validation_level: warn + version: ${{ steps.tag_name.outputs.current_version }} + path: ./CHANGELOG.md + - name: Build Changelog id: gen_changelog uses: mikepenz/release-changelog-builder-action@v5 diff --git a/.pypirc b/.pypirc new file mode 100644 index 00000000..aba5d5f4 --- /dev/null +++ b/.pypirc @@ -0,0 +1,3 @@ +[testpypi] + username = __token__ + password = pypi-AgENdGVzdC5weXBpLm9yZwIkZTc4MDc4ZDgtZjE3Yy00MzBkLThlNzgtMWM5OTYyYmQ4MmJlAAIqWzMsImQ0ODAwOGM2LTk5NzEtNGNkMS1iODRjLTE1ZmI5MzUzMTVmMCJdAAAGIGwt_2ZTCD_lxDBM2qgoO_JYk0yzX1xsIfIbbn4Dq6VX diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..73237732 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,221 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [v1.0.7] - 2025-08-21 + +### Added +- Doctest-style Examples across the public API surface of JavaAnalysis, PythonAnalysis, CAnalysis, and core CLDK helpers. Coverage includes Java CRUD operations and comment/docstring query APIs, plus concise inline examples for Python and C where applicable. +- Examples documenting expected NotImplementedError behavior for placeholder APIs (PythonAnalysis and CAnalysis) using doctest flags. + +### Changed +- Converted and standardized docstrings to strict Google style (Args, Returns, Raises, Examples) across edited modules. +- Standardized Examples to use the CLDK facade (e.g., `CLDK(language="java").analysis(...)`) instead of raw constructor calls. +- Normalized all doctest Example inputs to single-line strings to ensure reliable mkdocstrings rendering. +- Clarified `CLDK.analysis` return type with a precise union: `JavaAnalysis | PythonAnalysis | CAnalysis`. + +### Fixed +- Fixed README.md logo display on PyPI by updating image URLs to use raw GitHub URLs and maintaining theme-based auto-switching with proper fallback +- mkdocstrings rendering issues caused by multi-line doctest strings and formatting inconsistencies. +- Replaced confusing examples like `JavaAnalysis(None, None, ...)` with clear CLDK-based initialization patterns. +- Packaging: ensured the built wheel includes the `cldk` package by adding `packages = [{ include = "cldk" }]` to Poetry configuration. + +### Removed +- Multi-line doctest strings in Examples that broke mkdocstrings rendering; all examples are now single-line. + +## [v1.0.6] - 2025-07-23 + +### Added +- Added `argument_expr` field to JCallSite model for capturing actual parameter expressions in method calls +- Added Star History section to README.md for tracking project popularity + +### Changed +- Updated codeanalyzer jar to version 2.3.5 with support for call argument expressions and fully qualified parameter types +- Modified codeanalyzer.py to preserve fully qualified parameter types in method signatures instead of simplifying them +- Updated method signature format to use fully qualified type names (e.g., `java.lang.String` instead of `String`) +- Updated test fixtures with new analysis.json data reflecting the signature format changes + +### Fixed +- Fixed method signature handling to maintain fully qualified parameter types for better type resolution +- Updated test cases to use fully qualified method signatures for improved accuracy + +## [v1.0.5] - 2025-06-24 + +### Fixed +- Fixed issue #135 +- Analysis level compatibility checking for analysis.json with passed analysis level + +### Changed +- Updated treesitter analysis to use global declarations of parser and language + +## [v1.0.4] - 2025-06-11 + +### Added +- Added missing callable fields field validator + +### Changed +- Updated test fixture setup to use codeanalyzer jar from cldk/analysis/java/codeanalyzer/jar instead of test resources directory +- Updated analysis.json fixtures (daytrader8 and plantsbywebsphere) + +### Removed +- Removed dangling codeanalyzer jars from test resources +- Removed obsolete analysis.json fixture + +## [v1.0.3] - 2025-06-01 + +### Added +- Added code start line attribute to JCallable (corresponding to added attribute in the java code analyzer model) + +## [v1.0.2] - 2025-05-24 + +### Added +- Added test case and fixture for source analysis +- Added missing attributes in compilation unit model + +### Fixed +- Fixed handling of `source_code` option in Java codeanalyzer +- Updated core.py to match python analysis signature + +## [v1.0.1] - 2025-05-07 + +### Changed +- Updated treesitter analysis to use global declarations of parser and language + +## [v1.0.0] - 2025-04-29 + +### Added +- First stable release +- Updated contributing guidelines + +### Changed +- Updated README.md +- Updated codeanalyzer jar +- Updated java version in release automation + +## [v0.5.1] - 2025-03-13 + +### Changed +- Updated Java model to comply with codeanalyzer v2.3.1 +- Updated codeanalyzer jar to the latest from codeanalyzer-java +- Updated get_all_docstrings to return dict + +## [v0.5.0] - 2025-02-21 + +### Added +- Added release automation github actions +- Added Java 11 support in github actions +- Added release_config.json +- Added Comment parsing APIs at file, class, method, and docstring level +- Added support for parsing callable parameters and their location information +- Added Dev container instructions with Python, Java, C, and Rust support +- Added C/C++ analysis support +- Added CRUD operations support for Java JPA applications + +### Changed +- Consolidated analysis_level enums in __init__.py +- Updated codeanalyzer jar to the latest version +- Changed coverage minimum to 70% +- Updated documentation with mkdocs +- Updated badges and logos in README +- Added Discord community support + +### Removed +- Removed CodeQL dependency and refactored treesitter +- Removed ABCs from analysis +- Removed logic to find LLVM in linux OSes (only appears in Darwin) +- Removed redundant is_entry_point fields from JCallable and JType +- Removed unused parameters and code cleanup + +### Fixed +- Fixed various test cases and compatibility issues +- Fixed treesitter superclass identification issues +- Fixed entry point detection code +- Fixed recursive error issues + +## [v0.4.0] - 2024-11-13 + +### Fixed +- Fixed issue 67 - symbol table is none + +### Changed +- Updated poetry build rules to include codeanalyzer-*.jar +- Added test case to verify jar file exists + +## [v0.3.0] - 2024-11-12 + +### Added +- Support for reading slim JSON from codeanalyzer v1.1.0 +- Added more test tools (pylint, flake8, black, pspec, coverage) +- Added test coverage reporting + +### Changed +- Updated README.md to include the arXiv paper +- Removed obsolete test cases for unsupported languages + +## [v0.2.0] - 2024-10-11 + +### Added +- Added GitHub Action to publish manual releases +- Added PyPi badge to README.md + +## [v0.1.4] - 2024-10-21 + +### Fixed +- Fixed codeanalyzer.jar not being a PosixPath + +## [v0.1.3] - 2024-10-21 + +### Fixed +- Fixed calling the correct codeanalyzer jar on version 0.1.3 +- Removed auto-download of codeanalyzer jar + +## [v0.1.2] - 2024-10-17 + +### Fixed +- Fixed tree-sitter bug +- Defined self.captures explicitly + +## [0.1.0-dev] - 2024-10-07 + +### Added +- Initial development version +- Set version to über json support +- Support for slim JSONs from codeanalyzer +- IBM Copyright added to all source files +- Added code parsing support +- Added support for symbol table call graph +- Added notebook examples for code summarization and test generation +- Basic CLDK framework implementation + +### Changed +- Updated dependencies in pyproject.toml +- Added metadata for PyPi distribution +- Updated README with installation instructions + +### Fixed +- Fixed caller method implementation +- Fixed incremental analysis support +- Fixed download jar issues + +--- + +## Release Links + +- [v1.0.5]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.5 +- [v1.0.4]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.4 +- [v1.0.3]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.3 +- [v1.0.2]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.2 +- [v1.0.1]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.1 +- [v1.0.0]: https://github.com/codellm-devkit/python-sdk/releases/tag/v1.0.0 +- [v0.5.1]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.5.1 +- [v0.5.0]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.5.0 +- [v0.4.0]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.4.0 +- [v0.3.0]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.3.0 +- [v0.2.0]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.2.0 +- [v0.1.4]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.1.4 +- [v0.1.3]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.1.3 +- [v0.1.2]: https://github.com/codellm-devkit/python-sdk/releases/tag/v0.1.2 +- [0.1.0-dev]: https://github.com/codellm-devkit/python-sdk/releases/tag/0.1.0-dev diff --git a/README.md b/README.md index d0d797e4..e12efefa 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,4 @@ - - - - Logo - +Logo

@@ -251,13 +247,3 @@ We invite contributors of all levels of experience! We would love to see you get 1. Krishna, Rahul, Rangeet Pan, Raju Pavuluri, Srikanth Tamilselvam, Maja Vukovic, and Saurabh Sinha. "[Codellm-Devkit: A Framework for Contextualizing Code LLMs with Program Analysis Insights.](https://arxiv.org/pdf/2410.13007)" arXiv preprint arXiv:2410.13007 (2024). 2. Pan, Rangeet, Myeongsoo Kim, Rahul Krishna, Raju Pavuluri, and Saurabh Sinha. "[Multi-language Unit Test Generation using LLMs.](https://arxiv.org/abs/2409.03093)" arXiv preprint arXiv:2409.03093 (2024). 3. Pan, Rangeet, Rahul Krishna, Raju Pavuluri, Saurabh Sinha, and Maja Vukovic., "[Simplify your Code LLM solutions using CodeLLM Dev Kit (CLDK).](https://www.linkedin.com/pulse/simplify-your-code-llm-solutions-using-codellm-dev-kit-rangeet-pan-vnnpe/?trackingId=kZ3U6d8GSDCs8S1oApXZgg%3D%3D)", Blog. - -## Star History - - - - - - Star History Chart - - diff --git a/cldk/analysis/c/c_analysis.py b/cldk/analysis/c/c_analysis.py index 12cf9d96..3e443031 100644 --- a/cldk/analysis/c/c_analysis.py +++ b/cldk/analysis/c/c_analysis.py @@ -14,11 +14,12 @@ # limitations under the License. ################################################################################ -""" -Analysis model for C projects +"""C analysis utilities. + +Provides a high-level API to analyze C projects using a Clang-based analyzer +and to query functions, macros, typedefs, structs/unions, enums, and globals. """ -import os from pathlib import Path from typing import Dict, List, Optional import networkx as nx @@ -30,19 +31,31 @@ class CAnalysis: def __init__(self, project_dir: Path) -> None: - """Initialization method for C Analysis backend.""" + """Initialize the C analysis backend. + + Args: + project_dir (Path): Path to the C project directory. + """ if not isinstance(project_dir, Path): project_dir = Path(project_dir) self.c_application = self._init_application(project_dir) def _init_application(self, project_dir: Path) -> CApplication: - """Should initialize the C application object. + """Construct the C application model from project sources. Args: project_dir (Path): Path to the project directory. Returns: - CApplication: C application object. + CApplication: Application model. + + Examples: + Build an application model from a project directory: + + >>> from pathlib import Path + >>> ca = CAnalysis(project_dir=Path('.')) # doctest: +SKIP + >>> isinstance(ca.get_c_application(), CApplication) # doctest: +SKIP + True """ analyzer = ClangAnalyzer() @@ -56,185 +69,297 @@ def _init_application(self, project_dir: Path) -> CApplication: return CApplication(translation_units=translation_units) def get_c_application(self) -> CApplication: - """Obtain the C application object. + """Return the C application object. Returns: - CApplication: C application object. + CApplication: Application model. + + Examples: + >>> # Assuming CAnalysis was constructed + >>> isinstance(CAnalysis(project_dir=Path('.')).get_c_application(), CApplication) # doctest: +SKIP + True """ return self.c_application def get_imports(self) -> List[str]: + """Return all include/import statements in the project. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> from pathlib import Path + >>> CAnalysis(project_dir=Path('.')).get_imports() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. + """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_variables(self, **kwargs): + """Return all variables discovered across the project. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> from pathlib import Path + >>> CAnalysis(project_dir=Path('.')).get_variables() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. + """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_application_view(self) -> CApplication: + """Return the application view of the C project. + + Returns: + CApplication: Application model summarizing translation units. + + Examples: + >>> from pathlib import Path + >>> ca = CAnalysis(project_dir=Path('.')) # doctest: +SKIP + >>> isinstance(ca.get_application_view(), CApplication) # doctest: +SKIP + True + """ return self.c_application def get_symbol_table(self) -> Dict[str, CTranslationUnit]: + """Return a symbol table view keyed by file path. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> from pathlib import Path + >>> CAnalysis(project_dir=Path('.')).get_symbol_table() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. + """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_compilation_units(self) -> List[CTranslationUnit]: + """Return all compilation units parsed from C sources. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> from pathlib import Path + >>> CAnalysis(project_dir=Path('.')).get_compilation_units() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. + """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def is_parsable(self, source_code: str) -> bool: - """ - Check if the code is parsable using clang parser. + """Check if the source code is parsable using Clang. + Args: - source_code: source code + source_code (str): Source code to parse. Returns: - True if the code is parsable, False otherwise + bool: True if parsable, False otherwise. + + Examples: + >>> CAnalysis(project_dir=Path('.')).is_parsable('int f(){return 1;}') # doctest: +SKIP + True """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_call_graph(self) -> nx.DiGraph: - """Should return the call graph of the C code. + """Return the call graph of the C code. Returns: - nx.DiGraph: The call graph of the C code. + networkx.DiGraph: Call graph. + + Examples: + >>> isinstance(CAnalysis(project_dir=Path('.')).get_call_graph(), nx.DiGraph) # doctest: +SKIP + True """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_call_graph_json(self) -> str: - """Should return a serialized call graph in json. + """Return the call graph serialized as JSON. + + Returns: + str: Call graph encoded as JSON. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: Single-file mode unsupported. - Returns: - str: Call graph in json. + Examples: + >>> isinstance(CAnalysis(project_dir=Path('.')).get_call_graph_json(), str) # doctest: +SKIP + True """ raise NotImplementedError("Producing a call graph over a single file is not implemented yet.") def get_callers(self, function: CFunction) -> Dict: - """Should return a dictionary of callers of the target method. + """Return callers of a function. Args: - function (CFunction): A CFunction object. + function (CFunction): Target function. + + Returns: + dict: Mapping of callers to call sites/details. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: Not implemented yet. - Returns: - Dict: A dictionary of callers of target function. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_callers(CFunction(name='f', return_type='int', parameters=[], storage_class=None, is_inline=False, is_variadic=False, body='', comment='', call_sites=[], local_variables=[], start_line=1, end_line=1)) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Generating all callers over a single file is not implemented yet. """ raise NotImplementedError("Generating all callers over a single file is not implemented yet.") def get_callees(self, function: CFunction) -> Dict: - """Should return a dictionary of callees in a fuction. + """Return callees of a function. Args: - function (CFunction): A CFunction object. + function (CFunction): Source function. + + Returns: + dict: Callee details. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: Not implemented yet. - Returns: - Dict: Dictionary with callee details. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_callees(CFunction(name='f', return_type='int', parameters=[], storage_class=None, is_inline=False, is_variadic=False, body='', comment='', call_sites=[], local_variables=[], start_line=1, end_line=1)) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Generating all callees over a single file is not implemented yet. """ raise NotImplementedError("Generating all callees over a single file is not implemented yet.") def get_functions(self) -> Dict[str, CFunction]: - """Should return all functions in the project. - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return all functions in the project. Returns: - Dict[str, Dict[str, JCallable]]: Dictionary of dictionaries of all methods in the C code with qualified class name as key and dictionary of methods in that class. + dict[str, CFunction]: Functions keyed by signature/name for a translation unit. + + Examples: + >>> funcs = CAnalysis(project_dir=Path('.')).get_functions() # doctest: +SKIP + >>> isinstance(funcs, dict) # doctest: +SKIP + True """ for _, translation_unit in self.c_application.translation_units.items(): return translation_unit.functions def get_function(self, function_name: str, file_name: Optional[str]) -> CFunction | List[CFunction]: - """Should return a function object given the function name. + """Return a function object. Args: - function_name (str): The name of the function. - file_name (str): The name of the file containing the function. + function_name (str): Function name. + file_name (str | None): Optional file name. Returns: - CFunction: A method for the given qualified method name. If multiple functions with the same name exist, a list of functions is returned. + CFunction | list[CFunction]: Function object(s) matching the query. + + Examples: + >>> CAnalysis(project_dir=Path('.')).get_function('main', None) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_C_file(self, file_name: str) -> str: - """Should return a class given qualified class name. + """Return a C file path by name. Args: - file_name (str): The name of the file. + file_name (str): File name. + + Returns: + str: C source file path. Raises: - NotImplementedError: Raised when we do not support this function. + NotImplementedError: Not implemented yet. - Returns: - str: C file name containing the given qualified class. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_C_file('hello.c') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_C_compilation_unit(self, file_path: str) -> CTranslationUnit: - """Given the path of a C source file, returns the compilation unit object from the symbol table. + """Return the compilation unit for a C source file. Args: - file_path (str): Absolute path to C source file - - Raises: - NotImplementedError: Raised when we do not support this function. + file_path (str): Absolute path to a C source file. Returns: - CTranslationUnit: Compilation unit object for C source file + CTranslationUnit: Compilation unit object. + + Examples: + >>> # Retrieve a compilation unit by path + >>> cu = CAnalysis(project_dir=Path('.')).get_C_compilation_unit('file.c') # doctest: +SKIP + >>> (cu is None) or hasattr(cu, 'functions') # doctest: +SKIP + True """ return self.c_application.translation_units.get(file_path) def get_functions_in_file(self, file_name: str) -> List[CFunction]: - """Should return a dictionary of all methods of the given class. + """Return all functions in a given file. Args: - file_name (str): The name of the file. + file_name (str): File name. + + Returns: + list[CFunction]: Functions in the file. Raises: - NotImplementedError: Raised when we do not support this function. + NotImplementedError: Not implemented yet. - Returns: - Dict[str, JCallable]: A dictionary of all constructors of the given class. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_functions_in_file('file.c') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_macros(self) -> List[CMacro]: - """Should return a list of all macros in the C code. + """Return all macros in the project. + + Returns: + list[CMacro]: All macros. Raises: - NotImplementedError: Raised when we do not support this function. + NotImplementedError: Not implemented yet. - Returns: - List[CMacro]: A list of all macros in the C code. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_macros() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_macros_in_file(self, file_name: str) -> List[CMacro] | None: - """Should return a list of all macros in the given file. + """Return all macros in the given file. Args: - file_name (str): The name of the file. + file_name (str): File name. + + Returns: + list[CMacro] | None: Macros in the file, or None if not found. Raises: - NotImplementedError: Raised when we do not support this function. + NotImplementedError: Not implemented yet. - Returns: - List[CMacro]: A list of all macros in the given file. Returns None if no macros are found. + Examples: + >>> CAnalysis(project_dir=Path('.')).get_macros_in_file('file.c') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_includes(self) -> List[str]: - """Should return a list of all include statements across all files in the C code. + """Return all include statements across the project. Returns: - List[str]: A list of all include statements. Returns empty list if none found. + list[str]: All include statements. """ all_includes = [] for translation_unit in self.translation_units.values(): @@ -243,13 +368,13 @@ def get_includes(self) -> List[str]: def get_includes_in_file(self, file_name: str) -> List[str] | None: - """Should return a list of all include statements in the given file. + """Return include statements in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[str] | None: List of includes in the file, or None if file not found. + list[str] | None: Includes in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].includes @@ -257,10 +382,10 @@ def get_includes_in_file(self, file_name: str) -> List[str] | None: def get_macros(self) -> List[CMacro]: - """Should return a list of all macro definitions across all files in the C code. + """Return all macro definitions across the project. Returns: - List[CMacro]: A list of all macro definitions. Returns empty list if none found. + list[CMacro]: Macro definitions. """ all_macros = [] for translation_unit in self.translation_units.values(): @@ -269,13 +394,13 @@ def get_macros(self) -> List[CMacro]: def get_macros_in_file(self, file_name: str) -> List[CMacro] | None: - """Should return a list of all macro definitions in the given file. + """Return macro definitions in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[CMacro] | None: List of macros in the file, or None if file not found. + list[CMacro] | None: Macros in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].macros @@ -283,10 +408,10 @@ def get_macros_in_file(self, file_name: str) -> List[CMacro] | None: def get_typedefs(self) -> List[CTypedef]: - """Should return a list of all typedef declarations across all files in the C code. + """Return typedef declarations across the project. Returns: - List[CTypedef]: A list of all typedef declarations. Returns empty list if none found. + list[CTypedef]: Typedef declarations. """ all_typedefs = [] for translation_unit in self.translation_units.values(): @@ -295,13 +420,13 @@ def get_typedefs(self) -> List[CTypedef]: def get_typedefs_in_file(self, file_name: str) -> List[CTypedef] | None: - """Should return a list of all typedef declarations in the given file. + """Return typedef declarations in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[CTypedef] | None: List of typedefs in the file, or None if file not found. + list[CTypedef] | None: Typedefs in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].typedefs @@ -309,10 +434,10 @@ def get_typedefs_in_file(self, file_name: str) -> List[CTypedef] | None: def get_structs(self) -> List[CStruct]: - """Should return a list of all struct/union declarations across all files in the C code. + """Return struct/union declarations across the project. Returns: - List[CStruct]: A list of all struct/union declarations. Returns empty list if none found. + list[CStruct]: Struct/union declarations. """ all_structs = [] for translation_unit in self.translation_units.values(): @@ -321,13 +446,13 @@ def get_structs(self) -> List[CStruct]: def get_structs_in_file(self, file_name: str) -> List[CStruct] | None: - """Should return a list of all struct/union declarations in the given file. + """Return struct/union declarations in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[CStruct] | None: List of structs in the file, or None if file not found. + list[CStruct] | None: Structs in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].structs @@ -335,10 +460,10 @@ def get_structs_in_file(self, file_name: str) -> List[CStruct] | None: def get_enums(self) -> List[CEnum]: - """Should return a list of all enum declarations across all files in the C code. + """Return enum declarations across the project. Returns: - List[CEnum]: A list of all enum declarations. Returns empty list if none found. + list[CEnum]: Enum declarations. """ all_enums = [] for translation_unit in self.translation_units.values(): @@ -347,13 +472,13 @@ def get_enums(self) -> List[CEnum]: def get_enums_in_file(self, file_name: str) -> List[CEnum] | None: - """Should return a list of all enum declarations in the given file. + """Return enum declarations in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[CEnum] | None: List of enums in the file, or None if file not found. + list[CEnum] | None: Enums in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].enums @@ -361,13 +486,13 @@ def get_enums_in_file(self, file_name: str) -> List[CEnum] | None: def get_globals(self, file_name: str) -> List[CVariable] | None: - """Should return a list of all global variable declarations in the given file. + """Return global variable declarations in a file. Args: - file_name (str): The name of the file to search in. + file_name (str): File name. Returns: - List[CVariable] | None: List of globals in the file, or None if file not found. + list[CVariable] | None: Globals in the file, or None if not found. """ if file_name in self.translation_units: return self.translation_units[file_name].globals diff --git a/cldk/analysis/c/clang/clang_analyzer.py b/cldk/analysis/c/clang/clang_analyzer.py index 3ce57cdd..f25c7ab4 100644 --- a/cldk/analysis/c/clang/clang_analyzer.py +++ b/cldk/analysis/c/clang/clang_analyzer.py @@ -1,25 +1,39 @@ +"""Analyze C/C++ code using Clang's Python bindings. + +This module provides a thin wrapper around libclang to parse translation +units, extract functions, variables, includes, and call sites, and produce +CLDK C models. It attempts to locate libclang on macOS and Linux. +""" + import os -from pdb import set_trace import platform -from clang.cindex import Config from pathlib import Path from typing import List, Optional -from cldk.models.c import CFunction, CCallSite, CTranslationUnit, CApplication +from cldk.models.c import CFunction, CCallSite, CTranslationUnit import logging from cldk.models.c.models import CInclude, CParameter, CVariable, StorageClass -logger = logging.getLogger(__name__) - -# First, we only import Config from clang.cindex from clang.cindex import Config from clang.cindex import Index, TranslationUnit, CursorKind, TypeKind, CompilationDatabase +logger = logging.getLogger(__name__) + class ClangAnalyzer: - """Analyzes C code using Clang's Python bindings.""" + """Analyze C/C++ code using Clang's Python bindings. + + This analyzer creates a Clang index, optionally uses a compilation + database for compile flags, and walks the AST to build CLDK models. + """ def __init__(self, compilation_database_path: Optional[Path] = None): + """Initialize the analyzer and libclang configuration. + + Args: + compilation_database_path (Path | None): Optional path to a + compilation database (compile_commands.json directory). + """ # # Let's turn off Address sanitization for parsing code # # Initialize libclang at module level # try: @@ -49,10 +63,14 @@ def __init__(self, compilation_database_path: Optional[Path] = None): self.compilation_database = CompilationDatabase.fromDirectory(str(compilation_database_path)) def __find_libclang(self) -> str: - """ - Locates the libclang library on the system based on the operating system. - This function runs before any other Clang functionality is used, ensuring - proper initialization of the Clang environment. + """Locate the libclang library on the system. + + Returns: + str: Absolute path to the libclang shared library. + + Raises: + RuntimeError: If the operating system is unsupported or libclang + cannot be found. Error message includes installation hints. """ system = platform.system() @@ -68,11 +86,9 @@ def __find_libclang(self) -> str: # On Linux, we check various common installation paths elif system == "Linux": - from pathlib import Path - lib_paths = [Path("/usr/lib"), Path("/usr/lib64")] possible_paths = [str(p) for base in lib_paths if base.exists() for p in base.rglob("libclang*.so.17*")] - print(possible_paths) + logger.debug(f"Candidate libclang paths: {possible_paths}") install_instructions = "Install libclang development package using your system's package manager" else: raise RuntimeError(f"Unsupported operating system: {system}") @@ -87,7 +103,14 @@ def __find_libclang(self) -> str: raise RuntimeError(f"Could not find libclang library. \n" f"Please ensure LLVM is installed:\n{install_instructions}") def analyze_file(self, file_path: Path) -> CTranslationUnit: - """Analyzes a single C source file using Clang.""" + """Analyze a single C/C++ source file using Clang. + + Args: + file_path (Path): Path to the source file to analyze. + + Returns: + CTranslationUnit: Parsed translation unit model with functions and includes. + """ # Get compilation arguments if available compile_args = self._get_compile_args(file_path) @@ -110,7 +133,12 @@ def analyze_file(self, file_path: Path) -> CTranslationUnit: return translation_unit def _process_translation_unit(self, cursor, translation_unit: CTranslationUnit): - """Should process all declarations in a translation unit.""" + """Process all declarations in a translation unit. + + Args: + cursor: Root cursor of the translation unit. + translation_unit (CTranslationUnit): Model to populate. + """ for child in cursor.get_children(): if child.location.file and str(child.location.file) != translation_unit.file_path: @@ -126,22 +154,18 @@ def _process_translation_unit(self, cursor, translation_unit: CTranslationUnit): translation_unit.includes.append(include) def _process_inclusion(self, cursor): - """ - Processes an include directive, capturing both the include type and the included file. + """Process an include directive and capture metadata. Args: - cursor: Cursor to the include directive - translation_unit: Translation unit being processed + cursor: Cursor pointing to an inclusion directive. Returns: + CInclude: Include info with name, system/local flag, line number, and full text. - - In C/C++, we have two main types of includes: - 1. System includes: #include - Usually for standard libraries - 2. Local includes: #include "header.h" - Usually for your own headers - - The function captures this distinction and stores additional metadata about - the inclusion. + Notes: + C/C++ include forms: + - System includes: #include + - Local includes: #include "header.h" """ include_name = cursor.displayname include_location = cursor.location @@ -164,12 +188,16 @@ def _process_inclusion(self, cursor): return CInclude(name=include_name, is_system=is_system_include, line_number=include_location.line, full_text=full_text) def _extract_parameter(self, param) -> CParameter: - """ - Extracts parameter information, handling default values carefully. + """Extract parameter info, including a best-effort default value. - In C++, parameters can have default values, but accessing these requires - careful token handling since the tokens form a generator that can only - be consumed once. + In C++, parameters can have default values, but token streams are + generators and must be consumed carefully. + + Args: + param: Cursor for the parameter. + + Returns: + CParameter: Parameter model with name, type, and optional default. """ default_value = None try: @@ -182,7 +210,14 @@ def _extract_parameter(self, param) -> CParameter: return CParameter(name=param.spelling or f"placeholder_arg_{param.type.spelling.replace(' ', '_')}", type=param.type.spelling, default_value=default_value) def _extract_variable(self, cursor) -> CVariable: - """Extracts detailed variable information from a cursor.""" + """Extract detailed variable information from a cursor. + + Args: + cursor: Cursor for the variable declaration. + + Returns: + CVariable: Variable model with qualifiers and source span. + """ return CVariable( name=cursor.spelling, type=cursor.type.spelling, @@ -195,22 +230,22 @@ def _extract_variable(self, cursor) -> CVariable: ) def _extract_function_body(self, cursor) -> str: - """Extracts the body of a function. + """Extract a function's body as a string. Args: - cursor: Cursor to the function + cursor: Cursor to the function. Returns: - str: The function body + str: Function body text, or an empty string if unavailable. """ - if cursor.is_definition() == False: + if not cursor.is_definition(): return "" try: tokens = list(cursor.get_tokens()) try: body_start = next(i for i, t in enumerate(tokens) if t.spelling == "{") - except: + except Exception: return "" brace = 0 @@ -236,7 +271,14 @@ def _extract_function_body(self, cursor) -> str: return "" def _extract_function(self, cursor) -> CFunction: - """Extracts detailed function information from a cursor.""" + """Extract detailed function information from a cursor. + + Args: + cursor: Cursor for a function declaration/definition. + + Returns: + CFunction: Function model including signature, body, calls, and locals. + """ # Get storage class storage_class = None @@ -278,7 +320,14 @@ def _extract_function(self, cursor) -> CFunction: ) def _extract_call_site(self, cursor) -> CCallSite: - """Extracts information about a function call.""" + """Extract information about a function call. + + Args: + cursor: Cursor pointing to a call expression. + + Returns: + CCallSite: Call site model with callee name, arg types, and span. + """ # Determine if this is an indirect call (through function pointer) is_indirect = cursor.referenced is None and cursor.type.kind == TypeKind.FUNCTIONPROTO @@ -300,7 +349,14 @@ def _extract_call_site(self, cursor) -> CCallSite: ) def _get_compile_args(self, file_path: Path) -> List[str]: - """Gets compilation arguments for a file.""" + """Get compilation arguments for a file. + + Args: + file_path (Path): Source file path. + + Returns: + list[str]: Compiler arguments for parsing. + """ if not self.compilation_database: return ["-x", "c++", "-std=c++17"] diff --git a/cldk/analysis/commons/treesitter/treesitter_java.py b/cldk/analysis/commons/treesitter/treesitter_java.py index a7f68ad7..2f639c78 100644 --- a/cldk/analysis/commons/treesitter/treesitter_java.py +++ b/cldk/analysis/commons/treesitter/treesitter_java.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -TreesitterJava module +"""Java tree-sitter queries and helpers. + +Provides utilities to parse Java source text with tree-sitter and extract +classes, methods, interfaces, invocations, comments, and related metadata. """ import logging from itertools import groupby @@ -32,40 +34,33 @@ # pylint: disable=too-many-public-methods class TreesitterJava: - """ - Treesitter for Java usecases. - """ + """Tree-sitter helpers for Java use cases.""" def __init__(self) -> None: pass def method_is_not_in_class(self, method_name: str, class_body: str) -> bool: - """Check if a method is in a class. - - Parameters - ---------- - method_name : str - The name of the method to check. - class_body : str - The body of the class to check. - - Returns - ------- - bool - True if the method is in the class, False otherwise. + """Return True if the method is not declared in the class body. + + Args: + method_name (str): Method name to check. + class_body (str): Class source body. + + Returns: + bool: True if absent, False otherwise. """ methods_in_class = self.frame_query_and_capture_output("(method_declaration name: (identifier) @name)", class_body) return method_name not in {method.node.text.decode() for method in methods_in_class} def is_parsable(self, code: str) -> bool: - """ - Check if the code is parsable + """Check whether the Java code parses without syntax errors. + Args: - code: source code + code (str): Source code. Returns: - True if the code is parsable, False otherwise + bool: True if parsable, False otherwise. """ def syntax_error(node): @@ -86,37 +81,37 @@ def syntax_error(node): return False def get_raw_ast(self, code: str) -> Tree: - """ - Get the raw AST + """Parse and return the raw AST. + Args: - code: source code + code (str): Source code. Returns: - Tree: the raw AST + Tree: Parsed AST. """ return PARSER.parse(bytes(code, "utf-8")) def get_all_imports(self, source_code: str) -> Set[str]: - """Get a list of all the imports in a class. + """Return all import statements in the source. Args: - source_code (str): The source code to process. + source_code (str): Java source. Returns: - Set[str]: A set of all the imports in the class. + set[str]: Import identifiers. """ import_declarations: Captures = self.frame_query_and_capture_output(query="(import_declaration (scoped_identifier) @name)", code_to_process=source_code) return {capture.node.text.decode() for capture in import_declarations} # TODO: This typo needs to be fixed (i.e., package not pacakge) def get_pacakge_name(self, source_code: str) -> str: - """Get the package name from the source code. + """Return the package name from the source code. Args: - source_code (str): The source code to process. + source_code (str): Java source. Returns: - str: The package name. + str: Package declaration value, or None if absent. """ package_name: Captures = self.frame_query_and_capture_output(query="((package_declaration) @name)", code_to_process=source_code) if package_name: @@ -124,25 +119,25 @@ def get_pacakge_name(self, source_code: str) -> str: return None def get_class_name(self, source_code: str) -> str: - """Get the class name from the source code. + """Return the class name from the source code. Args: - source_code (str): The source code to process. + source_code (str): Java source. Returns: - str: The class name. + str: Class identifier. """ class_name = self.frame_query_and_capture_output("(class_declaration name: (identifier) @name)", source_code) return class_name[0].node.text.decode() def get_superclass(self, source_code: str) -> str: - """Get a list of all the superclasses in a class. + """Return the superclass name if present. Args: - source_code (str): The source code to process. + source_code (str): Java source. Returns: - Set[str]: A set of all the superclasses in the class. + str: Superclass identifier or empty string. """ superclass: Captures = self.frame_query_and_capture_output(query="(class_declaration (superclass (type_identifier) @superclass))", code_to_process=source_code) @@ -156,27 +151,27 @@ def get_superclass(self, source_code: str) -> str: return superclass[0].node.text.decode() def get_all_interfaces(self, source_code: str) -> Set[str]: - """Get a set of interfaces implemented by a class. + """Return interfaces implemented by a class. Args: - source_code (str): The source code to process. + source_code (str): Java source. Returns: - Set[str]: A set of all the interfaces implemented by the class. + set[str]: Interface identifiers. """ interfaces = self.frame_query_and_capture_output("(class_declaration (super_interfaces (type_list (type_identifier) @interface)))", code_to_process=source_code) return {interface.node.text.decode() for interface in interfaces} def frame_query_and_capture_output(self, query: str, code_to_process: str) -> Captures: - """Frame a query for the tree-sitter parser. - - Parameters - ---------- - query : str - The query to frame. - code_to_process : str - The code to process. + """Run a query and return captures from the AST. + + Args: + query (str): S-expression query string. + code_to_process (str): Java source. + + Returns: + Captures: Query captures for the AST root. """ framed_query: Query = LANGUAGE.query(query) tree = PARSER.parse(bytes(code_to_process, "utf-8")) @@ -189,9 +184,7 @@ def get_method_name_from_declaration(self, method_name_string: str) -> str: return captures[0].node.text.decode() def get_method_name_from_invocation(self, method_invocation: str) -> str: - """ - Using the tree-sitter query, extract the method name from the method invocation. - """ + """Extract the method name from a method invocation string.""" captures: Captures = self.frame_query_and_capture_output("(method_invocation name: (identifier) @method_name)", method_invocation) return captures[0].node.text.decode() @@ -202,24 +195,17 @@ def get_identifier_from_arbitrary_statement(self, statement: str) -> str: return captures[0].node.text.decode() def safe_ascend(self, node: Node, ascend_count: int) -> Node: - """Safely ascend the tree. If the node does not exist or if it has no parent, raise an error. - - Parameters - ---------- - node : Node - The node to ascend from. - ascend_count : int - The number of times to ascend the tree. - - Returns - ------- - Node - The node at the specified level of the tree. - - Raises - ------ - ValueError - If the node has no parent. + """Ascend parent pointers safely in the AST. + + Args: + node (Node): Starting node. + ascend_count (int): Levels to ascend. + + Returns: + Node: Ancestor node after ascending. + + Raises: + ValueError: If node is None or has no parent. """ if node is None: raise ValueError("Node does not exist.") @@ -231,22 +217,16 @@ def safe_ascend(self, node: Node, ascend_count: int) -> Node: return self.safe_ascend(node.parent, ascend_count - 1) def get_call_targets(self, method_body: str, declared_methods: dict) -> Set[str]: - """Should generate a list of call targets from the method body. - - Uses simple name resolution for finding the call targets. Nothing sophisticated here. Just a simple search - over the AST. - - Parameters - ---------- - method_body : Node - The method body. - declared_methods : dict - A dictionary of all declared methods in the class. - - Returns - ------- - List[str] - A list of call targets (methods). + """Return call targets referenced in a method body. + + Uses simple name resolution over the AST. + + Args: + method_body (str): Method source. + declared_methods (dict): Declared methods in the class. + + Returns: + set[str]: Call target method names. """ select_test_method_query = "(method_invocation name: (identifier) @method)" @@ -266,20 +246,14 @@ def get_call_targets(self, method_body: str, declared_methods: dict) -> Set[str] return call_targets def get_calling_lines(self, source_method_code: str, target_method_name: str) -> List[int]: - """ - Returns a list of line numbers in source method where target method is called. - - Parameters: - ----------- - source_method_code : str - source method code + """Return line numbers where the target method is called in the source method. - target_method_code : str + Args: + source_method_code (str): Source method code. + target_method_name (str): Target method signature or name. Returns: - -------- - List[int] - List of line numbers within in source method code block. + list[int]: Line numbers within the source method. """ if not source_method_code: return [] @@ -297,25 +271,20 @@ def get_calling_lines(self, source_method_code: str, target_method_name: str) -> method_name = c.node.text.decode() if method_name == target_method_name: target_call_lines.append(c.node.start_point[0]) - except: + except Exception: logger.warning(f"Unable to get calling lines for {target_method_name} in {source_method_code}.") return [] return target_call_lines def get_test_methods(self, source_class_code: str) -> Dict[str, str]: - """ - Returns a dictionary of method names and method bodies. + """Return methods annotated with @Test in a class. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + Args: + source_class_code (str): Java class source. Returns: - -------- - Dict[str,str] - Dictionary of method names and method bodies. + dict[str, str]: Map of method name to body. """ query = """ (method_declaration @@ -337,20 +306,14 @@ def get_test_methods(self, source_class_code: str) -> Dict[str, str]: return test_method_dict def get_methods_with_annotations(self, source_class_code: str, annotations: List[str]) -> Dict[str, List[Dict]]: - """ - Returns a dictionary of method names and method bodies. - - Parameters: - ----------- - source_class_code : str - String containing code for a java class. - annotations : List[str] - List of annotation strings. + """Return methods grouped by annotation. + + Args: + source_class_code (str): Java class source. + annotations (list[str]): Annotation names to include. + Returns: - -------- - Dict[str,List[Dict]] - Dictionary with annotations as keys and - a list of dictionaries containing method names and bodies, as values. + dict[str, list[dict]]: Mapping of annotation to list of method info dicts. """ query = """ (method_declaration @@ -377,34 +340,25 @@ def get_methods_with_annotations(self, source_class_code: str, annotations: List return annotation_method_dict def get_all_type_invocations(self, source_code: str) -> Set[str]: - """ - Given the source code, get all the type invocations in the source code. + """Return all type identifiers referenced in the source. - Parameters - ---------- - source_code : str - The source code to process. + Args: + source_code (str): Java source. - Returns - ------- - Set[str] - A set of all the type invocations in the source code. + Returns: + set[str]: Type identifiers. """ type_references: Captures = self.frame_query_and_capture_output("(type_identifier) @type_id", source_code) return {type_id.node.text.decode() for type_id in type_references} def get_method_return_type(self, source_code: str) -> str: - """Get the return type of a method. + """Return the return type of a method. - Parameters - ---------- - source_code : str - The source code to process. + Args: + source_code (str): Java method source. - Returns - ------- - str - The return type of the method. + Returns: + str: Return type identifier. """ type_references: Captures = self.frame_query_and_capture_output("(method_declaration type: ((type_identifier) @type_id))", source_code) @@ -412,19 +366,14 @@ def get_method_return_type(self, source_code: str) -> str: return type_references[0].node.text.decode() def get_lexical_tokens(self, code: str, filter_by_node_type: List[str] | None = None) -> List[str]: - """ - Get the lexical tokens given the code - - Parameters - ---------- - filter_by_node_type: If needed, filter the type of the node - code: Java code + """Return lexical tokens from the code. - Returns - ------- - List[str] - List of lexical tokens + Args: + code (str): Java source code. + filter_by_node_type (list[str] | None): Optional node type filter. + Returns: + list[str]: Collected token strings. """ tree = PARSER.parse(bytes(code, "utf-8")) root_node = tree.root_node @@ -445,18 +394,13 @@ def collect_leaf_token_values(node): return lexical_tokens def remove_all_comments(self, source_code: str) -> str: - """ - Remove all comments from the source code. + """Return source code with all comments removed. - Parameters - ---------- - source_code : str - The source code to process. + Args: + source_code (str): Java source. - Returns - ------- - str - The source code with all comments removed. + Returns: + str: Source with comments removed. """ # Remove any prefix comments/content before the package declaration @@ -483,17 +427,13 @@ def remove_all_comments(self, source_code: str) -> str: return self.make_pruned_code_prettier(pruned_source_code) def make_pruned_code_prettier(self, pruned_code: str) -> str: - """Make the pruned code prettier. + """Prettify the pruned code after comment removal. - Parameters - ---------- - pruned_code : str - The pruned code. + Args: + pruned_code (str): Source after pruning. - Returns - ------- - str - The prettified pruned code. + Returns: + str: Prettified source code. """ # First remove remaining block comments block_comments: Captures = self.frame_query_and_capture_output(query="((block_comment) @comment_block)", code_to_process=pruned_code) diff --git a/cldk/analysis/commons/treesitter/treesitter_python.py b/cldk/analysis/commons/treesitter/treesitter_python.py index a9f401ff..97dc5dc9 100644 --- a/cldk/analysis/commons/treesitter/treesitter_python.py +++ b/cldk/analysis/commons/treesitter/treesitter_python.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -TreesitterPython module +"""Python tree-sitter queries and helpers. + +Utilities to parse Python source code with tree-sitter and extract modules, +classes, functions, methods, imports, and call sites. """ import os @@ -33,21 +35,19 @@ class TreesitterPython: - """ - Tree sitter for Python use cases. - """ + """Tree-sitter helpers for Python use cases.""" def __init__(self) -> None: self.utils: TreeSitterUtils = TreeSitterUtils() def is_parsable(self, code: str) -> bool: - """ - Check if the code is parsable + """Check whether the Python code parses without syntax errors. + Args: - code: source code + code (str): Source code. Returns: - True if the code is parsable, False otherwise + bool: True if parsable, False otherwise. """ def syntax_error(node): @@ -69,27 +69,24 @@ def syntax_error(node): return False def get_raw_ast(self, code: str) -> Tree: - """ - Get the raw AST + """Parse and return the raw AST. + Args: - code: source code + code (str): Source code. Returns: - Tree: the raw AST + Tree: Parsed AST. """ return PARSER.parse(bytes(code, "utf-8")) def get_all_methods(self, module: str) -> List[PyMethod]: - """ - Get all the methods in the specific module. - Parameters - ---------- - module: code body of the module + """Return all methods declared in the module code body. - Returns - ------- - List[PyMethod]: returns all the method details within the module + Args: + module (str): Module source code. + Returns: + list[PyMethod]: Method details within the module. """ methods: List[PyMethod] = [] method_signatures: dict[str, List[int]] = {} @@ -102,28 +99,25 @@ def get_all_methods(self, module: str) -> List[PyMethod]: return methods def get_all_functions(self, module: str) -> List[PyMethod]: - """ - Get all the methods in the specific module. - Parameters - ---------- - module: code body of the module + """Return all top-level functions in the module code body. - Returns - ------- - List[PyMethod]: returns all the method details within the module + Args: + module (str): Module source code. + Returns: + list[PyMethod]: Function details within the module. """ methods: List[PyMethod] = [] functions: List[PyMethod] = [] method_signatures: dict[str, List[int]] = {} # Get the methods declared under class all_class_details: List[PyClass] = self.get_all_classes(module=module) - # Filter all method nodes - all_method_nodes: Captures = self.__get_method_nodes(module=module) + # Filter all method nodes: Captures = self.__get_method_nodes(module=module) for class_details in all_class_details: for method in class_details.methods: method_signatures[method.full_signature] = [method.start_line, method.end_line] methods.extend(class_details.methods) + all_method_nodes: Captures = self.__get_method_nodes(module=module) for method_node in all_method_nodes: method_details = self.__get_function_details(node=method_node.node) if method_details.full_signature not in method_signatures: @@ -135,16 +129,14 @@ def get_all_functions(self, module: str) -> List[PyMethod]: return functions def get_method_details(self, module: str, method_signature: str) -> PyMethod: - """ - Given the code body and the method signature, returns the method details related to that method - Parameters - ---------- - module: code body - method_signature: method signature - - Returns - ------- - PyMethod: Returns the method details related to that method + """Return details for a method signature in the module code body. + + Args: + module (str): Module source. + method_signature (str): Method signature. + + Returns: + PyMethod: Method details if found, else None. """ all_method_details = self.get_all_methods(module=module) for method in all_method_details: @@ -153,15 +145,13 @@ def get_method_details(self, module: str, method_signature: str) -> PyMethod: return None def get_all_imports(self, module: str) -> List[str]: - """ - Given the code body, returns the imports in that module - Parameters - ---------- - module: code body - - Returns - ------- - List[str]: List of imports + """Return all import statements present in the module code body. + + Args: + module (str): Module source. + + Returns: + list[str]: List of import statements. """ import_list = [] captures_from_import: Captures = self.utils.frame_query_and_capture_output(PARSER, LANGUAGE, "(((import_from_statement) @imports))", module) @@ -178,15 +168,13 @@ def get_module_details(self, module: str) -> PyModule: ) def get_all_imports_details(self, module: str) -> List[PyImport]: - """ - Given the code body, returns the imports in that module - Parameters - ---------- - module: code body - - Returns - ------- - List[PyImport]: List of imports + """Return import details for the module code body. + + Args: + module (str): Module source. + + Returns: + list[PyImport]: Import metadata. """ import_list = [] captures_from_import: Captures = self.utils.frame_query_and_capture_output(PARSER, LANGUAGE, "(((import_from_statement) @imports))", module) @@ -213,15 +201,13 @@ def get_all_fields(self, module: str): pass def get_all_classes(self, module: str) -> List[PyClass]: - """ - Given the code body of the module, returns details of all classes in it - Parameters - ---------- - module: code body - - Returns - ------- - List[PyClass]: returns details of all classes in it + """Return details of all classes declared in the module. + + Args: + module (str): Module source code. + + Returns: + list[PyClass]: Class metadata. """ classes: List[PyClass] = [] all_class_details: Captures = self.utils.frame_query_and_capture_output(PARSER, LANGUAGE, "(((class_definition) @class_name))", module) @@ -254,15 +240,13 @@ def get_all_classes(self, module: str) -> List[PyClass]: return classes def get_all_modules(self, application_dir: Path) -> List[PyModule]: - """ - Given an application directory, returns a list of modules - Parameters - ---------- - application_dir (Path): Location of the application directory - - Returns - ------- - List[PyModule]: returns a list of modules + """Return a list of modules under an application directory. + + Args: + application_dir (Path): Application root directory. + + Returns: + list[PyModule]: Modules discovered. """ modules: List[PyModule] = [] path_list = [os.path.join(dirpath, filename) for dirpath, _, filenames in os.walk(application_dir) for filename in filenames if filename.endswith(".py")] @@ -275,25 +259,18 @@ def __get_module(self, path: Path): module_qualified_name = str(module_qualified_path).replace(os.sep, ".") with open(module_qualified_path, "r") as file: py_module = self.get_module_details(module=file.read()) - qualified_name: str - methods: List[PyMethod] - functions: List[PyMethod] - classes: List[PyClass] - imports: List[PyImport] return PyModule(qualified_name=module_qualified_name, imports=py_module.imports, functions=py_module.functions, classes=py_module.classes) return None @staticmethod def __get_call_site_details(call_node: Node) -> PyCallSite: - """ - Get details about the call site information given a call node - Parameters - ---------- - call_node - - Returns - ------- - PyCallSite: Call site information + """Return call site information from a call node. + + Args: + call_node (Node): Call node. + + Returns: + PyCallSite: Call site information. """ start_line = call_node.start_point[0] start_column = call_node.start_point[1] diff --git a/cldk/analysis/commons/treesitter/utils/treesitter_utils.py b/cldk/analysis/commons/treesitter/utils/treesitter_utils.py index bf860cf4..11f5fbc9 100644 --- a/cldk/analysis/commons/treesitter/utils/treesitter_utils.py +++ b/cldk/analysis/commons/treesitter/utils/treesitter_utils.py @@ -14,8 +14,9 @@ # limitations under the License. ################################################################################ -""" -Three Sitter Utils module +"""Tree-sitter utilities. + +Helpers for framing queries and safe AST traversal. """ from tree_sitter import Query, Node @@ -25,38 +26,33 @@ class TreeSitterUtils: def frame_query_and_capture_output(self, parser, language, query: str, code_to_process: str) -> Captures: - """Frame a query for the tree-sitter parser. + """Frame a query for the tree-sitter parser and return captures. + + Args: + parser: A configured tree-sitter Parser instance. + language: The tree-sitter Language for the parser. + query (str): The S-expression query string. + code_to_process (str): Source code to parse. - Parameters - ---------- - query : str - The query to frame. - code_to_process : str - The code to process. + Returns: + Captures: Query captures from the root node. """ framed_query: Query = language.query(query) tree = parser.parse(bytes(code_to_process, "utf-8")) return Captures(framed_query.captures(tree.root_node)) def safe_ascend(self, node: Node, ascend_count: int) -> Node: - """Safely ascend the tree. If the node does not exist or if it has no parent, raise an error. + """Ascend parent pointers safely in the AST. - Parameters - ---------- - node : Node - The node to ascend from. - ascend_count : int - The number of times to ascend the tree. + Args: + node (Node): Starting node. + ascend_count (int): Number of levels to ascend. - Returns - ------- - Node - The node at the specified level of the tree. + Returns: + Node: The ancestor node after ascending. - Raises - ------ - ValueError - If the node has no parent. + Raises: + ValueError: If the node is None or has no parent when ascending. """ if node is None: raise ValueError("Node does not exist.") diff --git a/cldk/analysis/java/java_analysis.py b/cldk/analysis/java/java_analysis.py index de7053fe..e9660b10 100644 --- a/cldk/analysis/java/java_analysis.py +++ b/cldk/analysis/java/java_analysis.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -Java module +"""Java analysis utilities. + +Provides a high-level API to analyze Java projects or single-source inputs +using a Treesitter-based parser and the Code Analyzer backend. """ from pathlib import Path @@ -32,6 +34,12 @@ class JavaAnalysis: + """Analysis façade for Java code. + + This class exposes methods to query symbol tables, classes, methods, + call graphs, comments, and CRUD operations for a Java project or a single + source file, depending on the initialization parameters. + """ def __init__( self, @@ -43,22 +51,25 @@ def __init__( target_files: List[str] | None, eager_analysis: bool, ) -> None: - """Initialization method for Java Analysis backend. + """Initialize the Java analysis backend. Args: - project_dir (str | Path | None): The directory path of the project. - source_code (str | None): Java file for single source file analysis. - analysis_backend_path (str | None): The path to the analysis_backend, defaults to None and in the case of codeql, it is assumed that the cli is installed and available in the PATH. In the case of codeanalyzer the codeanalyzer.jar is downloaded from the lastest release. - analysis_json_path (str | Path | None): The path save the to the analysis database (analysis.json), defaults to None. If None, the analysis database is not persisted. - analysis_level (str): Analysis level (symbol-table, call-graph) - eager_analysis (bool): A flag indicating whether to perform eager analysis, defaults to False. If True, the analysis is performed eagerly. That is, the analysis.json file is created during analysis every time even if it already exists. + project_dir (str | Path | None): Directory path of the project. + source_code (str | None): Source text for single-file analysis. + analysis_backend_path (str | None): Path to the analysis backend. For + CodeQL, the CLI must be installed and on PATH. For CodeAnalyzer, + the JAR is downloaded from the latest release when not provided. + analysis_json_path (str | Path | None): Path to persist the analysis + database (analysis.json). If None, results are not persisted. + analysis_level (str): Analysis level. For example, "symbol-table" or + "call-graph". + target_files (list[str] | None): Optional list of target file paths to + constrain analysis (primarily supported for symbol-table). + eager_analysis (bool): If True, forces regeneration of analysis.json + on each run even if it exists. Raises: - NotImplementedError: Raised when anaysis backend is not supported. - - Attributes: - application (JApplication): The application view of the Java code. - + NotImplementedError: If the requested analysis backend is unsupported. """ self.project_dir = project_dir @@ -81,140 +92,217 @@ def __init__( ) def get_imports(self) -> List[str]: - """Should return all the imports in the source code. - - Raises: - NotImplementedError: Raised when this functionality is not suported. + """Return all import statements in the source code. Returns: - List[str]: List of all the imports. + list[str]: All import statements. + + Raises: + NotImplementedError: If this functionality is not supported. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_variables(self, **kwargs): - """_Returns all the variables. + """Return all variables discovered in the source code. + + Returns: + Any: Implementation-defined variable view. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If this functionality is not supported. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_service_entry_point_classes(self, **kwargs): - """Should return all service entry point classes. + """Return all service entry-point classes. + + Returns: + Any: Implementation-defined list or mapping of service classes. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If this functionality is not supported. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_service_entry_point_methods(self, **kwargs): - """Should return all the service entry point methods. + """Return all service entry-point methods. + + Returns: + Any: Implementation-defined list or mapping of service methods. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If this functionality is not supported. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_application_view(self) -> JApplication: - """Should return application view of the java code. + """Return the application view of the Java code. + + Returns: + JApplication: Application view of the Java code. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If single-file mode is used (unsupported here). - Returns: - JApplication: Application view of the java code. + Examples: + Get an application view using a project directory (backend required): + + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.symbol_table, + ... target_files=None, eager_analysis=False) + >>> isinstance(ja.source_code, type(None)) + True """ if self.source_code: raise NotImplementedError("Support for this functionality has not been implemented yet.") return self.backend.get_application_view() def get_symbol_table(self) -> Dict[str, JCompilationUnit]: - """Should return symbol table. + """Return the symbol table. Returns: - Dict[str, JCompilationUnit]: Symbol table + dict[str, JCompilationUnit]: Symbol table keyed by file path. + + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.symbol_table, + ... target_files=None, eager_analysis=False) + >>> isinstance(ja.get_symbol_table(), dict) # doctest: +SKIP + True """ return self.backend.get_symbol_table() def get_compilation_units(self) -> List[JCompilationUnit]: - """Should return a list of all compilation units in the java code. - - Raises: - NotImplementedError: Raised when this functionality is not supported. + """Return all compilation units in the Java code. Returns: - List[JCompilationUnit]: Compilation units of the Java code. + list[JCompilationUnit]: Compilation units of the Java code. + + Examples: + List all compilation units for a project (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> isinstance(ja.get_compilation_units(), list) # doctest: +SKIP + True """ return self.backend.get_compilation_units() def get_class_hierarchy(self) -> nx.DiGraph: - """Should return class hierarchy of the java code. - - Raises: - NotImplementedError: Raised when this functionality is not suported. + """Return the class hierarchy of the Java code. Returns: - nx.DiGraph: The class hierarchy of the Java code. + networkx.DiGraph: Class hierarchy. + + Raises: + NotImplementedError: Always, as the feature is not implemented yet. """ raise NotImplementedError("Class hierarchy is not implemented yet.") def is_parsable(self, source_code: str) -> bool: - """ - Check if the code is parsable + """Check if the source code is parsable. + Args: - source_code: source code + source_code (str): Source code to parse. Returns: - True if the code is parsable, False otherwise + bool: True if parsable, False otherwise. + + Examples: + >>> src = 'class A { void f(){} }' + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(source_code=src) + >>> ja.is_parsable(src) + True """ return self.treesitter_java.is_parsable(source_code) def get_raw_ast(self, source_code: str) -> Tree: - """ - Get the raw AST + """Parse and return the raw AST. + Args: - code: source code + source_code (str): Source code to parse. Returns: - Tree: the raw AST + Tree: Raw syntax tree. + + Examples: + >>> src = 'class A { void f(){} }' + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(source_code=src) + >>> ast = ja.get_raw_ast(src) + >>> ast.root_node is not None + True """ return self.treesitter_java.get_raw_ast(source_code) def get_call_graph(self) -> nx.DiGraph: - """Should return the call graph of the Java code. + """Return the call graph of the Java code. Returns: - nx.DiGraph: The call graph of the Java code. + networkx.DiGraph: Call graph. + + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.call_graph, + ... target_files=None, eager_analysis=False) + >>> isinstance(ja.get_call_graph(), nx.DiGraph) # doctest: +SKIP + True """ return self.backend.get_call_graph() def get_call_graph_json(self) -> str: - """Should return a serialized call graph in json. + """Return the call graph serialized as JSON. + + Returns: + str: Call graph encoded as JSON. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If single-file mode is used (unsupported here). - Returns: - str: Call graph in json. + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.call_graph, + ... target_files=None, eager_analysis=False) + >>> isinstance(ja.get_call_graph_json(), str) # doctest: +SKIP + True """ if self.source_code: raise NotImplementedError("Producing a call graph over a single file is not implemented yet.") return self.backend.get_call_graph_json() def get_callers(self, target_class_name: str, target_method_declaration: str, using_symbol_table: bool = False) -> Dict: - """Should return a dictionary of callers of the target method. + """Return all callers of a target method. Args: - target_class_name (str): Qualified target class name. - target_method_declaration (str): Target method names - using_symbol_table (bool, optional): Whether to use symbol table. Defaults to False. + target_class_name (str): Qualified name of the target class. + target_method_declaration (str): Target method signature. + using_symbol_table (bool): Whether to use the symbol table. Defaults to False. + + Returns: + dict: Mapping of callers to call sites/details. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If single-file mode is used (unsupported here). - Returns: - Dict: A dictionary of callers of target method. + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.call_graph, + ... target_files=None, eager_analysis=False) + >>> callers = ja.get_callers('com.example.A', 'f()') # doctest: +SKIP + >>> isinstance(callers, dict) # doctest: +SKIP + True """ if self.source_code: @@ -222,57 +310,89 @@ def get_callers(self, target_class_name: str, target_method_declaration: str, us return self.backend.get_all_callers(target_class_name, target_method_declaration, using_symbol_table) def get_callees(self, source_class_name: str, source_method_declaration: str, using_symbol_table: bool = False) -> Dict: - """Should return a dictionary of callees by the given method in the given class. + """Return all callees of a given method. Args: - source_class_name (str): Qualified class name where the given method is. - source_method_declaration (str): Given method - using_symbol_table (bool): Whether to use symbol table. Defaults to false. + source_class_name (str): Qualified class name containing the method. + source_method_declaration (str): Method signature. + using_symbol_table (bool): Whether to use the symbol table. Defaults to False. + + Returns: + dict: Mapping of callees to call sites/details. Raises: - NotImplementedError: Raised when this functionality is not suported. + NotImplementedError: If single-file mode is used (unsupported here). - Returns: - Dict: Dictionary with callee details. + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.call_graph, + ... target_files=None, eager_analysis=False) + >>> callees = ja.get_callees('com.example.A', 'f()') # doctest: +SKIP + >>> isinstance(callees, dict) # doctest: +SKIP + True """ if self.source_code: raise NotImplementedError("Generating all callees over a single file is not implemented yet.") return self.backend.get_all_callees(source_class_name, source_method_declaration, using_symbol_table) def get_methods(self) -> Dict[str, Dict[str, JCallable]]: - """Should return all methods in the Java code. - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return all methods in the Java code. Returns: - Dict[str, Dict[str, JCallable]]: Dictionary of dictionaries of all methods in the Java code with qualified class name as key and dictionary of methods in that class. + dict[str, dict[str, JCallable]]: Methods grouped by qualified class name. + + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.symbol_table, + ... target_files=None, eager_analysis=False) + >>> methods = ja.get_methods() # doctest: +SKIP + >>> isinstance(methods, dict) # doctest: +SKIP + True """ return self.backend.get_all_methods_in_application() def get_classes(self) -> Dict[str, JType]: - """Should return all classes in the Java code. - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return all classes in the Java code. Returns: - Dict[str, JType]: A dictionary of all classes in the Java code, with qualified class names as keys. + dict[str, JType]: Classes keyed by qualified class name. + + Examples: + >>> from cldk.analysis import AnalysisLevel + >>> ja = JavaAnalysis(project_dir='path/to/project', source_code=None, + ... analysis_backend_path=None, analysis_json_path=None, + ... analysis_level=AnalysisLevel.symbol_table, + ... target_files=None, eager_analysis=False) + >>> classes = ja.get_classes() # doctest: +SKIP + >>> isinstance(classes, dict) # doctest: +SKIP + True """ return self.backend.get_all_classes() def get_classes_by_criteria(self, inclusions=None, exclusions=None) -> Dict[str, JType]: - """Should return a dictionary of all classes with the given criteria, in the Java code. + """Return classes filtered by simple inclusion/exclusion criteria. Args: - inclusions (List, optional): inlusion criteria for the classes. Defaults to None. - exclusions (List, optional): exclusion criteria for the classes. Defaults to None. - - Raises: - NotImplementedError: Raised when we do not support this function. + inclusions (list[str] | None): If provided, only classes whose name + contains any of these substrings are included. + exclusions (list[str] | None): If provided, classes whose name contains + any of these substrings are excluded. Returns: - Dict[str, JType]: A dict of all classes in the Java code, with qualified class names as keys + dict[str, JType]: Matching classes keyed by qualified class name. + + Examples: + Filter classes using simple contains checks (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> filtered = ja.get_classes_by_criteria(inclusions=['Service'], exclusions=['Test']) # doctest: +SKIP + >>> isinstance(filtered, dict) # doctest: +SKIP + True """ if exclusions is None: exclusions = [] @@ -294,377 +414,530 @@ def get_classes_by_criteria(self, inclusions=None, exclusions=None) -> Dict[str, return class_dict def get_class(self, qualified_class_name: str) -> JType: - """Should return a class object given qualified class name. + """Return a class object. Args: - qualified_class_name (str): The qualified name of the class. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - JType: Class object for the given qualified class name. + JType: Class object for the given class. + + Examples: + Look up a class by its qualified name (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> c = ja.get_class('com.example.A') # doctest: +SKIP + >>> c is not None # doctest: +SKIP + True """ return self.backend.get_class(qualified_class_name) def get_method(self, qualified_class_name: str, qualified_method_name: str) -> JCallable: - """Should return a method object given qualified class and method names. + """Return a method object. Args: - qualified_class_name (str): The qualified name of the class. - qualified_method_name (str): The qualified name of the method. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. + qualified_method_name (str): Qualified method name. Returns: - JCallable: A method for the given qualified method name. + JCallable: Method object. + + Examples: + Look up a method by its qualified signature (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> m = ja.get_method('com.example.A', 'f()') # doctest: +SKIP + >>> m is not None # doctest: +SKIP + True """ return self.backend.get_method(qualified_class_name, qualified_method_name) def get_method_parameters(self, qualified_class_name: str, qualified_method_name: str) -> List[str]: - """Should return a list of method parameters given qualified class and method names. + """Return the parameter types/names for a method. Args: - qualified_class_name (str): The qualified name of the class. - qualified_method_name (str): The qualified name of the method. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. + qualified_method_name (str): Qualified method name. Returns: - JCallable: A method for the given qualified method name. + list[str]: Method parameters. + + Examples: + List parameters for a method (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> params = ja.get_method_parameters('com.example.A', 'g(int, java.lang.String)') # doctest: +SKIP + >>> isinstance(params, list) # doctest: +SKIP + True """ return self.backend.get_method_parameters(qualified_class_name, qualified_method_name) def get_java_file(self, qualified_class_name: str) -> str: - """Should return a class given qualified class name. + """Return the Java file path containing a class. Args: - qualified_class_name (str): The qualified name of the class. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - str: Java file name containing the given qualified class. + str: Path to the Java file. + + Examples: + Get the source file path for a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> path = ja.get_java_file('com.example.A') # doctest: +SKIP + >>> isinstance(path, str) # doctest: +SKIP + True """ return self.backend.get_java_file(qualified_class_name) def get_java_compilation_unit(self, file_path: str) -> JCompilationUnit: - """Given the path of a Java source file, returns the compilation unit object from the symbol table. + """Return the compilation unit for a Java source file. Args: - file_path (str): Absolute path to Java source file - - Raises: - NotImplementedError: Raised when we do not support this function. + file_path (str): Absolute path to a Java source file. Returns: - JCompilationUnit: Compilation unit object for Java source file + JCompilationUnit: Compilation unit object. + + Examples: + Parse a Java file into a compilation unit (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> cu = ja.get_java_compilation_unit('/abs/path/to/A.java') # doctest: +SKIP + >>> cu is not None # doctest: +SKIP + True """ return self.backend.get_java_compilation_unit(file_path) def get_methods_in_class(self, qualified_class_name) -> Dict[str, JCallable]: - """Should return a dictionary of all methods of the given class. + """Return all methods of a class. Args: - qualified_class_name (str): qualified class name - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - Dict[str, JCallable]: A dictionary of all constructors of the given class. + dict[str, JCallable]: Methods keyed by signature. + + Examples: + List methods declared in a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> methods = ja.get_methods_in_class('com.example.A') # doctest: +SKIP + >>> isinstance(methods, dict) # doctest: +SKIP + True """ return self.backend.get_all_methods_in_class(qualified_class_name) def get_constructors(self, qualified_class_name) -> Dict[str, JCallable]: - """Should return a dictionary of all constructors of the given class. + """Return all constructors of a class. Args: - qualified_class_name (str): qualified class name - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - Dict[str, JCallable]: A dictionary of all constructors of the given class. + dict[str, JCallable]: Constructors keyed by signature. + + Examples: + List constructors declared in a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> ctors = ja.get_constructors('com.example.A') # doctest: +SKIP + >>> isinstance(ctors, dict) # doctest: +SKIP + True """ return self.backend.get_all_constructors(qualified_class_name) def get_fields(self, qualified_class_name) -> List[JField]: - """Should return a dictionary of all fields of the given class + """Return all fields of a class. Args: - qualified_class_name (str): qualified class name - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - List[JField]: A list of all fields of the given class. + list[JField]: All fields of the class. + + Examples: + List fields declared in a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> fields = ja.get_fields('com.example.A') # doctest: +SKIP + >>> isinstance(fields, list) # doctest: +SKIP + True """ return self.backend.get_all_fields(qualified_class_name) def get_nested_classes(self, qualified_class_name) -> List[JType]: - """Should return a dictionary of all nested classes of the given class + """Return all nested classes of a class. Args: - qualified_class_name (str): qualified class name - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - List[JType]: A list of nested classes for the given class. + list[JType]: Nested classes. + + Examples: + List nested classes declared in a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> nested = ja.get_nested_classes('com.example.A') # doctest: +SKIP + >>> isinstance(nested, list) # doctest: +SKIP + True """ return self.backend.get_all_nested_classes(qualified_class_name) def get_sub_classes(self, qualified_class_name) -> Dict[str, JType]: - """Should return a dictionary of all sub-classes of the given class + """Return all subclasses of a class. Args: - qualified_class_name (str): qualified class name + qualified_class_name (str): Qualified class name. Returns: - Dict[str, JType]: A dictionary of all sub-classes of the given class, and class details + dict[str, JType]: Subclasses keyed by qualified name. + + Examples: + List subclasses of a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> subs = ja.get_sub_classes('com.example.A') # doctest: +SKIP + >>> isinstance(subs, dict) # doctest: +SKIP + True """ return self.backend.get_all_sub_classes(qualified_class_name=qualified_class_name) def get_extended_classes(self, qualified_class_name) -> List[str]: - """Should return a list of all extended classes for the given class. - Args: - qualified_class_name (str): The qualified name of the class. + """Return all extended superclasses for a class. - Raises: - NotImplementedError: Raised when we do not support this function. + Args: + qualified_class_name (str): Qualified class name. Returns: - List[str]: A list of extended classes for the given class. + list[str]: Extended classes. + + Examples: + List extended classes for a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> supers = ja.get_extended_classes('com.example.A') # doctest: +SKIP + >>> isinstance(supers, list) # doctest: +SKIP + True """ return self.backend.get_extended_classes(qualified_class_name) def get_implemented_interfaces(self, qualified_class_name: str) -> List[str]: - """Should return a list of all implemented interfaces for the given class. + """Return all implemented interfaces for a class. Args: - qualified_class_name (str): The qualified name of the class. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. Returns: - List[str]: A list of implemented interfaces for the given class. + list[str]: Implemented interfaces. + + Examples: + List implemented interfaces for a class (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> ifaces = ja.get_implemented_interfaces('com.example.A') # doctest: +SKIP + >>> isinstance(ifaces, list) # doctest: +SKIP + True """ return self.backend.get_implemented_interfaces(qualified_class_name) def __get_class_call_graph_using_symbol_table(self, qualified_class_name: str, method_signature: str | None = None) -> (List)[Tuple[JMethodDetail, JMethodDetail]]: - """A call graph using symbol table for a given class and a given method. + """Return class-level call graph using the symbol table. Args: - qualified_class_name (str): The qualified name of the class. - method_signature (str | None, optional): The signature of the method in the class.. Defaults to None. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. + method_signature (str | None): Optional method signature to scope the graph. Returns: - List[Tuple[JMethodDetail, JMethodDetail]]: An edge list of the call graph for the given class and method. + list[tuple[JMethodDetail, JMethodDetail]]: Edge list of caller -> callee. """ return self.backend.get_class_call_graph_using_symbol_table(qualified_class_name, method_signature) def get_class_call_graph(self, qualified_class_name: str, method_signature: str | None = None, using_symbol_table: bool = False) -> List[Tuple[JMethodDetail, JMethodDetail]]: - """A call graph for a given class and (optionally) a given method. + """Return a class-level call graph. Args: - qualified_class_name (str): The qualified name of the class. - method_signature (str | None, optional): The signature of the method in the class.. Defaults to None. - using_symbol_table (bool, optional): Generate call graph using symbol table. Defaults to False. - - Raises: - NotImplementedError: Raised when we do not support this function. + qualified_class_name (str): Qualified class name. + method_signature (str | None): Optional method signature to scope the graph. + using_symbol_table (bool): If True, use the symbol table for resolution. Returns: - List[Tuple[JMethodDetail, JMethodDetail]]: An edge list of the call graph for the given class and method. + list[tuple[JMethodDetail, JMethodDetail]]: Edge list of caller -> callee. + + Examples: + Build a class-level call graph (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project', analysis_level='call-graph') + >>> edges = ja.get_class_call_graph('com.example.A') # doctest: +SKIP + >>> isinstance(edges, list) # doctest: +SKIP + True """ if using_symbol_table: return self.__get_class_call_graph_using_symbol_table(qualified_class_name=qualified_class_name, method_signature=method_signature) return self.backend.get_class_call_graph(qualified_class_name, method_signature) def get_entry_point_classes(self) -> Dict[str, JType]: - """Should return a dictionary of all entry point classes in the Java code. - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return all entry-point classes. Returns: - Dict[str, JType]: A dict of all entry point classes in the Java code, with qualified class names as keys + dict[str, JType]: Entry-point classes keyed by qualified class name. + + Examples: + List entry-point classes in an application (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> eps = ja.get_entry_point_classes() # doctest: +SKIP + >>> isinstance(eps, dict) # doctest: +SKIP + True """ return self.backend.get_all_entry_point_classes() def get_entry_point_methods(self) -> Dict[str, Dict[str, JCallable]]: - """Should return a dictionary of all entry point methods in the Java code with qualified class name as key and dictionary of methods in that class as value - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return all entry-point methods grouped by class. Returns: - Dict[str, Dict[str, JCallable]]: A dictionary of dictionaries of entry point methods in the Java code. + dict[str, dict[str, JCallable]]: Entry-point methods keyed by class. + + Examples: + List entry-point methods in an application (backend required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> epms = ja.get_entry_point_methods() # doctest: +SKIP + >>> isinstance(epms, dict) # doctest: +SKIP + True """ return self.backend.get_all_entry_point_methods() def remove_all_comments(self) -> str: """Remove all comments from the source code. - Raises: - NotImplementedError: Raised when we do not support this function. - Returns: - str: The source code with all comments removed. + str: Source code with comments removed. + + Examples: + Remove comments from inline source code: + + >>> from cldk import CLDK + >>> src = 'class A { /* c */ // d\n void f(){} }'.replace('\n', ' ') + >>> ja = CLDK(language="java").analysis(source_code=src) + >>> cleaned = ja.remove_all_comments() # doctest: +SKIP + >>> '/*' in cleaned or '//' in cleaned # doctest: +SKIP + False """ return self.backend.remove_all_comments(self.source_code) def get_methods_with_annotations(self, annotations: List[str]) -> Dict[str, List[Dict]]: - """Should return a dictionary of method names and method bodies. + """Return methods grouped by the given annotations. Args: - annotations (List[str]): List of annotation strings. - - Raises: - NotImplementedError: Raised when we do not support this function. + annotations (list[str]): Annotations to search for. Returns: - Dict[str, List[Dict]]: Dictionary with annotations as keys and a list of dictionaries containing method names and bodies, as values. + dict[str, list[dict]]: Methods and bodies keyed by annotation. + + Raises: + NotImplementedError: This functionality is not implemented yet. """ # TODO: This call is missing some implementation. The logic currently resides in java_sitter but tree_sitter will no longer be option, rather it will be default and common. Need to implement this differently. Somthing like, self.commons.treesitter.get_methods_with_annotations(annotations) raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_test_methods(self) -> Dict[str, str]: - """Should return a dictionary of method names and method bodies - - Raises: - NotImplementedError: Raised when we do not support this function. + """Return test methods discovered in the source code. Returns: - Dict[str, str]: Dictionary of method names and method bodies. + dict[str, str]: Mapping of method signature to body. + + Examples: + Extract test methods from inline source code: + + >>> from cldk import CLDK + >>> src = 'import org.junit.Test; class A { @Test public void t(){} }' + >>> ja = CLDK(language="java").analysis(source_code=src) + >>> tests = ja.get_test_methods() # doctest: +SKIP + >>> isinstance(tests, dict) # doctest: +SKIP + True """ return self.treesitter_java.get_test_methods(source_class_code=self.source_code) def get_calling_lines(self, target_method_name: str) -> List[int]: - """Should return a list of line numbers in source method block where target method is called. + """Return line numbers where a target method is called within a method body. Args: - target_method_name (str): target method name. - - Raises: - NotImplementedError: Raised when we do not support this function. + target_method_name (str): Target method name. Returns: - List[int]: List of line numbers within in source method code block. + list[int]: Line numbers within the source method code block. + + Raises: + NotImplementedError: This functionality is not implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_call_targets(self, declared_methods: dict) -> Set[str]: - """Uses simple name resolution for finding the call targets. Nothing sophiscticed here. Just a simple search over the AST. + """Return call targets using simple name resolution over the AST. Args: - declared_methods (dict): A dictionary of all declared methods in the class. + declared_methods (dict): All methods declared in the class. Returns: - Set[str]: A list of call targets (methods). + set[str]: Discovered call targets. + + Raises: + NotImplementedError: This functionality is not implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_all_crud_operations(self) -> List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: - """Should return a dictionary of all CRUD operations in the source code. + """Return all CRUD operations in the source code. Returns: - List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: A list of all CRUD operations in the source code. + list[dict[str, Union[JType, JCallable, list[JCRUDOperation]]]]: CRUD operations grouped by class/method. + + Examples: + Get all CRUD operations discovered by the backend (project required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> ops = ja.get_all_crud_operations() # doctest: +SKIP + >>> isinstance(ops, list) # doctest: +SKIP + True """ return self.backend.get_all_crud_operations() def get_all_create_operations(self) -> List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: - """Should return a list of all create operations in the source code. + """Return all create operations in the source code. Returns: - List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: A list of all create operations in the source code. + list[dict[str, Union[JType, JCallable, list[JCRUDOperation]]]]: Create operations. + + Examples: + Get create operations (project required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> creates = ja.get_all_create_operations() # doctest: +SKIP + >>> isinstance(creates, list) # doctest: +SKIP + True """ return self.backend.get_all_create_operations() def get_all_read_operations(self) -> List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: - """Should return a list of all read operations in the source code. + """Return all read operations in the source code. Returns: - List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: A list of all read operations in the source code. + list[dict[str, Union[JType, JCallable, list[JCRUDOperation]]]]: Read operations. + + Examples: + Get read operations (project required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> reads = ja.get_all_read_operations() # doctest: +SKIP + >>> isinstance(reads, list) # doctest: +SKIP + True """ return self.backend.get_all_read_operations() def get_all_update_operations(self) -> List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: - """Should return a list of all update operations in the source code. + """Return all update operations in the source code. Returns: - List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: A list of all update operations in the source code. + list[dict[str, Union[JType, JCallable, list[JCRUDOperation]]]]: Update operations. + + Examples: + Get update operations (project required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> updates = ja.get_all_update_operations() # doctest: +SKIP + >>> isinstance(updates, list) # doctest: +SKIP + True """ return self.backend.get_all_update_operations() def get_all_delete_operations(self) -> List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: - """Should return a list of all delete operations in the source code. + """Return all delete operations in the source code. Returns: - List[Dict[str, Union[JType, JCallable, List[JCRUDOperation]]]]: A list of all delete operations in the source code. + list[dict[str, Union[JType, JCallable, list[JCRUDOperation]]]]: Delete operations. + + Examples: + Get delete operations (project required): + + >>> from cldk import CLDK + >>> ja = CLDK(language="java").analysis(project_path='path/to/project') + >>> deletes = ja.get_all_delete_operations() # doctest: +SKIP + >>> isinstance(deletes, list) # doctest: +SKIP + True """ return self.backend.get_all_delete_operations() # Some APIs to process comments def get_comments_in_a_method(self, qualified_class_name: str, method_signature: str) -> List[JComment]: - """Get all comments in a method. + """Return all comments in a method. Args: - qualified_class_name (str): Qualified name of the class. - method_signature (str): Signature of the method. + qualified_class_name (str): Qualified class name. + method_signature (str): Method signature. Returns: - List[str]: List of comments in the method. + list[JComment]: Comments in the method. """ return self.backend.get_comments_in_a_method(qualified_class_name, method_signature) def get_comments_in_a_class(self, qualified_class_name: str) -> List[JComment]: - """Get all comments in a class. + """Return all comments in a class. Args: - qualified_class_name (str): Qualified name of the class. + qualified_class_name (str): Qualified class name. Returns: - List[str]: List of comments in the class. + list[JComment]: Comments in the class. """ return self.backend.get_comments_in_a_class(qualified_class_name) def get_comment_in_file(self, file_path: str) -> List[JComment]: - """Get all comments in a file. + """Return all comments in a file. Args: - file_path (str): Path to the file. + file_path (str): Absolute file path. Returns: - List[str]: List of comments in the file. + list[JComment]: Comments in the file. """ return self.backend.get_comment_in_file(file_path) def get_all_comments(self) -> Dict[str, List[JComment]]: - """Get all comments in the Java application. + """Return all comments grouped by file. Returns: - Dict[str, List[str]]: Dictionary of file paths and their corresponding comments. + dict[str, list[JComment]]: Mapping of file path to comments. """ return self.backend.get_all_comments() def get_all_docstrings(self) -> Dict[str, List[JComment]]: - """Get all docstrings in the Java application. + """Return all docstrings grouped by file. Returns: - Dict[str, List[str]]: Dictionary of file paths and their corresponding docstrings. + dict[str, list[JComment]]: Mapping of file path to docstrings. """ return self.backend.get_all_docstrings() diff --git a/cldk/analysis/python/python_analysis.py b/cldk/analysis/python/python_analysis.py index de248ad0..2a5a2398 100644 --- a/cldk/analysis/python/python_analysis.py +++ b/cldk/analysis/python/python_analysis.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -Python module +"""Python analysis utilities. + +Provides a high-level API to query modules, classes, functions, and methods +from Python projects or single-source inputs using Treesitter. """ from pathlib import Path @@ -26,7 +28,12 @@ class PythonAnalysis: - """Python Analysis Class""" + """Analysis façade for Python code. + + Args: + project_dir (str | Path | None): Directory path of the project. + source_code (str | None): Source text for single-file analysis. + """ def __init__( self, @@ -38,108 +45,224 @@ def __init__( self.analysis_backend: TreesitterPython = TreesitterPython() def get_methods(self) -> List[PyMethod]: - """ - Given an application or a source code, get all the methods + """Return all methods. + + Returns: + list[PyMethod]: Methods discovered in the source code. + + Examples: + >>> src = 'class C: def f(self): pass def g(self): pass' + >>> pa = PythonAnalysis(project_dir=None, source_code=src) # doctest: +SKIP + >>> len(pa.get_methods()) # doctest: +SKIP + 2 """ return self.analysis_backend.get_all_methods(self.source_code) def get_functions(self) -> List[PyMethod]: - """ - Given an application or a source code, get all the methods + """Return all functions. + + Returns: + list[PyMethod]: Functions discovered in the source code. + + Examples: + >>> src = 'def f(): return 1' + >>> pa = PythonAnalysis(project_dir=None, source_code=src) + >>> [m.full_signature for m in pa.get_functions()] + ['f()'] """ return self.analysis_backend.get_all_functions(self.source_code) def get_modules(self) -> List[PyModule]: - """ - Given the project directory, get all the modules + """Return all modules in the project directory. + + Returns: + list[PyModule]: Modules discovered under project_dir. + + Examples: + Create a temporary project and discover modules: + + >>> import os, tempfile + >>> d = tempfile.mkdtemp() + >>> _ = open(os.path.join(d, 'a.py'), 'w').write('print(1)') + >>> _ = open(os.path.join(d, 'b.py'), 'w').write('print(2)') + >>> pa = PythonAnalysis(project_dir=d, source_code=None) + >>> len(pa.get_modules()) >= 2 + True """ return self.analysis_backend.get_all_modules(self.project_dir) def get_method_details(self, method_signature: str) -> PyMethod: - """ - Given the code body and the method signature, returns the method details related to that method - Parameters - ---------- - method_signature: method signature - - Returns - ------- - PyMethod: Returns the method details related to that method + """Return details for a given method signature. + + Args: + method_signature (str): Method signature to look up. + + Returns: + PyMethod: Method details. + + Examples: + >>> src = 'class C: def add(self, a, b): return a+b' + >>> pa = PythonAnalysis(project_dir=None, source_code=src) # doctest: +SKIP + >>> pa.get_method_details('add(self, a, b)').full_signature # doctest: +SKIP + 'add(self, a, b)' """ return self.analysis_backend.get_method_details(self.source_code, method_signature) def is_parsable(self, source_code: str) -> bool: - """ - Check if the code is parsable + """Check if the source code is parsable. + Args: - source_code: source code + source_code (str): Source code to parse. Returns: - True if the code is parsable, False otherwise + bool: True if parsable, False otherwise. + + Examples: + >>> PythonAnalysis(None, None).is_parsable('def f(): pass') + True + >>> PythonAnalysis(None, None).is_parsable('def f(): pass if') + False """ return TreesitterPython().is_parsable(source_code) def get_raw_ast(self, source_code: str) -> str: - """ - Get the raw AST + """Parse and return the raw AST. + Args: - code: source code + source_code (str): Source code to parse. Returns: - Tree: the raw AST + str: Raw AST representation. + + Examples: + >>> ast = PythonAnalysis(None, None).get_raw_ast('def f(): pass') + >>> isinstance(ast, str) + True """ return TreesitterPython().get_raw_ast(source_code) def get_imports(self) -> List[PyImport]: - """ - Given an application or a source code, get all the imports + """Return all import statements. + + Returns: + list[PyImport]: Imports discovered in the source code. + + Examples: + >>> src = 'import os; from math import sqrt; from x import *' + >>> pa = PythonAnalysis(project_dir=None, source_code=src) + >>> len(pa.get_imports()) + 3 """ return self.analysis_backend.get_all_imports_details(self.source_code) def get_variables(self, **kwargs): - """ - Given an application or a source code, get all the variables + """Return all variables discovered in the source code. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='x=1') + >>> pa.get_variables() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_classes(self) -> List[PyClass]: - """ - Given an application or a source code, get all the classes + """Return all classes. + + Returns: + list[PyClass]: Classes discovered in the source code. + + Examples: + >>> src = 'class A: pass' + >>> pa = PythonAnalysis(project_dir=None, source_code=src) + >>> [c.class_name for c in pa.get_classes()] + ['A'] """ return self.analysis_backend.get_all_classes(self.source_code) def get_classes_by_criteria(self, **kwargs): - """ - Given an application or a source code, get all the classes given the inclusion and exclution criteria + """Return classes filtered by inclusion/exclusion criteria. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: pass') + >>> pa.get_classes_by_criteria() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_sub_classes(self, **kwargs): - """ - Given an application or a source code, get all the sub-classes + """Return all subclasses. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: pass') + >>> pa.get_sub_classes() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_nested_classes(self, **kwargs): - """ - Given an application or a source code, get all the nested classes + """Return all nested classes. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: class B: pass') + >>> pa.get_nested_classes() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_constructors(self, **kwargs): - """ - Given an application or a source code, get all the constructors + """Return all constructors. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: def __init__(self): pass') + >>> pa.get_constructors() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_methods_in_class(self, **kwargs): - """ - Given an application or a source code, get all the methods within the given class + """Return all methods within a given class. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: def f(self): pass') + >>> pa.get_methods_in_class() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") def get_fields(self, **kwargs): - """ - Given an application or a source code, get all the fields + """Return all fields. + + Raises: + NotImplementedError: This functionality is not implemented yet. + + Examples: + >>> pa = PythonAnalysis(project_dir=None, source_code='class A: x=1') + >>> pa.get_fields() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + NotImplementedError: Support for this functionality has not been implemented yet. """ raise NotImplementedError("Support for this functionality has not been implemented yet.") diff --git a/cldk/core.py b/cldk/core.py index 7024d32a..27cf4651 100644 --- a/cldk/core.py +++ b/cldk/core.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -Core module +"""Core CLDK module. + +Provides the top-level CLDK entry point used to initialize language-specific +analysis, Treesitter parsers, and related utilities. """ from pathlib import Path @@ -28,7 +30,6 @@ from cldk.analysis.java import JavaAnalysis from cldk.analysis.commons.treesitter import TreesitterJava from cldk.analysis.python.python_analysis import PythonAnalysis -from cldk.analysis.python.python_analysis import PythonAnalysis from cldk.utils.exceptions import CldkInitializationException from cldk.utils.sanitization.java import TreesitterSanitizer @@ -36,18 +37,16 @@ class CLDK: - """ - The CLDK base class. + """Core class for the Code Language Development Kit (CLDK). - Parameters - ---------- - language : str - The programming language of the project. + Initialize with the desired programming language and use the exposed + helpers to perform language-specific analysis. - Attributes - ---------- - language : str - The programming language of the project. + Args: + language (str): Programming language (e.g., "java", "python", "c"). + + Attributes: + language (str): Programming language of the project. """ def __init__(self, language: str): @@ -62,46 +61,34 @@ def analysis( target_files: List[str] | None = None, analysis_backend_path: str | None = None, analysis_json_path: str | Path = None, - ) -> JavaAnalysis: - """ - Initialize the preprocessor based on the specified language. - - Parameters - ---------- - project_path : str or Path - The directory path of the project. - source_code : str, optional - The source code of the project, defaults to None. If None, it is assumed that the whole project is being - analyzed. - analysis_backend_path : str, optional - The path to the analysis backend, defaults to None where it assumes the default backend path. - analysis_json_path : str or Path, optional - The path save the to the analysis database (analysis.json), defaults to None. If None, the analysis database - is not persisted. - eager : bool, optional - A flag indicating whether to perform eager analysis, defaults to False. If True, the analysis is performed - eagerly. That is, the analysis.json file is created during analysis every time even if it already exists. - analysis_level: str, optional - Analysis levels. Refer to AnalysisLevel. - target_files: List[str] | None, optional - The target files (paths) for which the analysis will run or get modified. Currently, this feature only supported - with symbol table analysis. In the future, we will add this feature to other analysis levels. - Returns - ------- - JavaAnalysis - The initialized JavaAnalysis object. - - Raises - ------ - CldkInitializationException - If neither project_path nor source_code is provided. - NotImplementedError - If the specified language is not implemented yet. + ) -> JavaAnalysis | PythonAnalysis | CAnalysis: + """Initialize a language-specific analysis façade. Args: - analysis_level: - target_files: - analysis_level: + project_path (str | Path | None): Directory path of the project. + source_code (str | None): Source code for single-file analysis. + eager (bool): If True, forces regeneration of analysis databases. + analysis_level (str): Analysis level. See AnalysisLevel. + target_files (list[str] | None): Files to constrain analysis (optional). + analysis_backend_path (str | None): Path to the analysis backend. + analysis_json_path (str | Path | None): Path to persist analysis database. + + Returns: + JavaAnalysis | PythonAnalysis | CAnalysis: Initialized analysis façade for the chosen language. + + Raises: + CldkInitializationException: If both or neither of project_path and source_code are provided. + NotImplementedError: If the specified language is unsupported. + + Examples: + Initialize Python analysis with inline source code and verify type: + + >>> from cldk import CLDK + >>> cldk = CLDK(language="python") + >>> analysis = cldk.analysis(source_code='def f(): return 1') + >>> from cldk.analysis.python import PythonAnalysis + >>> isinstance(analysis, PythonAnalysis) + True """ if project_path is None and source_code is None: @@ -131,14 +118,21 @@ def analysis( raise NotImplementedError(f"Analysis support for {self.language} is not implemented yet.") def treesitter_parser(self): - """ - Parse the project using treesitter. + """Return a Treesitter parser for the selected language. - Returns - ------- - List - A list of treesitter nodes. + Returns: + TreesitterJava: Parser for Java language. + Raises: + NotImplementedError: If the language is unsupported. + + Examples: + Get a Java Treesitter parser: + + >>> from cldk import CLDK + >>> parser = CLDK(language="java").treesitter_parser() + >>> parser.__class__.__name__ + 'TreesitterJava' """ if self.language == "java": return TreesitterJava() @@ -146,20 +140,25 @@ def treesitter_parser(self): raise NotImplementedError(f"Treesitter parser for {self.language} is not implemented yet.") def tree_sitter_utils(self, source_code: str) -> [TreesitterSanitizer | NotImplementedError]: # type: ignore - """ - Parse the project using treesitter. + """Return Treesitter utilities for the selected language. + + Args: + source_code (str): Source code to initialize the utilities with. + + Returns: + TreesitterSanitizer: Utility wrapper for Java Treesitter operations. - Parameters - ---------- - source_code : str, optional - The source code of the project, defaults to None. If None, it is assumed that the whole project is being - analyzed. + Raises: + NotImplementedError: If the language is unsupported. - Returns - ------- - List - A list of treesitter nodes. + Examples: + Create Java Treesitter sanitizer utilities: + >>> from cldk import CLDK + >>> utils = CLDK(language="java").tree_sitter_utils('class A {}') + >>> from cldk.utils.sanitization.java import TreesitterSanitizer + >>> isinstance(utils, TreesitterSanitizer) + True """ if self.language == "java": return TreesitterSanitizer(source_code=source_code) diff --git a/cldk/utils/sanitization/java/treesitter_sanitizer.py b/cldk/utils/sanitization/java/treesitter_sanitizer.py index 06b023c3..47560259 100644 --- a/cldk/utils/sanitization/java/treesitter_sanitizer.py +++ b/cldk/utils/sanitization/java/treesitter_sanitizer.py @@ -14,8 +14,11 @@ # limitations under the License. ################################################################################ -""" -Treesitter Sanitizer module +"""Sanitize Java classes around a focal method using tree-sitter. + +This module provides utilities to prune Java source code by keeping only a +focal test/method and its transitive callees, and by removing unused fields, +imports, and inner classes. """ import logging @@ -29,8 +32,19 @@ class TreesitterSanitizer: + """Sanitize Java source code using tree-sitter queries. + + The sanitizer focuses a class to a given focal method by keeping that + method and its callees, then removes unused fields, imports, and inner + classes. It also strips block comments before processing. + """ def __init__(self, source_code): + """Initialize the sanitizer. + + Args: + source_code (str): The full Java source code to sanitize. + """ self.source_code = source_code self.sanitized_code = deepcopy(self.source_code) self.__javasitter = TreesitterJava() @@ -38,17 +52,18 @@ def __init__(self, source_code): def keep_only_focal_method_and_its_callees(self, focal_method: str) -> str: """Remove all methods except the focal method and its callees. - Parameters - ---------- - focal_method : str - The of the focal method. - source_code : str - The source code to process. - - Returns - ------- - str - The pruned source code. + Args: + focal_method (str): The name of the focal method. + + Returns: + str: The pruned source code. + + Examples: + >>> src = 'class A { void keep(){} void drop(){} }' + >>> tz = TreesitterSanitizer(src) + >>> out = tz.keep_only_focal_method_and_its_callees('keep') + >>> 'drop' in out + False """ method_declaration: Captures = self.__javasitter.frame_query_and_capture_output(query="((method_declaration) " "@method_declaration)", code_to_process=self.sanitized_code) declared_methods = {self.__javasitter.get_method_name_from_declaration(capture.node.text.decode()): capture.node.text.decode() for capture in method_declaration} @@ -58,32 +73,26 @@ def keep_only_focal_method_and_its_callees(self, focal_method: str) -> str: return self.__javasitter.make_pruned_code_prettier(self.sanitized_code) def remove_unused_imports(self, sanitized_code: str) -> str: - """ - Remove all unused imports from the source code. Assuming you have removed all unused fields in a class and - you are given a pruned source code, visit every child and look for all type identifiers and other identifiers - and compare them with every element in the used imports list. - - Parameters - ---------- - source_code : str - The source code to process. - - sanitized_code : str - The source code after having performed certain operations like removing unused methods, etc. - - Returns - ------- - str - The pruned source code with all the unused imports removed. - - Steps - ----- - + To compare, split string by '.' and compare if the last element is in the set of identifiers or type identifiers. - + Create a set called ids_and_typeids to capture all the indentified identifiers and type identifiers in the source code. - + Next, create a set called unused_imports. - + If import ends_with("*"), ignore it since we'll keep all wildcard imports. - + For every other import statements, compare if the last element in the import string is in the set of ids_and_typeids. If not, add it to the unused_imports set. - + Finally, remove all the unused imports from the source code and prettify it. + """Remove imports not referenced in the class body. + + Assumes fields have already been pruned. Scans type identifiers and + other identifiers to decide which imports are used. Wildcard imports + are preserved. + + Args: + sanitized_code (str): Source code after earlier pruning steps. + + Returns: + str: The pruned source code with unused imports removed. + + Notes: + - Compare the last segment of each import against collected + identifiers and type identifiers. + - Keep wildcard imports (ending with '*'). + Examples: + >>> src = 'import java.util.List; class A { }' + >>> TreesitterSanitizer(src).remove_unused_imports(src) + '' """ pruned_source_code: str = deepcopy(sanitized_code) import_declarations: Captures = self.__javasitter.frame_query_and_capture_output(query="((import_declaration) @imports)", code_to_process=self.source_code) @@ -112,7 +121,7 @@ def remove_unused_imports(self, sanitized_code: str) -> str: query="((scoped_identifier) @scoped_identifier)", code_to_process=import_declaration.node.text.decode() ) import_str = import_statement.captures[0].node.text.decode() - if not import_str.split(".")[-1] in ids_and_typeids: + if import_str.split(".")[-1] not in ids_and_typeids: unused_imports.add(import_declaration.node.text.decode()) for unused_import in unused_imports: @@ -121,20 +130,22 @@ def remove_unused_imports(self, sanitized_code: str) -> str: return self.__javasitter.make_pruned_code_prettier(pruned_source_code) def remove_unused_fields(self, sanitized_code: str) -> str: - """ - Take the pruned source code and remove all the unused fields. - - Parameters - ---------- - source_code : str - Source code after having removed all unused methods. - - Implementation details - ---------------------- - + Get all the field declarations in the file -> used_fields - + Get all the identifiers inside every declared method and constructor in the file. - + Then, loop over every used_fields, get the field name, and add it to unused if the identifier doesn't match any known identifier - in the previous step. + """Remove fields not referenced in any method or constructor. + + Args: + sanitized_code (str): Source after removing unused methods. + + Returns: + str: Source with unused fields removed. + + Notes: + - Collect identifiers used in all methods and constructors, then + drop field declarations whose identifiers don't appear. + Examples: + >>> src = 'class A { int x; void f(){ int y = 1; } }' + >>> out = TreesitterSanitizer(src).remove_unused_fields(src) + >>> 'int x;' in out + False """ pruned_source_code: str = deepcopy(sanitized_code) unused_fields: List[Captures.Capture] = list() @@ -176,28 +187,22 @@ def remove_unused_fields(self, sanitized_code: str) -> str: return self.__javasitter.make_pruned_code_prettier(pruned_source_code) def remove_unused_classes(self, sanitized_code: str) -> str: - """ - Remove inner classes that are no longer used. - - Parameters - ---------- - sanitized_code : str - The sanitized code to process. - - Implementation steps - --------------------- - + Make a deep copy of the source code. - + Get the focal class name by getting the name of the outermost class. - + Get a dictionary of the class name and class declarations in the source code. - + Create unused_classes, a dictionary to hold all the classes that are not used. We seed it with all the classes. - + Create a to_process stack to hold the classes to process. Seed it with the focal class. - + Create a processed_so_far set to hold all the processed classes to avoid reprocessing classes repeatedly. - + While to_process is not empty, pop the first class from the to_process stack, add it to processed_so_far set, - remove the inner classes from the current class, and get all the type invocations in the current class. - + Add all the type invocations to the to_process stack iff: - 1. they are not in the processed_so_far set, and 2. They are in the all_classes dictionary (i.e., they are defined in the class). - + Loop until to_process is empty. - + Finally, remove all the unused classes from the source code and prettify it. + """Remove unused inner classes. + + Args: + sanitized_code (str): The sanitized code to process. + + Returns: + str: The pruned source code with unused inner classes removed. + + Notes: + - Start from the outermost class, traverse type invocations, and + keep only reachable inner classes. + Examples: + >>> src = 'class A { class B{} }' + >>> out = TreesitterSanitizer(src).remove_unused_classes(src) + >>> 'class B' in out + False """ focal_class = self.__javasitter.frame_query_and_capture_output(query="(class_declaration name: (identifier) @name)", code_to_process=self.source_code) @@ -205,7 +210,7 @@ def remove_unused_classes(self, sanitized_code: str) -> str: # We use [0] because there may be several nested classes, # we'll consider the outermost class as the focal class. focal_class_name = focal_class[0].node.text.decode() - except: + except Exception: return "" pruned_source_code = deepcopy(sanitized_code) @@ -238,7 +243,7 @@ def remove_unused_classes(self, sanitized_code: str) -> str: # Find all the type_references in the current class. type_references: Set[str] = self.__javasitter.get_all_type_invocations(current_class_without_inner_class) - to_process.update({type_reference for type_reference in type_references if type_reference in all_classes and not type_reference in processed_so_far}) + to_process.update({type_reference for type_reference in type_references if type_reference in all_classes and type_reference not in processed_so_far}) for _, unused_class_body in unused_classes.items(): pruned_source_code = pruned_source_code.replace(unused_class_body, "") @@ -246,35 +251,18 @@ def remove_unused_classes(self, sanitized_code: str) -> str: return self.__javasitter.make_pruned_code_prettier(pruned_source_code) def _unused_methods(self, focal_method: str, declared_methods: Dict) -> Dict: - """ - Parameters - ---------- - focal_method : str - The focal method that acts the starting point. If nothing, every other method execpt this one will be unused - declared_methods : dict - A dictionary of all the declared methods in the focal class - - Returns - ------- - Dict[str, str] - A dictionary of unused methods, where the key is the method name and value is the method body. - - Implementation details - ---------------------- - 1. Create a dictionary to hold all the methods to be processed (call this to_process). - 2. Make a deep copy of the declared methods dict to initialize the unused_methods. The intuition here is that - every method is unused initially, and we'll pop them out as we encounter them in the class. - 3. Start by initializing the to_process queue with the focal method. - 4. There may be recursive or cyclic calls, let's create a set called processed_so_far to hold all the processed - methods so we don't keep cycling between the methods or getting stuck at an infinite loop. - 5. In while loop, dequeue the first element from the to_process dictionary, put it processed_so_far set, and - obtain all the invoked methods in that (dequeued) method. We will assume that returned invoked methods are - only those that are declared in the class. - 6. For each of the invoked methods, and enqueue them in the to_process queue iff it hasn't been seen in - processed_so_far - 7. Loop until to_process is empty. - 6. It is now safe to assume that all the remaining methods in unused_methods dictionary really are unused and - may be removed from the file. + """Compute methods unused given a focal method. + + Args: + focal_method (str): Starting method name; all others are candidates for removal. + declared_methods (dict): Map of method name to its body text. + + Returns: + dict[str, str]: Unused methods with their bodies. + + Notes: + - Traverse the call graph from the focal method using in-class call + targets; anything unreached is unused. """ unused_methods = deepcopy(declared_methods) # A deep copy of unused methods. @@ -306,21 +294,18 @@ def _unused_methods(self, focal_method: str, declared_methods: Dict) -> Dict: return unused_methods def sanitize_focal_class(self, focal_method: str) -> str: - """Remove all methods except the focal method and its callees. + """Produce sanitized source focused on a focal method. - Given the focal method name and the entire source code, the output will be the pruned source code. + Args: + focal_method (str): The focal method declaration text or name. - Parameters - ---------- - focal_method : str - The name of the focal method. - source_code_file : Path - The path to the source code file. + Returns: + str: Pruned source code with only relevant members retained. - Returns - ------- - str - The pruned source code. + Examples: + >>> src = 'class A { void keep(){} void drop(){} }' + >>> TreesitterSanitizer(src).sanitize_focal_class('keep') + 'class A { void keep(){} }' """ focal_method_name = self.__javasitter.get_method_name_from_declaration(focal_method) diff --git a/cldk/utils/sanitization/java/treesitter_utils.py b/cldk/utils/sanitization/java/treesitter_utils.py index 570e1a4e..5bd13891 100644 --- a/cldk/utils/sanitization/java/treesitter_utils.py +++ b/cldk/utils/sanitization/java/treesitter_utils.py @@ -14,8 +14,10 @@ # limitations under the License. ################################################################################ -""" -Treesitter Utils module +"""Java test sanitization helpers using tree-sitter. + +Utilities to analyze Java test classes and methods for deduplication, +merging, and extraction of assertions, fields, and annotations. """ import re @@ -33,29 +35,23 @@ def _replace_in_source( original_test_method_dict: dict, modified_test_method_dict: dict, ): - """ - Returns a modified source using original test methods and modified ones. - - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + """Return source with original methods replaced by modified ones. - original_test_method_dict: dict - Dictionary of original test methods in the java class. - - modified_test_method_dict: dict - Dictionary of modified test methods + Args: + source_class_code (str): Java class source code. + original_test_method_dict (dict): Original test methods by name. + modified_test_method_dict (dict): Modified test methods by name. Returns: - -------- - str - modified source after removing duplicate test methods and merging decomposed ones. - - - Comments - -------- - # It's modifying the source code produced by an LLM. + str: Source after removing duplicates and merging decomposed tests. + + Examples: + >>> src = 'class T { void a(){} void b(){} }' + >>> original = {'a': 'void a(){}'} + >>> modified = {'a': 'void a(){int x=1;}'} + >>> out = _replace_in_source(src, original, modified) + >>> 'int x=1' in out + True """ modified_source = deepcopy(source_class_code) for _, body in original_test_method_dict.items(): @@ -68,15 +64,19 @@ def _replace_in_source( def separate_assertions(source_method_code: str) -> tuple[str, str]: - """ - Separate assertions and non assertions parts + """Split a test method body into assertions and non-assertions. Args: - source_method_code: test method body + source_method_code (str): Test method body. Returns: - tuple[str, str]: - assertions and non assertions parts + tuple[str, str]: A pair of strings: (assertions_block, non_assertions_block). + + Examples: + >>> body = 'int x=1; assertTrue(x>0); return;' + >>> asserts, rest = separate_assertions(body) + >>> 'assertTrue' in asserts and 'assertTrue' not in rest + True """ code_split = source_method_code.splitlines() assert_block = "" @@ -107,18 +107,19 @@ def separate_assertions(source_method_code: str) -> tuple[str, str]: def is_empty_test_class(source_class_code: str) -> bool: - """ - Checks if a test class has no test methods by looking for methods with @Test annotations + """Return True if no @Test methods exist in the class. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + Args: + source_class_code (str): Java class source code. Returns: - -------- - bool - True if no test methods False if there are test methods + bool: True if class has no tests, False otherwise. + + Examples: + >>> is_empty_test_class('class A { @Test void t(){} }') + False + >>> is_empty_test_class('class A { void t(){} }') + True """ test_methods_dict = java_sitter.get_test_methods(source_class_code) print(test_methods_dict) @@ -126,17 +127,18 @@ def is_empty_test_class(source_class_code: str) -> bool: def get_all_field_access(source_class_code: str) -> Dict[str, list[list[int]]]: - """_summary_ + """Return field access positions in a class. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + Args: + source_class_code (str): Java class source. Returns: - -------- - Dict[str, [[int, int], [int, int]]] - Dictionary with field names as keys and a list of starting and ending line, and starting and ending column. + dict[str, list[list[int]]]: Field name to [[start_line, start_col], [end_line, end_col]]. + + Examples: + >>> src = 'class A { int x; void f(){ this.x = 1; } }' + >>> isinstance(get_all_field_access(src), dict) + True """ query = """ (field_access @@ -161,18 +163,18 @@ def get_all_field_access(source_class_code: str) -> Dict[str, list[list[int]]]: def get_all_fields_with_annotations(source_class_code: str) -> Dict[str, Dict]: - """ - Returns a dictionary of field names and field bodies. + """Return fields with annotations and bodies. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + Args: + source_class_code (str): Java class source. Returns: - -------- - Dict[str,Dict] - Dictionary with field names as keys and a dictionary of annotation and body as values. + dict[str, dict]: Field name to {'annotation': str | None, 'body': str}. + + Examples: + >>> src = '@Deprecated class A { @Inject int x; }' + >>> isinstance(get_all_fields_with_annotations(src), dict) + True """ query = """ (field_declaration @@ -202,18 +204,18 @@ def get_all_fields_with_annotations(source_class_code: str) -> Dict[str, Dict]: def get_all_methods_with_test_with_lines(source_class_code: str) -> Dict[str, List[int]]: - """ - Returns a dictionary of method names and method bodies. + """Return @Test methods with their start and end lines. + + Args: + source_class_code (str): Java class source. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. Returns: - -------- - Dict[str,List[int]] - Dictionary with test name as keys and - starting and ending lines as values. + dict[str, list[int]]: Map of test method name to [start_line, end_line]. + + Examples: + >>> src = 'class A { @Test void t(){} }' + >>> get_all_methods_with_test_with_lines(src) + {'t': [0, 0]} # doctest: +ELLIPSIS """ query = """ (method_declaration @@ -238,18 +240,13 @@ def get_all_methods_with_test_with_lines(source_class_code: str) -> Dict[str, Li def _remove_duplicates_empties(test_method_dict: dict) -> tuple[dict[Any, Any], int | Any, int | Any]: - """ - removes all the duplicates in the test methods. + """Remove duplicate and empty test methods. - Parameters: - ----------- - code_block : str - Code block of a test method. + Args: + test_method_dict (dict): Map of test name to body. Returns: - -------- - tuple[dict, int] - Dictionary of remaining methods after dedup and number of methods removed because of dedup. + tuple[dict, int, int]: (deduped_methods, num_duplicates_removed, num_empty_removed). """ methods_kept = [] num_dup_methods_removed = 0 @@ -277,18 +274,13 @@ def _remove_duplicates_empties(test_method_dict: dict) -> tuple[dict[Any, Any], def _compose_decomposed(self, test_method_dict: dict) -> tuple[dict, int]: - """ - merges all the test methods that only have assertions as different and rest of the code same. + """Merge methods that differ only by assertions. - Parameters: - ----------- - code_block : str - Code block of a test method. + Args: + test_method_dict (dict): Map of test name to body. Returns: - -------- - tuple[dict, int] - Dictionary of merged methods and number of methods removed because of merging. + tuple[dict, int]: (merged_methods, num_merged). """ composed_test_method_dict = {} num_merged_methods = 0 @@ -312,18 +304,13 @@ def _compose_decomposed(self, test_method_dict: dict) -> tuple[dict, int]: def _separate_assertions(code_block: str) -> tuple[str, str]: - """ - separate assertions from the code block of a test method. + """Separate assertions from the rest of the code block. - Parameters: - ----------- - code_block : str - Code block of a test method. + Args: + code_block (str): Test method body (without surrounding braces). Returns: - -------- - tuple[str,str] - assertions and code block without assertions. + tuple[str, str]: (assert_block, code_block_without_assertions). """ code_block_lines = code_block.split(";") # remove new lines and tabs, but not spaces within lines (need to keep them for assertions) @@ -350,18 +337,13 @@ def _separate_assertions(code_block: str) -> tuple[str, str]: def dedup_and_merge(self, source_class_code: str) -> tuple[LiteralString | Any, Any, Any, int]: - """ - Returns a modified source after removing duplicate test methods and merging decomposed ones. + """Return source after deduping and merging test methods. - Parameters: - ----------- - source_class_code : str - String containing code for a java class. + Args: + source_class_code (str): Java class source. Returns: - -------- - str - modified source after removing duplicate test methods and merging decomposed ones. + tuple[str, int, int, int]: (modified_source, num_dups_removed, num_empty_removed, num_merged). """ test_method_dict = java_sitter.get_test_methods(source_class_code) diff --git a/pyproject.toml b/pyproject.toml index 0856a39a..aad47650 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cldk" -version = "1.0.6" +version = "1.0.7" description = "The official Python SDK for Codellm-Devkit." authors = ["Rahul Krishna ", "Rangeet Pan ", "Saurabh Sinhas ", "Raju Pavuluri "] @@ -27,7 +27,6 @@ include = [ "cldk/analysis/java/codeanalyzer/jar/*.jar" ] - [tool.backend-versions] codeanalyzer-java = "2.3.3"