diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md deleted file mode 100644 index fff2114..0000000 --- a/DEVELOPMENT.md +++ /dev/null @@ -1,297 +0,0 @@ -# Development Guide - -This guide provides information for developers who want to contribute to or extend CodeWiki. - -## Project Structure - -``` -codewiki/ -├── codewiki/ # Main package -│ ├── cli/ # CLI implementation -│ │ ├── commands/ # CLI commands (config, generate) -│ │ ├── models/ # Data models -│ │ ├── utils/ # Utilities -│ │ └── adapters/ # External integrations -│ ├── src/ # Web application -│ │ ├── be/ # Backend (dependency analysis, agents) -│ │ │ ├── agent_orchestrator.py -│ │ │ ├── agent_tools/ -│ │ │ ├── cluster_modules.py -│ │ │ ├── dependency_analyzer/ -│ │ │ ├── documentation_generator.py -│ │ │ └── llm_services.py -│ │ └── fe/ # Frontend (web interface) -│ │ ├── web_app.py -│ │ ├── routes.py -│ │ ├── github_processor.py -│ │ └── visualise_docs.py -│ ├── templates/ # HTML templates -│ └── run_web_app.py # Web app entry point -├── docker/ # Docker configuration -│ ├── Dockerfile -│ ├── docker-compose.yml -│ └── env.example -├── img/ # Images and assets -├── paper/ # Research paper source -├── tests/ # Test suite -├── output/ # Generated documentation output -├── pyproject.toml # Project metadata -├── requirements.txt # Python dependencies -└── README.md # Main documentation -``` - -## Development Setup - -### Prerequisites - -- Python 3.12+ -- Node.js (for mermaid validation) -- Git -- Tree-sitter language parsers - -### Installation - -```bash -# Clone the repository -git clone https://github.com/FSoft-AI4Code/CodeWiki.git -cd CodeWiki - -# Create virtual environment -python3.12 -m venv .venv -source .venv/bin/activate # On Windows: .venv\Scripts\activate - -# Install in development mode -pip install -e . - -# Install development dependencies -pip install -r requirements.txt -``` - -## Core Components - -### Backend Architecture - -#### 1. Dependency Analysis (`src/be/dependency_analyzer/`) - -- **AST Parser**: Tree-sitter based parsing for 7 languages -- **Dependency Graph Builder**: Constructs call graphs and dependency relationships -- **Analyzers**: Language-specific analyzers (Python, Java, JavaScript, TypeScript, C, C++, C#) - -#### 2. Module Clustering (`src/be/cluster_modules.py`) - -- Hierarchical decomposition of repository structure -- Feature-oriented module partitioning -- Topological sorting for dependency ordering - -#### 3. Agent System (`src/be/agent_orchestrator.py`) - -- Recursive agent-based documentation generation -- Dynamic delegation for complex modules -- Cross-module reference management - -#### 4. Agent Tools (`src/be/agent_tools/`) - -- `read_code_components.py`: Code reading utilities -- `generate_sub_module_documentations.py`: Sub-module documentation generation -- `str_replace_editor.py`: Documentation editing tools -- `deps.py`: Dependency traversal tools - -### Frontend Architecture - -#### Web Application (`src/fe/`) - -- **FastAPI Backend**: `web_app.py`, `routes.py` -- **GitHub Integration**: `github_processor.py` -- **Documentation Viewer**: `visualise_docs.py` -- **Background Processing**: `background_worker.py` - -### CLI Architecture - -#### Command Structure (`cli/commands/`) - -- `config.py`: Configuration management (API settings + agent instructions) -- `generate.py`: Documentation generation with customization options - -#### Models (`cli/models/`) - -- `config.py`: Configuration data models including `AgentInstructions` -- `job.py`: Job tracking models - -#### Utilities (`cli/utils/`) - -- `fs.py`: File system operations -- `validation.py`: Input validation -- `progress.py`: Progress tracking -- `logging.py`: Logging configuration - -### Agent Instructions System - -The `AgentInstructions` model (`cli/models/config.py`) enables customization: - -```python -@dataclass -class AgentInstructions: - include_patterns: Optional[List[str]] = None # e.g., ["*.cs"] - exclude_patterns: Optional[List[str]] = None # e.g., ["*Tests*"] - focus_modules: Optional[List[str]] = None # e.g., ["src/core"] - doc_type: Optional[str] = None # api, architecture, etc. - custom_instructions: Optional[str] = None # Free-form text -``` - -**How it flows through the system:** - -1. **CLI Options** (`generate.py`) → Runtime `AgentInstructions` -2. **Persistent Config** (`~/.codewiki/config.json`) → Default `AgentInstructions` -3. **Backend Config** (`src/config.py`) → `agent_instructions` dict -4. **Dependency Analyzer** → Uses `include_patterns` and `exclude_patterns` for file filtering -5. **Agent Orchestrator** → Injects `custom_instructions` into LLM prompts - -## Extending Agent Instructions - -To add new customization options to the agent instructions system: - -1. **Update the model** in `cli/models/config.py`: - -```python -@dataclass -class AgentInstructions: - # ... existing fields ... - new_option: Optional[str] = None # Add new field -``` - -2. **Update serialization methods** (`to_dict`, `from_dict`, `is_empty`, `get_prompt_addition`) - -3. **Add CLI options** in `cli/commands/generate.py` and `cli/commands/config.py` - -4. **Update backend Config** if the option affects analysis (`src/config.py`) - -5. **Use in relevant components**: - - File filtering → `dependency_analyzer/ast_parser.py` - - Prompts → `be/prompt_template.py` - - Agent creation → `be/agent_orchestrator.py` - ---- - -## Adding Support for New Languages - -To add support for a new programming language: - -1. **Add language analyzer** in `src/be/dependency_analyzer/analyzers/`: - -```python -# new_language.py -from .base import BaseAnalyzer - -class NewLanguageAnalyzer(BaseAnalyzer): - def __init__(self): - super().__init__("new_language") - - def extract_dependencies(self, ast_node): - # Implement dependency extraction - pass - - def extract_components(self, ast_node): - # Implement component extraction - pass -``` - -2. **Register the analyzer** in `src/be/dependency_analyzer/ast_parser.py`: - -```python -LANGUAGE_ANALYZERS = { - # ... existing languages ... - "new_language": NewLanguageAnalyzer, -} -``` - -3. **Add file extensions** in configuration - -4. **Add tests** for the new language - -## Testing - -```bash -# Run all tests -pytest - -# Run specific test file -pytest tests/test_dependency_analyzer.py - -# Run with coverage -pytest --cov=codewiki tests/ -``` - -## Code Style - -- Follow PEP 8 for Python code -- Use type hints where applicable -- Write docstrings for public functions and classes -- Keep functions focused and modular - -## Contributing - -1. **Fork the repository** -2. **Create a feature branch**: `git checkout -b feature/your-feature` -3. **Make your changes** -4. **Write/update tests** -5. **Ensure tests pass**: `pytest` -6. **Commit your changes**: `git commit -am 'Add new feature'` -7. **Push to the branch**: `git push origin feature/your-feature` -8. **Submit a pull request** - -## Documentation Generation Workflow - -```mermaid -graph TB - A[Repository Input] --> B[Dependency Graph Construction] - B --> C[Hierarchical Decomposition] - C --> D[Module Tree] - D --> E[Recursive Agent Processing] - E --> F{Complexity Check} - F -->|Complex| G[Dynamic Delegation] - F -->|Simple| H[Generate Documentation] - G --> E - H --> I[Cross-Module References] - I --> J[Hierarchical Assembly] - J --> K[Comprehensive Documentation] -``` - -## Debugging - -### Enable Verbose Logging - -```bash -# CLI -codewiki generate --verbose - -# Environment variable -export CODEWIKI_LOG_LEVEL=DEBUG -``` - -### Common Issues - -**Tree-sitter parser errors**: -- Ensure language parsers are properly installed -- Check file encoding (UTF-8 expected) - -**LLM API errors**: -- Verify API keys and endpoints -- Check rate limits -- Enable retry logic - -**Memory issues with large repositories**: -- Adjust module decomposition threshold -- Increase delegation depth limit - -## Performance Optimization - -- **Caching**: Results are cached to avoid redundant processing -- **Parallel Processing**: Multiple modules can be processed concurrently -- **Incremental Updates**: Only process changed modules (future work) - -## Support - -For development questions: -- GitHub Issues: https://github.com/FSoft-AI4Code/CodeWiki/issues -- Main Documentation: [README.md](README.md) - diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d521cdc --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +MIT License + +Copyright (c) 2026 Code2Test Contributors +Copyright (c) 2025 FSoft-AI4Code (portions derived from CodeWiki, +https://github.com/FSoft-AI4Code/CodeWiki) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index c007746..615b9b3 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Code2Test picks the model in this order: 4. Gemini, when the API key saved with `code2test config set --api-key` is a Google key. 5. `openai:gpt-4o-mini` otherwise. -Each provider reads its usual key variable, for example `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`. +Each provider reads its usual key variable, for example `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`. A key saved with `code2test config set --api-key` is stored in the system keychain and exported as `OPENAI_API_KEY` (and as the Google key variables) when Code2Test runs, replacing any value already in the environment. ```bash export ANTHROPIC_API_KEY=... @@ -100,8 +100,9 @@ The suite Code2Test writes therefore always passes: every test either passes or | `code2test benchmark [PATH]` | Measure mutation score and diagnosis accuracy of the generated tests | | `code2test intent show\|edit\|export [NAME]` | Review or correct inferred intents | | `code2test report` | Summarize stored intents by confidence | -| `code2test init` | Create a `.code2test` configuration for a repository | -| `code2test config set\|show\|validate` | Store API credentials and model settings | +| `code2test init` | Write a starter `.code2test/config.json` (the other commands take their settings from options, not from this file) | +| `code2test config set\|show\|validate` | Store the API key and model (`--api-key`, `--main-model`); `validate` sends the model one test request | +| `code2test version` | Show the installed version | ### `code2test test` @@ -161,6 +162,7 @@ The benchmark also scores the diagnosis agent on failures whose cause is known. | `.code2test/metrics.json` | Metrics of the last `test --auto` run | | `.code2test/benchmark.json` | Results of the last `benchmark` run, including uncaught mutants | | `.code2test/dependency_graphs/` | The analyzed dependency graph | +| `.code2test/config.json` | Starter settings written by `code2test init` (not read by other commands) | | `tests/reports/report.html`, `report.json` | Run reports, when `--report` is used | ## Results @@ -187,7 +189,7 @@ On these well-documented libraries, nearly every inferred intent came from the d ## Limitations -- **Python only.** Other languages are analyzed but get no tests; the tool says so when it skips them. +- **Python only.** Files in other languages are part of the dependency graph but get no tests; the tool says so when it skips them. - **Behavior is inferred, not specified.** Expected values come from the inferred intent and the source. Review generated tests before relying on them, as you would review a colleague's. - **Each run calls the model.** A run sends source code to the configured provider and costs API usage: one or more calls per component, plus repairs. - **Results come from two libraries and two models of one provider, one run each.** Larger codebases, code with external services, other models and run-to-run variance are not yet measured. @@ -195,21 +197,29 @@ On these well-documented libraries, nearly every inferred intent came from the d ## Development ```bash +git clone https://github.com/Cirkitly/code2test.git +cd code2test pip install -e ".[dev]" pytest ``` -The test suite drives the whole pipeline offline with scripted models, so no API key is needed. CI runs it on every push. +The tests in `tests/` drive the whole pipeline offline with scripted models, so no API key is needed. CI ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) runs them on every push and pull request. + +When changing the pipeline: + +- Add or update a test in `tests/` for every behavior change. `tests/test_pipeline.py` shows how to script model responses. +- To check a change against a real model, run `code2test test --auto`, `code2test verify` and `code2test benchmark` on a small library with its own tests removed, as [`research/scripts/run_experiments.sh`](research/scripts/run_experiments.sh) does. +- Keep [docs/architecture.md](docs/architecture.md) in step with the code it describes. Further documentation: -- [docs/architecture.md](docs/architecture.md): components and data flow +- [docs/architecture.md](docs/architecture.md): modules and data flow - [docs/quickstart.md](docs/quickstart.md): a step-by-step first run +- [docs/task.md](docs/task.md): implementation status - [docs/proposal.md](docs/proposal.md): the original project proposal and its success criteria - [docs/real_model_run.md](docs/real_model_run.md): the first real-model runs and the failures they exposed - [research/](research/): the paper, experiment scripts and raw results -- [docs/task.md](docs/task.md): implementation status ## License -MIT +MIT; see [LICENSE](LICENSE). The dependency analyzer is derived from [CodeWiki](https://github.com/FSoft-AI4Code/CodeWiki) (MIT, copyright 2025 FSoft-AI4Code), whose notice is kept in the LICENSE file. diff --git a/code2test/__init__.py b/code2test/__init__.py index dadd40d..5dc9cd0 100644 --- a/code2test/__init__.py +++ b/code2test/__init__.py @@ -1,10 +1,10 @@ """ -Code2Test: Intelligent Test Generation for Legacy Codebases. +Code2Test: intent-first test generation for legacy Python code. This package provides a CLI tool for generating comprehensive test suites -for codebases lacking tests using intent-first AI analysis. +for Python code that lacks tests, using intent-first LLM analysis. -Built on FSoft AI4Code's CodeWiki framework. +Dependency analysis derived from FSoft AI4Code's CodeWiki. """ __version__ = "0.1.0" diff --git a/code2test/cli/adapters/__init__.py b/code2test/cli/adapters/__init__.py deleted file mode 100644 index 6204b52..0000000 --- a/code2test/cli/adapters/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Adapters for integrating with backend modules.""" - -__all__ = [] - diff --git a/code2test/cli/adapters/doc_generator.py b/code2test/cli/adapters/doc_generator.py deleted file mode 100644 index 97311cb..0000000 --- a/code2test/cli/adapters/doc_generator.py +++ /dev/null @@ -1,293 +0,0 @@ -""" -CLI adapter for documentation generator backend. - -This adapter wraps the existing backend documentation_generator.py -and provides CLI-specific functionality like progress reporting. -""" - -from pathlib import Path -from typing import Dict, Any -import time -import asyncio -import os -import logging -import sys - - -from code2test.cli.utils.progress import ProgressTracker -from code2test.cli.models.job import DocumentationJob, LLMConfig -from code2test.cli.utils.errors import APIError - -# Import backend modules -from code2test.src.be.documentation_generator import DocumentationGenerator -from code2test.src.config import Config as BackendConfig, set_cli_context - - -class CLIDocumentationGenerator: - """ - CLI adapter for documentation generation with progress reporting. - - This class wraps the backend documentation generator and adds - CLI-specific features like progress tracking and error handling. - """ - - def __init__( - self, - repo_path: Path, - output_dir: Path, - config: Dict[str, Any], - verbose: bool = False, - generate_html: bool = False - ): - """ - Initialize the CLI documentation generator. - - Args: - repo_path: Repository path - output_dir: Output directory - config: LLM configuration - verbose: Enable verbose output - generate_html: Whether to generate HTML viewer - """ - self.repo_path = repo_path - self.output_dir = output_dir - self.config = config - self.verbose = verbose - self.generate_html = generate_html - self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) - self.job = DocumentationJob() - - # Setup job metadata - self.job.repository_path = str(repo_path) - self.job.repository_name = repo_path.name - self.job.output_directory = str(output_dir) - self.job.llm_config = LLMConfig( - main_model=config.get('main_model', ''), - cluster_model=config.get('cluster_model', ''), - base_url=config.get('base_url', '') - ) - - # Configure backend logging - self._configure_backend_logging() - - def _configure_backend_logging(self): - """Configure backend logger for CLI use with colored output.""" - from code2test.src.be.dependency_analyzer.utils.logging_config import ColoredFormatter - - # Get backend logger (parent of all backend modules) - backend_logger = logging.getLogger('codewiki.src.be') - - # Remove existing handlers to avoid duplicates - backend_logger.handlers.clear() - - if self.verbose: - # In verbose mode, show INFO and above - backend_logger.setLevel(logging.INFO) - - # Create console handler with formatting - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(logging.INFO) - - # Use colored formatter for better readability - colored_formatter = ColoredFormatter() - console_handler.setFormatter(colored_formatter) - - # Add handler to logger - backend_logger.addHandler(console_handler) - else: - # In non-verbose mode, suppress backend logs (use WARNING level to hide INFO/DEBUG) - backend_logger.setLevel(logging.WARNING) - - # Create console handler for warnings and errors only - console_handler = logging.StreamHandler(sys.stderr) - console_handler.setLevel(logging.WARNING) - - # Use colored formatter even for warnings/errors - colored_formatter = ColoredFormatter() - console_handler.setFormatter(colored_formatter) - - backend_logger.addHandler(console_handler) - - # Prevent propagation to root logger to avoid duplicate messages - backend_logger.propagate = False - - def generate(self) -> DocumentationJob: - """ - Generate documentation with progress tracking. - - Returns: - Completed DocumentationJob - - Raises: - APIError: If LLM API call fails - """ - self.job.start() - start_time = time.time() - - try: - # Set CLI context for backend - set_cli_context(True) - - # Create backend config with CLI settings - backend_config = BackendConfig.from_cli( - repo_path=str(self.repo_path), - output_dir=str(self.output_dir), - llm_base_url=self.config.get('base_url'), - llm_api_key=self.config.get('api_key'), - main_model=self.config.get('main_model'), - cluster_model=self.config.get('cluster_model'), - fallback_model=self.config.get('fallback_model'), - max_tokens=self.config.get('max_tokens', 32768), - max_token_per_module=self.config.get('max_token_per_module', 36369), - max_token_per_leaf_module=self.config.get('max_token_per_leaf_module', 16000), - max_depth=self.config.get('max_depth', 2), - agent_instructions=self.config.get('agent_instructions') - ) - - # Run backend documentation generation - asyncio.run(self._run_backend_generation(backend_config)) - - # Stage 4: HTML Generation (optional) - if self.generate_html: - self._run_html_generation() - - # Stage 5: Finalization (metadata already created by backend) - self._finalize_job() - - # Complete job - generation_time = time.time() - start_time - self.job.complete() - - return self.job - - except APIError as e: - self.job.fail(str(e)) - raise - except Exception as e: - self.job.fail(str(e)) - raise - - async def _run_backend_generation(self, backend_config: BackendConfig): - """Run the backend documentation generation with progress tracking.""" - - # Stage 1: Dependency Analysis - self.progress_tracker.start_stage(1, "Dependency Analysis") - if self.verbose: - self.progress_tracker.update_stage(0.2, "Initializing dependency analyzer...") - - # Create documentation generator - doc_generator = DocumentationGenerator(backend_config) - - if self.verbose: - self.progress_tracker.update_stage(0.5, "Parsing source files...") - - # Build dependency graph - try: - components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph() - self.job.statistics.total_files_analyzed = len(components) - self.job.statistics.leaf_nodes = len(leaf_nodes) - - if self.verbose: - self.progress_tracker.update_stage(1.0, f"Found {len(leaf_nodes)} leaf nodes") - except Exception as e: - raise APIError(f"Dependency analysis failed: {e}") - - self.progress_tracker.complete_stage() - - # Stage 2: Module Clustering - self.progress_tracker.start_stage(2, "Module Clustering") - if self.verbose: - self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...") - - # Import clustering function - from code2test.src.be.cluster_modules import cluster_modules - from code2test.src.utils import file_manager - from code2test.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME - - working_dir = str(self.output_dir.absolute()) - file_manager.ensure_directory(working_dir) - first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) - module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - - try: - if os.path.exists(first_module_tree_path): - module_tree = file_manager.load_json(first_module_tree_path) - else: - module_tree = cluster_modules(leaf_nodes, components, backend_config) - file_manager.save_json(module_tree, first_module_tree_path) - - file_manager.save_json(module_tree, module_tree_path) - self.job.module_count = len(module_tree) - - if self.verbose: - self.progress_tracker.update_stage(1.0, f"Created {len(module_tree)} modules") - except Exception as e: - raise APIError(f"Module clustering failed: {e}") - - self.progress_tracker.complete_stage() - - # Stage 3: Documentation Generation - self.progress_tracker.start_stage(3, "Documentation Generation") - if self.verbose: - self.progress_tracker.update_stage(0.1, "Generating module documentation...") - - try: - # Run the actual documentation generation - await doc_generator.generate_module_documentation(components, leaf_nodes) - - if self.verbose: - self.progress_tracker.update_stage(0.9, "Creating repository overview...") - - # Create metadata - doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - - # Collect generated files - for file_path in os.listdir(working_dir): - if file_path.endswith('.md') or file_path.endswith('.json'): - self.job.files_generated.append(file_path) - - except Exception as e: - raise APIError(f"Documentation generation failed: {e}") - - self.progress_tracker.complete_stage() - - def _run_html_generation(self): - """Run HTML generation stage.""" - self.progress_tracker.start_stage(4, "HTML Generation") - - from code2test.cli.html_generator import HTMLGenerator - - # Generate HTML - html_generator = HTMLGenerator() - - if self.verbose: - self.progress_tracker.update_stage(0.3, "Loading module tree and metadata...") - - repo_info = html_generator.detect_repository_info(self.repo_path) - - # Generate HTML with auto-loading of module_tree and metadata from docs_dir - output_path = self.output_dir / "index.html" - html_generator.generate( - output_path=output_path, - title=repo_info['name'], - repository_url=repo_info['url'], - github_pages_url=repo_info['github_pages_url'], - docs_dir=self.output_dir # Auto-load module_tree and metadata from here - ) - - self.job.files_generated.append("index.html") - - if self.verbose: - self.progress_tracker.update_stage(1.0, "Generated index.html") - - self.progress_tracker.complete_stage() - - def _finalize_job(self): - """Finalize the job (metadata already created by backend).""" - # Just verify metadata exists - metadata_path = self.output_dir / "metadata.json" - if not metadata_path.exists(): - # Create our own if backend didn't - with open(metadata_path, 'w') as f: - f.write(self.job.to_json()) - diff --git a/code2test/cli/commands/config.py b/code2test/cli/commands/config.py index 607efd7..1913bd9 100644 --- a/code2test/cli/commands/config.py +++ b/code2test/cli/commands/config.py @@ -1,39 +1,22 @@ """ -Configuration commands for CodeWiki CLI. +Configuration commands for the Code2Test CLI. """ import json +import os import sys +from typing import Optional + import click -from typing import Optional, List from code2test.cli.config_manager import ConfigManager -from code2test.cli.models.config import AgentInstructions -from code2test.cli.utils.errors import ( - ConfigurationError, - handle_error, - EXIT_SUCCESS, - EXIT_CONFIG_ERROR -) -from code2test.cli.utils.validation import ( - validate_url, - validate_api_key, - validate_model_name, - is_top_tier_model, - mask_api_key -) - - -def parse_patterns(patterns_str: str) -> List[str]: - """Parse comma-separated patterns into a list.""" - if not patterns_str: - return [] - return [p.strip() for p in patterns_str.split(',') if p.strip()] +from code2test.cli.utils.errors import ConfigurationError, handle_error, EXIT_CONFIG_ERROR +from code2test.cli.utils.validation import validate_api_key, validate_model_name, mask_api_key @click.group(name="config") def config_group(): - """Manage CodeWiki configuration (API credentials and settings).""" + """Manage Code2Test configuration (API key and model).""" pass @@ -41,198 +24,50 @@ def config_group(): @click.option( "--api-key", type=str, - help="LLM API key (stored securely in system keychain)" -) -@click.option( - "--base-url", - type=str, - help="LLM API base URL (e.g., https://api.anthropic.com)" + help="LLM API key (stored in the system keychain; exported as OPENAI_API_KEY when Code2Test runs)" ) @click.option( "--main-model", type=str, - help="Primary model for documentation generation" -) -@click.option( - "--cluster-model", - type=str, - help="Model for module clustering (recommend top-tier)" -) -@click.option( - "--fallback-model", - type=str, - help="Fallback model for documentation generation" -) -@click.option( - "--max-tokens", - type=int, - help="Maximum tokens for LLM response (default: 32768)" -) -@click.option( - "--max-token-per-module", - type=int, - help="Maximum tokens per module for clustering (default: 36369)" + help="Model for test generation, as a pydantic-ai model string (e.g. anthropic:claude-sonnet-5-5)" ) -@click.option( - "--max-token-per-leaf-module", - type=int, - help="Maximum tokens per leaf module (default: 16000)" -) -@click.option( - "--max-depth", - type=int, - help="Maximum depth for hierarchical decomposition (default: 2)" -) -def config_set( - api_key: Optional[str], - base_url: Optional[str], - main_model: Optional[str], - cluster_model: Optional[str], - fallback_model: Optional[str], - max_tokens: Optional[int], - max_token_per_module: Optional[int], - max_token_per_leaf_module: Optional[int], - max_depth: Optional[int] -): +def config_set(api_key: Optional[str], main_model: Optional[str]): """ - Set configuration values for CodeWiki. - - API keys are stored securely in your system keychain: - • macOS: Keychain Access - • Windows: Credential Manager - • Linux: Secret Service (GNOME Keyring, KWallet) - + Set the API key and the model Code2Test uses. + + The API key is stored in your system keychain (macOS Keychain, Windows + Credential Manager, or Linux Secret Service). The CODE2TEST_MODEL + environment variable, when set, takes precedence over the saved model. + Examples: - - \b - # Set all configuration - $ codewiki config set --api-key sk-abc123 --base-url https://api.anthropic.com \\ - --main-model claude-sonnet-4 --cluster-model claude-sonnet-4 --fallback-model glm-4p5 - - \b - # Update only API key - $ codewiki config set --api-key sk-new-key - - \b - # Set max tokens for LLM response - $ codewiki config set --max-tokens 16384 - + \b - # Set all max token settings - $ codewiki config set --max-tokens 32768 --max-token-per-module 40000 --max-token-per-leaf-module 20000 - + # Key and model for test generation + $ code2test config set --api-key sk-abc123 --main-model openai:gpt-4o-mini + \b - # Set max depth for hierarchical decomposition - $ codewiki config set --max-depth 3 + # Update only the API key + $ code2test config set --api-key sk-new-key """ try: - # Check if at least one option is provided - if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth]): + if not api_key and not main_model: click.echo("No options provided. Use --help for usage information.") sys.exit(EXIT_CONFIG_ERROR) - - # Validate inputs before saving - validated_data = {} - - if api_key: - validated_data['api_key'] = validate_api_key(api_key) - - if base_url: - validated_data['base_url'] = validate_url(base_url) - - if main_model: - validated_data['main_model'] = validate_model_name(main_model) - - if cluster_model: - validated_data['cluster_model'] = validate_model_name(cluster_model) - - if fallback_model: - validated_data['fallback_model'] = validate_model_name(fallback_model) - - if max_tokens is not None: - if max_tokens < 1: - raise ConfigurationError("max_tokens must be a positive integer") - validated_data['max_tokens'] = max_tokens - - if max_token_per_module is not None: - if max_token_per_module < 1: - raise ConfigurationError("max_token_per_module must be a positive integer") - validated_data['max_token_per_module'] = max_token_per_module - - if max_token_per_leaf_module is not None: - if max_token_per_leaf_module < 1: - raise ConfigurationError("max_token_per_leaf_module must be a positive integer") - validated_data['max_token_per_leaf_module'] = max_token_per_leaf_module - - if max_depth is not None: - if max_depth < 1: - raise ConfigurationError("max_depth must be a positive integer") - validated_data['max_depth'] = max_depth - - # Create config manager and save + + api_key = validate_api_key(api_key) if api_key else None + main_model = validate_model_name(main_model) if main_model else None + manager = ConfigManager() - manager.load() # Load existing config if present - - manager.save( - api_key=validated_data.get('api_key'), - base_url=validated_data.get('base_url'), - main_model=validated_data.get('main_model'), - cluster_model=validated_data.get('cluster_model'), - fallback_model=validated_data.get('fallback_model'), - max_tokens=validated_data.get('max_tokens'), - max_token_per_module=validated_data.get('max_token_per_module'), - max_token_per_leaf_module=validated_data.get('max_token_per_leaf_module'), - max_depth=validated_data.get('max_depth') - ) - - # Display success messages + manager.load() # Keep settings saved earlier + manager.save(api_key=api_key, main_model=main_model) + click.echo() if api_key: - if manager.keyring_available: - click.secho("✓ API key saved to system keychain", fg="green") - else: - click.secho( - "⚠️ System keychain unavailable. API key stored in encrypted file.", - fg="yellow" - ) - - if base_url: - click.secho(f"✓ Base URL: {base_url}", fg="green") - + click.secho("✓ API key saved to system keychain", fg="green") if main_model: click.secho(f"✓ Main model: {main_model}", fg="green") - - if cluster_model: - click.secho(f"✓ Cluster model: {cluster_model}", fg="green") - - # Warn if not using top-tier model for clustering - if not is_top_tier_model(cluster_model): - click.secho( - "\n⚠️ Cluster model is not a top-tier LLM. " - "Documentation quality may be suboptimal.", - fg="yellow" - ) - click.echo( - " Recommended models: claude-opus, claude-sonnet-4, gpt-4, gpt-4-turbo" - ) - - if fallback_model: - click.secho(f"✓ Fallback model: {fallback_model}", fg="green") - - if max_tokens: - click.secho(f"✓ Max tokens: {max_tokens}", fg="green") - - if max_token_per_module: - click.secho(f"✓ Max token per module: {max_token_per_module}", fg="green") - - if max_token_per_leaf_module: - click.secho(f"✓ Max token per leaf module: {max_token_per_leaf_module}", fg="green") - - if max_depth: - click.secho(f"✓ Max depth: {max_depth}", fg="green") - click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True)) - + except ConfigurationError as e: click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) sys.exit(e.exit_code) @@ -241,439 +76,103 @@ def config_set( @config_group.command(name="show") -@click.option( - "--json", - "output_json", - is_flag=True, - help="Output in JSON format" -) +@click.option("--json", "output_json", is_flag=True, help="Output in JSON format") def config_show(output_json: bool): """ - Display current configuration. - - API keys are masked for security (showing only first and last 4 characters). - + Display the current configuration. + + The API key is masked, showing only its first and last 4 characters. + Examples: - - \b - # Display configuration - $ codewiki config show - + \b - # Display as JSON - $ codewiki config show --json + $ code2test config show + $ code2test config show --json """ try: manager = ConfigManager() - if not manager.load(): click.secho("\n✗ Configuration not found.", fg="red", err=True) - click.echo("\nPlease run 'codewiki config set' to configure your API credentials:") - click.echo(" codewiki config set --api-key --base-url \\") - click.echo(" --main-model --cluster-model --fallback-model ") - click.echo("\nFor more help: codewiki config set --help") + click.echo("\nPlease run 'code2test config set' to configure your API key and model:") + click.echo(" code2test config set --api-key --main-model ") + click.echo("\nFor more help: code2test config set --help") sys.exit(EXIT_CONFIG_ERROR) - + config = manager.get_config() api_key = manager.get_api_key() - + env_model = os.environ.get("CODE2TEST_MODEL") + if output_json: - # JSON output output = { "api_key": mask_api_key(api_key) if api_key else "Not set", - "api_key_storage": "keychain" if manager.keyring_available else "encrypted_file", - "base_url": config.base_url if config else "", "main_model": config.main_model if config else "", - "cluster_model": config.cluster_model if config else "", - "fallback_model": config.fallback_model if config else "glm-4p5", - "default_output": config.default_output if config else "docs", - "max_tokens": config.max_tokens if config else 32768, - "max_token_per_module": config.max_token_per_module if config else 36369, - "max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000, - "max_depth": config.max_depth if config else 2, - "agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {}, - "config_file": str(manager.config_file_path) + "code2test_model_env": env_model or "", + "config_file": str(manager.config_file_path), } click.echo(json.dumps(output, indent=2)) + return + + click.echo() + click.secho("Code2Test Configuration", fg="blue", bold=True) + click.echo("━" * 40) + click.echo() + if api_key: + click.echo(f" API Key: {mask_api_key(api_key)} (in system keychain)") else: - # Human-readable output - click.echo() - click.secho("CodeWiki Configuration", fg="blue", bold=True) - click.echo("━" * 40) - click.echo() - - click.secho("Credentials", fg="cyan", bold=True) - if api_key: - storage = "system keychain" if manager.keyring_available else "encrypted file" - click.echo(f" API Key: {mask_api_key(api_key)} (in {storage})") - else: - click.secho(" API Key: Not set", fg="yellow") - - click.echo() - click.secho("API Settings", fg="cyan", bold=True) - if config: - click.echo(f" Base URL: {config.base_url or 'Not set'}") - click.echo(f" Main Model: {config.main_model or 'Not set'}") - click.echo(f" Cluster Model: {config.cluster_model or 'Not set'}") - click.echo(f" Fallback Model: {config.fallback_model or 'Not set'}") - else: - click.secho(" Not configured", fg="yellow") - - click.echo() - click.secho("Output Settings", fg="cyan", bold=True) - if config: - click.echo(f" Default Output: {config.default_output}") - - click.echo() - click.secho("Token Settings", fg="cyan", bold=True) - if config: - click.echo(f" Max Tokens: {config.max_tokens}") - click.echo(f" Max Token/Module: {config.max_token_per_module}") - click.echo(f" Max Token/Leaf Module: {config.max_token_per_leaf_module}") - - click.echo() - click.secho("Decomposition Settings", fg="cyan", bold=True) - if config: - click.echo(f" Max Depth: {config.max_depth}") - - click.echo() - click.secho("Agent Instructions", fg="cyan", bold=True) - if config and config.agent_instructions and not config.agent_instructions.is_empty(): - agent = config.agent_instructions - if agent.include_patterns: - click.echo(f" Include patterns: {', '.join(agent.include_patterns)}") - if agent.exclude_patterns: - click.echo(f" Exclude patterns: {', '.join(agent.exclude_patterns)}") - if agent.focus_modules: - click.echo(f" Focus modules: {', '.join(agent.focus_modules)}") - if agent.doc_type: - click.echo(f" Doc type: {agent.doc_type}") - if agent.custom_instructions: - click.echo(f" Custom instructions: {agent.custom_instructions[:50]}...") - else: - click.secho(" Using defaults (no custom settings)", fg="yellow") - - click.echo() - click.echo(f"Configuration file: {manager.config_file_path}") - click.echo() - + click.secho(" API Key: Not set", fg="yellow") + click.echo(f" Main Model: {(config.main_model if config else '') or 'Not set'}") + if env_model: + click.echo(f" CODE2TEST_MODEL: {env_model} (overrides the main model)") + click.echo() + click.echo(f"Configuration file: {manager.config_file_path}") + click.echo() + except Exception as e: sys.exit(handle_error(e)) @config_group.command(name="validate") -@click.option( - "--quick", - is_flag=True, - help="Skip API connectivity test" -) -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Show detailed validation steps" -) +@click.option("--quick", is_flag=True, help="Skip the test request to the model") +@click.option("--verbose", "-v", is_flag=True, help="Show the full error on failure") def config_validate(quick: bool, verbose: bool): """ - Validate configuration and test LLM API connectivity. - - Checks: - • Configuration file exists and is valid - • API key is present - • API settings are correctly formatted - • (Optional) API connectivity test - + Check which model Code2Test will use and that it answers. + + Resolves the model exactly as 'code2test test' does, then sends it one + short request (unless --quick). + Examples: - - \b - # Full validation with API test - $ codewiki config validate - - \b - # Quick validation (config only) - $ codewiki config validate --quick - + \b - # Verbose output - $ codewiki config validate --verbose + $ code2test config validate + $ code2test config validate --quick """ + from code2test.cli.display import DisplayManager + from code2test.cli.utils.model import resolve_model + try: click.echo() - click.secho("Validating configuration...", fg="blue", bold=True) - click.echo() - - manager = ConfigManager() - - # Step 1: Check config file - if verbose: - click.echo("[1/5] Checking configuration file...") - click.echo(f" Path: {manager.config_file_path}") - - if not manager.load(): - click.secho("✗ Configuration file not found", fg="red") - click.echo() - click.echo("Error: Configuration is incomplete. Run 'codewiki config set --help' for setup instructions.") - sys.exit(EXIT_CONFIG_ERROR) - - if verbose: - click.secho(" ✓ File exists", fg="green") - click.secho(" ✓ Valid JSON format", fg="green") - else: - click.secho("✓ Configuration file exists", fg="green") - - # Step 2: Check API key - if verbose: - click.echo() - click.echo("[2/5] Checking API key...") - storage = "system keychain" if manager.keyring_available else "encrypted file" - click.echo(f" Storage: {storage}") - - api_key = manager.get_api_key() - if not api_key: - click.secho("✗ API key missing", fg="red") - click.echo() - click.echo("Error: API key not set. Run 'codewiki config set --api-key '") - sys.exit(EXIT_CONFIG_ERROR) - - if verbose: - click.secho(f" ✓ API key retrieved", fg="green") - click.secho(f" ✓ Length: {len(api_key)} characters", fg="green") - else: - click.secho("✓ API key present (stored in keychain)", fg="green") - - # Step 3: Check base URL - config = manager.get_config() - if verbose: - click.echo() - click.echo("[3/5] Checking base URL...") - click.echo(f" URL: {config.base_url}") - - if not config.base_url: - click.secho("✗ Base URL not set", fg="red") - sys.exit(EXIT_CONFIG_ERROR) - - try: - validate_url(config.base_url) - if verbose: - click.secho(" ✓ Valid HTTPS URL", fg="green") - else: - click.secho(f"✓ Base URL valid: {config.base_url}", fg="green") - except ConfigurationError as e: - click.secho(f"✗ Invalid base URL: {e.message}", fg="red") - sys.exit(EXIT_CONFIG_ERROR) - - # Step 4: Check models - if verbose: + model = resolve_model(DisplayManager()) + click.secho(f"✓ Model: {model}", fg="green") + + if quick: click.echo() - click.echo("[4/5] Checking model configuration...") - click.echo(f" Main model: {config.main_model}") - click.echo(f" Cluster model: {config.cluster_model}") - click.echo(f" Fallback model: {config.fallback_model}") - - if not config.main_model or not config.cluster_model or not config.fallback_model: - click.secho("✗ Models not configured", fg="red") - sys.exit(EXIT_CONFIG_ERROR) - - if verbose: - click.secho(" ✓ Models configured", fg="green") - else: - click.secho(f"✓ Main model configured: {config.main_model}", fg="green") - click.secho(f"✓ Cluster model configured: {config.cluster_model}", fg="green") - click.secho(f"✓ Fallback model configured: {config.fallback_model}", fg="green") - - # Warn about non-top-tier cluster model - if not is_top_tier_model(config.cluster_model): - click.secho( - "⚠️ Cluster model is not top-tier. Consider using claude-sonnet-4 or gpt-4.", - fg="yellow" - ) - - # Step 5: API connectivity test (unless --quick) - if not quick: - try: - from openai import OpenAI - client = OpenAI(api_key=api_key, base_url=config.base_url) - response = client.models.list() - click.secho("✓ API connectivity test successful", fg="green") - except Exception as e: - click.secho("✗ API connectivity test failed", fg="red") - sys.exit(EXIT_CONFIG_ERROR) - - # Success - click.echo() - click.secho("✓ Configuration is valid!", fg="green", bold=True) - click.echo() - - except ConfigurationError as e: - click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) - sys.exit(e.exit_code) - except Exception as e: - sys.exit(handle_error(e, verbose=verbose)) + return + from pydantic_ai import Agent -@config_group.command(name="agent") -@click.option( - "--include", - "-i", - type=str, - default=None, - help="Comma-separated file patterns to include (e.g., '*.cs,*.py')", -) -@click.option( - "--exclude", - "-e", - type=str, - default=None, - help="Comma-separated patterns to exclude (e.g., '*Tests*,*Specs*')", -) -@click.option( - "--focus", - "-f", - type=str, - default=None, - help="Comma-separated modules/paths to focus on (e.g., 'src/core,src/api')", -) -@click.option( - "--doc-type", - "-t", - type=click.Choice(['api', 'architecture', 'user-guide', 'developer'], case_sensitive=False), - default=None, - help="Default type of documentation to generate", -) -@click.option( - "--instructions", - type=str, - default=None, - help="Custom instructions for the documentation agent", -) -@click.option( - "--clear", - is_flag=True, - help="Clear all agent instructions", -) -def config_agent( - include: Optional[str], - exclude: Optional[str], - focus: Optional[str], - doc_type: Optional[str], - instructions: Optional[str], - clear: bool -): - """ - Configure default agent instructions for documentation generation. - - These settings are used as defaults when running 'codewiki generate'. - Runtime options (--include, --exclude, etc.) override these defaults. - - Examples: - - \b - # Set include patterns for C# projects - $ codewiki config agent --include "*.cs" - - \b - # Exclude test projects - $ codewiki config agent --exclude "*Tests*,*Specs*,test_*" - - \b - # Focus on specific modules - $ codewiki config agent --focus "src/core,src/api" - - \b - # Set default doc type - $ codewiki config agent --doc-type architecture - - \b - # Add custom instructions - $ codewiki config agent --instructions "Focus on public APIs and include usage examples" - - \b - # Clear all agent instructions - $ codewiki config agent --clear - """ - try: - manager = ConfigManager() - - if not manager.load(): - click.secho("\n✗ Configuration not found.", fg="red", err=True) - click.echo("\nPlease run 'codewiki config set' first to configure your API credentials.") + try: + Agent(model).run_sync("Reply with the single word OK.") + except Exception as e: + message = str(e) if verbose else (str(e).splitlines() or [type(e).__name__])[0] + click.secho(f"✗ The model did not answer: {message}", fg="red") sys.exit(EXIT_CONFIG_ERROR) - - config = manager.get_config() - - if clear: - # Clear all agent instructions - config.agent_instructions = AgentInstructions() - manager.save() - click.echo() - click.secho("✓ Agent instructions cleared", fg="green") - click.echo() - return - - # Check if at least one option is provided - if not any([include, exclude, focus, doc_type, instructions]): - # Display current settings - click.echo() - click.secho("Agent Instructions", fg="blue", bold=True) - click.echo("━" * 40) - click.echo() - - agent = config.agent_instructions - if agent and not agent.is_empty(): - if agent.include_patterns: - click.echo(f" Include patterns: {', '.join(agent.include_patterns)}") - if agent.exclude_patterns: - click.echo(f" Exclude patterns: {', '.join(agent.exclude_patterns)}") - if agent.focus_modules: - click.echo(f" Focus modules: {', '.join(agent.focus_modules)}") - if agent.doc_type: - click.echo(f" Doc type: {agent.doc_type}") - if agent.custom_instructions: - click.echo(f" Custom instructions: {agent.custom_instructions}") - else: - click.secho(" No agent instructions configured (using defaults)", fg="yellow") - - click.echo() - click.echo("Use 'codewiki config agent --help' for usage information.") - click.echo() - return - - # Update agent instructions - current = config.agent_instructions or AgentInstructions() - - if include is not None: - current.include_patterns = parse_patterns(include) if include else None - if exclude is not None: - current.exclude_patterns = parse_patterns(exclude) if exclude else None - if focus is not None: - current.focus_modules = parse_patterns(focus) if focus else None - if doc_type is not None: - current.doc_type = doc_type if doc_type else None - if instructions is not None: - current.custom_instructions = instructions if instructions else None - - config.agent_instructions = current - manager.save() - - # Display success messages - click.echo() - if include: - click.secho(f"✓ Include patterns: {parse_patterns(include)}", fg="green") - if exclude: - click.secho(f"✓ Exclude patterns: {parse_patterns(exclude)}", fg="green") - if focus: - click.secho(f"✓ Focus modules: {parse_patterns(focus)}", fg="green") - if doc_type: - click.secho(f"✓ Doc type: {doc_type}", fg="green") - if instructions: - click.secho(f"✓ Custom instructions set", fg="green") - - click.echo("\n" + click.style("Agent instructions updated successfully.", fg="green", bold=True)) + + click.secho("✓ The model answered a test request", fg="green") click.echo() - + except ConfigurationError as e: click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) sys.exit(e.exit_code) except Exception as e: - sys.exit(handle_error(e)) - + sys.exit(handle_error(e, verbose=verbose)) diff --git a/code2test/cli/commands/generate.py b/code2test/cli/commands/generate.py deleted file mode 100644 index 29ce2fb..0000000 --- a/code2test/cli/commands/generate.py +++ /dev/null @@ -1,420 +0,0 @@ -""" -Generate command for documentation generation. -""" - -import sys -import logging -import traceback -from pathlib import Path -from typing import Optional, List, Tuple -import click -import time - -from code2test.cli.config_manager import ConfigManager -from code2test.cli.utils.errors import ( - ConfigurationError, - RepositoryError, - APIError, - handle_error, - EXIT_SUCCESS, -) -from code2test.cli.utils.repo_validator import ( - validate_repository, - check_writable_output, - is_git_repository, - get_git_commit_hash, - get_git_branch, -) -from code2test.cli.utils.logging import create_logger -from code2test.cli.adapters.doc_generator import CLIDocumentationGenerator -from code2test.cli.utils.instructions import display_post_generation_instructions -from code2test.cli.models.job import GenerationOptions -from code2test.cli.models.config import AgentInstructions - - -def parse_patterns(patterns_str: str) -> List[str]: - """Parse comma-separated patterns into a list.""" - if not patterns_str: - return [] - return [p.strip() for p in patterns_str.split(',') if p.strip()] - - -@click.command(name="generate") -@click.option( - "--output", - "-o", - type=click.Path(), - default="docs", - help="Output directory for generated documentation (default: ./docs)", -) -@click.option( - "--create-branch", - is_flag=True, - help="Create a new git branch for documentation changes", -) -@click.option( - "--github-pages", - is_flag=True, - help="Generate index.html for GitHub Pages deployment", -) -@click.option( - "--no-cache", - is_flag=True, - help="Force full regeneration, ignoring cache", -) -@click.option( - "--include", - "-i", - type=str, - default=None, - help="Comma-separated file patterns to include (e.g., '*.cs,*.py'). Overrides defaults.", -) -@click.option( - "--exclude", - "-e", - type=str, - default=None, - help="Comma-separated patterns to exclude (e.g., '*Tests*,*Specs*,test_*')", -) -@click.option( - "--focus", - "-f", - type=str, - default=None, - help="Comma-separated modules/paths to focus on (e.g., 'src/core,src/api')", -) -@click.option( - "--doc-type", - "-t", - type=click.Choice(['api', 'architecture', 'user-guide', 'developer'], case_sensitive=False), - default=None, - help="Type of documentation to generate", -) -@click.option( - "--instructions", - type=str, - default=None, - help="Custom instructions for the documentation agent", -) -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Show detailed progress and debug information", -) -@click.option( - "--max-tokens", - type=int, - default=None, - help="Maximum tokens for LLM response (overrides config)", -) -@click.option( - "--max-token-per-module", - type=int, - default=None, - help="Maximum tokens per module for clustering (overrides config)", -) -@click.option( - "--max-token-per-leaf-module", - type=int, - default=None, - help="Maximum tokens per leaf module (overrides config)", -) -@click.option( - "--max-depth", - type=int, - default=None, - help="Maximum depth for hierarchical decomposition (overrides config)", -) -@click.pass_context -def generate_command( - ctx, - output: str, - create_branch: bool, - github_pages: bool, - no_cache: bool, - include: Optional[str], - exclude: Optional[str], - focus: Optional[str], - doc_type: Optional[str], - instructions: Optional[str], - verbose: bool, - max_tokens: Optional[int], - max_token_per_module: Optional[int], - max_token_per_leaf_module: Optional[int], - max_depth: Optional[int] -): - """ - Generate comprehensive documentation for a code repository. - - Analyzes the current repository and generates documentation using LLM-powered - analysis. Documentation is output to ./docs/ by default. - - Examples: - - \b - # Basic generation - $ codewiki generate - - \b - # With git branch creation and GitHub Pages - $ codewiki generate --create-branch --github-pages - - \b - # Force full regeneration - $ codewiki generate --no-cache - - \b - # C# project: only .cs files, exclude tests - $ codewiki generate --include "*.cs" --exclude "*Tests*,*Specs*" - - \b - # Focus on specific modules with architecture docs - $ codewiki generate --focus "src/core,src/api" --doc-type architecture - - \b - # Custom instructions - $ codewiki generate --instructions "Focus on public APIs and include usage examples" - - \b - # Override max tokens for this generation - $ codewiki generate --max-tokens 16384 - - \b - # Set all max token limits - $ codewiki generate --max-tokens 32768 --max-token-per-module 40000 --max-token-per-leaf-module 20000 - - \b - # Override max depth for hierarchical decomposition - $ codewiki generate --max-depth 3 - """ - logger = create_logger(verbose=verbose) - start_time = time.time() - - # Suppress httpx INFO logs - logging.getLogger("httpx").setLevel(logging.WARNING) - - try: - # Pre-generation checks - logger.step("Validating configuration...", 1, 4) - - # Load configuration - config_manager = ConfigManager() - if not config_manager.load(): - raise ConfigurationError( - "Configuration not found or invalid.\n\n" - "Please run 'codewiki config set' to configure your LLM API credentials:\n" - " codewiki config set --api-key --base-url \\\n" - " --main-model --cluster-model \n\n" - "For more help: codewiki config --help" - ) - - if not config_manager.is_configured(): - raise ConfigurationError( - "Configuration is incomplete. Please run 'codewiki config validate'" - ) - - config = config_manager.get_config() - api_key = config_manager.get_api_key() - - logger.success("Configuration valid") - - # Validate repository - logger.step("Validating repository...", 2, 4) - - repo_path = Path.cwd() - repo_path, languages = validate_repository(repo_path) - - logger.success(f"Repository valid: {repo_path.name}") - if verbose: - logger.debug(f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}") - - # Check git repository - if not is_git_repository(repo_path): - if create_branch: - raise RepositoryError( - "Not a git repository.\n\n" - "The --create-branch flag requires a git repository.\n\n" - "To initialize a git repository: git init" - ) - else: - logger.warning("Not a git repository. Git features unavailable.") - - # Validate output directory - output_dir = Path(output).expanduser().resolve() - check_writable_output(output_dir.parent) - - logger.success(f"Output directory: {output_dir}") - - # Check for existing documentation - if output_dir.exists() and list(output_dir.glob("*.md")): - if not click.confirm( - f"\n{output_dir} already contains documentation. Overwrite?", - default=True - ): - logger.info("Generation cancelled by user.") - sys.exit(EXIT_SUCCESS) - - # Git branch creation (if requested) - branch_name = None - if create_branch: - logger.step("Creating git branch...", 3, 4) - - from code2test.cli.git_manager import GitManager - - git_manager = GitManager(repo_path) - - # Check clean working directory - is_clean, status_msg = git_manager.check_clean_working_directory() - if not is_clean: - raise RepositoryError( - "Working directory has uncommitted changes.\n\n" - f"{status_msg}\n\n" - "Cannot create documentation branch with uncommitted changes.\n" - "Please commit or stash your changes first:\n" - " git add -A && git commit -m \"Your message\"\n" - " # or\n" - " git stash" - ) - - # Create branch - branch_name = git_manager.create_documentation_branch() - logger.success(f"Created branch: {branch_name}") - - # Generate documentation - logger.step("Generating documentation...", 4, 4) - click.echo() - - # Create generation options - generation_options = GenerationOptions( - create_branch=create_branch, - github_pages=github_pages, - no_cache=no_cache, - custom_output=output if output != "docs" else None - ) - - # Create runtime agent instructions from CLI options - runtime_instructions = None - if any([include, exclude, focus, doc_type, instructions]): - runtime_instructions = AgentInstructions( - include_patterns=parse_patterns(include) if include else None, - exclude_patterns=parse_patterns(exclude) if exclude else None, - focus_modules=parse_patterns(focus) if focus else None, - doc_type=doc_type, - custom_instructions=instructions, - ) - - if verbose: - if include: - logger.debug(f"Include patterns: {parse_patterns(include)}") - if exclude: - logger.debug(f"Exclude patterns: {parse_patterns(exclude)}") - if focus: - logger.debug(f"Focus modules: {parse_patterns(focus)}") - if doc_type: - logger.debug(f"Doc type: {doc_type}") - if instructions: - logger.debug(f"Custom instructions: {instructions}") - - # Log max token settings if verbose - if verbose: - effective_max_tokens = max_tokens if max_tokens is not None else config.max_tokens - effective_max_token_per_module = max_token_per_module if max_token_per_module is not None else config.max_token_per_module - effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module - effective_max_depth = max_depth if max_depth is not None else config.max_depth - logger.debug(f"Max tokens: {effective_max_tokens}") - logger.debug(f"Max token/module: {effective_max_token_per_module}") - logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}") - logger.debug(f"Max depth: {effective_max_depth}") - - # Get agent instructions (merge runtime with persistent) - agent_instructions_dict = None - if runtime_instructions and not runtime_instructions.is_empty(): - # Merge with persistent settings - merged = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or (config.agent_instructions.include_patterns if config.agent_instructions else None), - exclude_patterns=runtime_instructions.exclude_patterns or (config.agent_instructions.exclude_patterns if config.agent_instructions else None), - focus_modules=runtime_instructions.focus_modules or (config.agent_instructions.focus_modules if config.agent_instructions else None), - doc_type=runtime_instructions.doc_type or (config.agent_instructions.doc_type if config.agent_instructions else None), - custom_instructions=runtime_instructions.custom_instructions or (config.agent_instructions.custom_instructions if config.agent_instructions else None), - ) - agent_instructions_dict = merged.to_dict() - elif config.agent_instructions and not config.agent_instructions.is_empty(): - agent_instructions_dict = config.agent_instructions.to_dict() - - # Create generator - generator = CLIDocumentationGenerator( - repo_path=repo_path, - output_dir=output_dir, - config={ - 'main_model': config.main_model, - 'cluster_model': config.cluster_model, - 'fallback_model': config.fallback_model, - 'base_url': config.base_url, - 'api_key': api_key, - 'agent_instructions': agent_instructions_dict, - # Max token settings (runtime overrides take precedence) - 'max_tokens': max_tokens if max_tokens is not None else config.max_tokens, - 'max_token_per_module': max_token_per_module if max_token_per_module is not None else config.max_token_per_module, - 'max_token_per_leaf_module': max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module, - # Max depth setting (runtime override takes precedence) - 'max_depth': max_depth if max_depth is not None else config.max_depth, - }, - verbose=verbose, - generate_html=github_pages - ) - - # Run generation - job = generator.generate() - - # Post-generation - generation_time = time.time() - start_time - - # Get repository info - repo_url = None - commit_hash = get_git_commit_hash(repo_path) - current_branch = get_git_branch(repo_path) - - if is_git_repository(repo_path): - try: - import git - repo = git.Repo(repo_path) - if repo.remotes: - repo_url = repo.remotes.origin.url - except: - pass - - # Display instructions - display_post_generation_instructions( - output_dir=output_dir, - repo_name=repo_path.name, - repo_url=repo_url, - branch_name=branch_name, - github_pages=github_pages, - files_generated=job.files_generated, - statistics={ - 'module_count': job.module_count, - 'total_files_analyzed': job.statistics.total_files_analyzed, - 'generation_time': generation_time, - 'total_tokens_used': job.statistics.total_tokens_used, - } - ) - - except ConfigurationError as e: - logger.error(e.message) - logger.error(f"Traceback: {traceback.format_exc()}") - sys.exit(e.exit_code) - except RepositoryError as e: - logger.error(e.message) - logger.error(f"Traceback: {traceback.format_exc()}") - sys.exit(e.exit_code) - except APIError as e: - logger.error(e.message) - logger.error(f"Traceback: {traceback.format_exc()}") - sys.exit(e.exit_code) - except KeyboardInterrupt: - click.echo("\n\nInterrupted by user") - sys.exit(130) - except Exception as e: - sys.exit(handle_error(e, verbose=verbose)) - diff --git a/code2test/cli/commands/init.py b/code2test/cli/commands/init.py index 7c905d2..aee4528 100644 --- a/code2test/cli/commands/init.py +++ b/code2test/cli/commands/init.py @@ -71,4 +71,4 @@ def init_command(path: str) -> None: json.dump(config, f, indent=2) display.success(f"Initialized Code2Test configuration in {config_file}") - display.info("Run 'code2test generate' to start generating tests!") + display.info("Run 'code2test test .' to start generating tests.") diff --git a/code2test/cli/commands/report.py b/code2test/cli/commands/report.py index bfcfb86..f16d3b3 100644 --- a/code2test/cli/commands/report.py +++ b/code2test/cli/commands/report.py @@ -22,12 +22,14 @@ def report_command( output: str ) -> None: """ - Generate quality and coverage reports. + Summarize the stored intents by confidence. + + Run it from the project root after 'code2test test'. The coverage report + is not implemented yet. \b Examples: code2test report summary # View summary report - code2test report coverage --format html # Generate HTML coverage report """ display = DisplayManager() diff --git a/code2test/cli/commands/test.py b/code2test/cli/commands/test.py index 389b0f4..8f8068c 100644 --- a/code2test/cli/commands/test.py +++ b/code2test/cli/commands/test.py @@ -53,7 +53,7 @@ "--framework", type=click.Choice(["pytest", "unittest"]), default="pytest", - help="Test framework to use" + help="Test framework (v1 generates pytest tests; unittest is not supported yet)" ) @click.option( "--include", diff --git a/code2test/cli/commands/verify.py b/code2test/cli/commands/verify.py index de15b05..2fc2742 100644 --- a/code2test/cli/commands/verify.py +++ b/code2test/cli/commands/verify.py @@ -35,7 +35,7 @@ def verify_command( """ Run and verify generated tests. - Runs pytest from the repository root (the current directory), reports + Runs pytest with the current directory as the repository root, reports each failure, and exits with code 1 if any test fails. \b diff --git a/code2test/cli/config_manager.py b/code2test/cli/config_manager.py index 88c6f04..c6cc31f 100644 --- a/code2test/cli/config_manager.py +++ b/code2test/cli/config_manager.py @@ -25,11 +25,11 @@ class ConfigManager: """ - Manages CodeWiki configuration with secure keyring storage for API keys. - - Storage: - - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager, - Linux Secret Service) + Manages Code2Test configuration with secure keyring storage for API keys. + + Storage (locations kept from CodeWiki, so existing configurations keep working): + - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager, + Linux Secret Service), service name "codewiki" - Other settings: ~/.codewiki/config.json """ @@ -81,79 +81,31 @@ def load(self) -> bool: except (json.JSONDecodeError, FileSystemError) as e: raise ConfigurationError(f"Failed to load configuration: {e}") - def save( - self, - api_key: Optional[str] = None, - base_url: Optional[str] = None, - main_model: Optional[str] = None, - cluster_model: Optional[str] = None, - fallback_model: Optional[str] = None, - default_output: Optional[str] = None, - max_tokens: Optional[int] = None, - max_token_per_module: Optional[int] = None, - max_token_per_leaf_module: Optional[int] = None, - max_depth: Optional[int] = None - ): + def save(self, api_key: Optional[str] = None, main_model: Optional[str] = None): """ Save configuration to file and keyring. - + Args: api_key: API key (stored in keyring) - base_url: LLM API base URL - main_model: Primary model - cluster_model: Clustering model - fallback_model: Fallback model - default_output: Default output directory - max_tokens: Maximum tokens for LLM response - max_token_per_module: Maximum tokens per module for clustering - max_token_per_leaf_module: Maximum tokens per leaf module - max_depth: Maximum depth for hierarchical decomposition + main_model: pydantic-ai model string for test generation """ # Ensure config directory exists try: ensure_directory(CONFIG_DIR) except FileSystemError as e: raise ConfigurationError(f"Cannot create config directory: {e}") - + # Load existing config or create new if self._config is None: if CONFIG_FILE.exists(): self.load() else: - from code2test.cli.models.config import AgentInstructions - self._config = Configuration( - base_url="", - main_model="", - cluster_model="", - fallback_model="glm-4p5", - default_output="docs", - agent_instructions=AgentInstructions() - ) - - # Update fields if provided - if base_url is not None: - self._config.base_url = base_url + self._config = Configuration() + if main_model is not None: self._config.main_model = main_model - if cluster_model is not None: - self._config.cluster_model = cluster_model - if fallback_model is not None: - self._config.fallback_model = fallback_model - if default_output is not None: - self._config.default_output = default_output - if max_tokens is not None: - self._config.max_tokens = max_tokens - if max_token_per_module is not None: - self._config.max_token_per_module = max_token_per_module - if max_token_per_leaf_module is not None: - self._config.max_token_per_leaf_module = max_token_per_leaf_module - if max_depth is not None: - self._config.max_depth = max_depth - - # Validate configuration (only if base fields are set) - if self._config.base_url and self._config.main_model and self._config.cluster_model: - self._config.validate() - + self._config.validate() + # Save API key to keyring if api_key is not None: self._api_key = api_key diff --git a/code2test/cli/git_manager.py b/code2test/cli/git_manager.py deleted file mode 100644 index 8a4ec80..0000000 --- a/code2test/cli/git_manager.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Git operations manager for CodeWiki CLI. -""" - -from pathlib import Path -from datetime import datetime -from typing import Optional, Tuple -import git -from git.exc import GitCommandError - -from code2test.cli.utils.errors import RepositoryError - - -class GitManager: - """ - Manages git operations for documentation generation. - - Handles: - - Status checking - - Branch creation - - Committing documentation - - Remote detection - """ - - def __init__(self, repo_path: Path): - """ - Initialize git manager. - - Args: - repo_path: Path to git repository - - Raises: - RepositoryError: If not a valid git repository - """ - self.repo_path = Path(repo_path).expanduser().resolve() - - try: - self.repo = git.Repo(repo_path, search_parent_directories=True) - except git.InvalidGitRepositoryError: - raise RepositoryError( - f"Not a git repository: {repo_path}\n\n" - "To initialize a git repository: git init" - ) - - def check_clean_working_directory(self) -> Tuple[bool, str]: - """ - Check if working directory is clean (no uncommitted changes). - - Returns: - Tuple of (is_clean, status_message) - """ - if self.repo.is_dirty(untracked_files=True): - status_lines = [] - - # Changed files - changed = [item.a_path for item in self.repo.index.diff(None)] - if changed: - status_lines.append(f"Modified: {', '.join(changed[:3])}") - if len(changed) > 3: - status_lines.append(f"... and {len(changed) - 3} more") - - # Untracked files - untracked = self.repo.untracked_files - if untracked: - status_lines.append(f"Untracked: {', '.join(untracked[:3])}") - if len(untracked) > 3: - status_lines.append(f"... and {len(untracked) - 3} more") - - return False, "\n".join(status_lines) - - return True, "Working directory is clean" - - def create_documentation_branch(self, force: bool = False) -> str: - """ - Create a new documentation branch with timestamp. - - Args: - force: Force creation even if dirty working directory - - Returns: - Branch name - - Raises: - RepositoryError: If working directory is dirty (unless force=True) - """ - # Check working directory - if not force: - is_clean, status_msg = self.check_clean_working_directory() - if not is_clean: - raise RepositoryError( - "Working directory has uncommitted changes.\n\n" - f"{status_msg}\n\n" - "Cannot create documentation branch with uncommitted changes.\n" - "Please commit or stash your changes first:\n" - " git status\n" - " git add -A && git commit -m \"Your message\"\n" - " # or\n" - " git stash\n\n" - "Then re-run: codewiki generate --create-branch" - ) - - # Generate branch name with timestamp - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") - branch_name = f"docs/codewiki-{timestamp}" - - # Check if branch already exists (shouldn't happen with timestamp) - existing_branches = [b.name for b in self.repo.branches] - if branch_name in existing_branches: - # Append counter - counter = 1 - while f"{branch_name}-{counter}" in existing_branches: - counter += 1 - branch_name = f"{branch_name}-{counter}" - - try: - # Create and checkout new branch - new_branch = self.repo.create_head(branch_name) - new_branch.checkout() - return branch_name - except GitCommandError as e: - raise RepositoryError(f"Failed to create branch: {e}") - - def commit_documentation( - self, - docs_path: Path, - message: Optional[str] = None - ) -> str: - """ - Commit generated documentation. - - Args: - docs_path: Path to documentation directory - message: Commit message (optional) - - Returns: - Commit hash - - Raises: - RepositoryError: If commit fails - """ - if message is None: - message = "Add generated documentation\n\nGenerated by CodeWiki CLI" - - try: - # Add documentation files - self.repo.index.add([str(docs_path)]) - - # Commit - commit = self.repo.index.commit(message) - - return commit.hexsha - except GitCommandError as e: - raise RepositoryError(f"Failed to commit documentation: {e}") - - def get_remote_url(self, remote_name: str = "origin") -> Optional[str]: - """ - Get remote repository URL. - - Args: - remote_name: Name of remote (default: origin) - - Returns: - Remote URL or None if no remote - """ - try: - remote = self.repo.remote(remote_name) - return remote.url - except ValueError: - return None - - def get_current_branch(self) -> str: - """ - Get current branch name. - - Returns: - Branch name - """ - try: - return self.repo.active_branch.name - except TypeError: - # Detached HEAD - return "HEAD" - - def get_commit_hash(self) -> str: - """ - Get current commit hash. - - Returns: - Commit hash - """ - return self.repo.head.commit.hexsha - - def branch_exists(self, branch_name: str) -> bool: - """ - Check if a branch exists. - - Args: - branch_name: Branch name to check - - Returns: - True if exists, False otherwise - """ - return branch_name in [b.name for b in self.repo.branches] - - def get_github_pr_url(self, branch_name: str) -> Optional[str]: - """ - Get GitHub PR creation URL for a branch. - - Args: - branch_name: Branch name - - Returns: - PR URL or None if not a GitHub repo - """ - remote_url = self.get_remote_url() - if not remote_url or "github.com" not in remote_url: - return None - - # Clean URL - base_url = remote_url.rstrip('/').replace('.git', '') - - # Convert SSH to HTTPS - if base_url.startswith('git@github.com:'): - base_url = base_url.replace('git@github.com:', '/') - - return f"{base_url}/compare/{branch_name}" - diff --git a/code2test/cli/html_generator.py b/code2test/cli/html_generator.py deleted file mode 100644 index 995edb4..0000000 --- a/code2test/cli/html_generator.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -HTML generator for GitHub Pages documentation viewer. -""" - -import json -from pathlib import Path -from typing import Optional, Dict, Any - -from code2test.cli.utils.errors import FileSystemError -from code2test.cli.utils.fs import safe_write, safe_read - - -class HTMLGenerator: - """ - Generates static HTML documentation viewer for GitHub Pages. - - Creates a self-contained index.html with embedded styles, scripts, - and configuration for client-side markdown rendering. - """ - - def __init__(self, template_dir: Optional[Path] = None): - """ - Initialize HTML generator. - - Args: - template_dir: Path to template directory (default: package templates) - """ - if template_dir is None: - # Use package templates - template_dir = Path(__file__).parent.parent / "templates" / "github_pages" - - self.template_dir = Path(template_dir) - - - def load_module_tree(self, docs_dir: Path) -> Dict[str, Any]: - """ - Load module tree from documentation directory. - - Args: - docs_dir: Documentation directory path - - Returns: - Module tree structure - """ - module_tree_path = docs_dir / "module_tree.json" - if not module_tree_path.exists(): - # Fallback to a simple structure - return { - "Overview": { - "description": "Repository overview", - "components": [], - "children": {} - } - } - - try: - content = safe_read(module_tree_path) - return json.loads(content) - except Exception as e: - raise FileSystemError(f"Failed to load module tree: {e}") - - def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]: - """ - Load metadata from documentation directory. - - Args: - docs_dir: Documentation directory path - - Returns: - Metadata dictionary or None if not found - """ - metadata_path = docs_dir / "metadata.json" - if not metadata_path.exists(): - return None - - try: - content = safe_read(metadata_path) - return json.loads(content) - except Exception: - # Non-critical, return None - return None - - def generate( - self, - output_path: Path, - title: str, - module_tree: Optional[Dict[str, Any]] = None, - repository_url: Optional[str] = None, - github_pages_url: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - docs_dir: Optional[Path] = None, - metadata: Optional[Dict[str, Any]] = None - ): - """ - Generate HTML documentation viewer. - - Args: - output_path: Output file path (index.html) - title: Documentation title - module_tree: Module tree structure (auto-loaded from docs_dir if not provided) - repository_url: GitHub repository URL - github_pages_url: Expected GitHub Pages URL - config: Additional configuration - docs_dir: Documentation directory (for auto-loading module_tree and metadata) - metadata: Metadata dictionary (auto-loaded from docs_dir if not provided) - """ - # Auto-load module_tree and metadata from docs_dir if not provided - if docs_dir: - if module_tree is None: - module_tree = self.load_module_tree(docs_dir) - if metadata is None: - metadata = self.load_metadata(docs_dir) - - # Default values - if module_tree is None: - module_tree = {} - if config is None: - config = {} - - # Load template - template_path = self.template_dir / "viewer_template.html" - if not template_path.exists(): - raise FileSystemError(f"Template not found: {template_path}") - - template_content = safe_read(template_path) - - # Build info content HTML - info_content = self._build_info_content(metadata) - show_info = "block" if info_content else "none" - - # Build repository link - repo_link = "" - if repository_url: - repo_link = f'🔗 View Repository' - - # Determine docs base path - # For GitHub Pages: relative path to docs folder - # For local: relative path to docs folder - docs_base_path = "" - if docs_dir and output_path.parent != docs_dir: - # Calculate relative path from output to docs - try: - docs_base_path = Path(docs_dir.name).as_posix() - except Exception: - docs_base_path = "." - - # Prepare JSON data for embedding - config_json = json.dumps(config, indent=2) - module_tree_json = json.dumps(module_tree, indent=2) - metadata_json = json.dumps(metadata, indent=2) if metadata else "null" - - # Replace placeholders - html_content = template_content - replacements = { - "{{TITLE}}": self._escape_html(title), - "{{REPO_LINK}}": repo_link, - "{{SHOW_INFO}}": show_info, - "{{INFO_CONTENT}}": info_content, - "{{CONFIG_JSON}}": config_json, - "{{MODULE_TREE_JSON}}": module_tree_json, - "{{METADATA_JSON}}": metadata_json, - "{{DOCS_BASE_PATH}}": docs_base_path, - } - - for placeholder, value in replacements.items(): - html_content = html_content.replace(placeholder, value) - - # Write output - output_path = Path(output_path) - output_path.parent.mkdir(parents=True, exist_ok=True) - safe_write(output_path, html_content) - - def _build_info_content(self, metadata: Optional[Dict[str, Any]]) -> str: - """ - Build HTML content for repo info section. - - Args: - metadata: Metadata dictionary - - Returns: - HTML string for info content - """ - if not metadata or not metadata.get('generation_info'): - return "" - - info = metadata.get('generation_info', {}) - stats = metadata.get('statistics', {}) - - html_parts = [] - - if info.get('main_model'): - html_parts.append(f'
Model: {self._escape_html(info["main_model"])}
') - - if info.get('timestamp'): - try: - from datetime import datetime - timestamp = info['timestamp'] - # Parse ISO format timestamp - if isinstance(timestamp, str): - dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - formatted_date = dt.strftime('%Y-%m-%d') - html_parts.append(f'
Generated: {formatted_date}
') - except Exception: - pass - - if info.get('commit_id'): - commit_short = info['commit_id'][:8] - html_parts.append(f'
Commit: {commit_short}
') - - if stats.get('total_components'): - components_str = f"{stats['total_components']:,}" - html_parts.append(f'
Components: {components_str}
') - - if stats.get('max_depth'): - html_parts.append(f'
Max Depth: {stats["max_depth"]}
') - - return '\n '.join(html_parts) - - def _escape_html(self, text: str) -> str: - """ - Escape HTML special characters. - - Args: - text: Text to escape - - Returns: - Escaped text - """ - return (text - .replace('&', '&') - .replace('<', '<') - .replace('>', '>') - .replace('"', '"') - .replace("'", ''')) - - - - def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]: - """ - Detect repository information from git. - - Args: - repo_path: Repository path - - Returns: - Dictionary with 'name', 'url', 'github_pages_url' - """ - info = { - 'name': repo_path.name, - 'url': None, - 'github_pages_url': None, - } - - try: - import git - repo = git.Repo(repo_path) - - # Get repository name - info['name'] = repo_path.name - - # Get remote URL - if repo.remotes: - remote_url = repo.remotes.origin.url - - # Clean URL - if remote_url.startswith('git@github.com:'): - remote_url = remote_url.replace('git@github.com:', '/') - - remote_url = remote_url.rstrip('/').replace('.git', '') - info['url'] = remote_url - - # Compute GitHub Pages URL - if 'github.com' in remote_url: - parts = remote_url.split('/') - if len(parts) >= 2: - owner = parts[-2] - repo = parts[-1] - info['github_pages_url'] = f"https://{owner}.github.io/{repo}/" - - except Exception: - pass - - return info - diff --git a/code2test/cli/main.py b/code2test/cli/main.py index b410e6b..3ff1d99 100644 --- a/code2test/cli/main.py +++ b/code2test/cli/main.py @@ -14,10 +14,11 @@ @click.pass_context def cli(ctx): """ - Code2Test: Intelligent Test Generation for Legacy Codebases. - - Generate comprehensive test suites using intent-first AI analysis. - Supports Python, Java, JavaScript, TypeScript, C, C++, and C#. + Code2Test: intent-first test generation for legacy Python code. + + Infers what each function and class is meant to do, generates pytest + tests against that intent, runs and repairs them, and writes only tests + that pass or are marked as expected failures (suspected bugs). \b Quick Start: @@ -34,13 +35,12 @@ def cli(ctx): def version(): """Display version information.""" click.echo(f"Code2Test v{__version__}") - click.echo("Intent-first test generation for legacy codebases") - click.echo("Built on FSoft AI4Code's CodeWiki framework") + click.echo("Intent-first test generation for legacy Python code") + click.echo("Dependency analysis derived from FSoft AI4Code's CodeWiki") # Import commands from code2test.cli.commands.config import config_group -from code2test.cli.commands.generate import generate_command from code2test.cli.commands.test import test_command from code2test.cli.commands.verify import verify_command from code2test.cli.commands.intent import intent_command @@ -48,11 +48,8 @@ def version(): from code2test.cli.commands.init import init_command from code2test.cli.commands.benchmark import benchmark_command -# Register command groups -cli.add_command(config_group) -cli.add_command(generate_command, name="generate") # Legacy docs command - -# Register new test generation commands +# Register commands +cli.add_command(config_group) # code2test config cli.add_command(test_command) # code2test test cli.add_command(verify_command) # code2test verify cli.add_command(intent_command) # code2test intent diff --git a/code2test/cli/models/config.py b/code2test/cli/models/config.py index 7260176..60b27b7 100644 --- a/code2test/cli/models/config.py +++ b/code2test/cli/models/config.py @@ -1,242 +1,40 @@ """ -Configuration data models for CodeWiki CLI. +Persistent user settings for the Code2Test CLI. -This module contains the Configuration class which represents persistent -user settings stored in ~/.codewiki/config.json. These settings are converted -to the backend Config class when running documentation generation. +Stored in ~/.codewiki/config.json (the location is kept from CodeWiki, which +Code2Test grew out of, so existing configurations keep working). The API key +itself is kept in the system keychain, not in this file. """ -from dataclasses import dataclass, asdict, field -from typing import Optional, List -from pathlib import Path +from dataclasses import dataclass -from code2test.cli.utils.validation import ( - validate_url, - validate_api_key, - validate_model_name, -) - - -@dataclass -class AgentInstructions: - """ - Custom instructions for the documentation agent. - - Allows users to customize: - - File filtering (include/exclude patterns) - - Module focus (prioritize certain modules) - - Documentation type (API docs, architecture docs, etc.) - - Custom instructions for the LLM - - Attributes: - include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) - exclude_patterns: File/directory patterns to exclude (e.g., ["*Tests*", "*test*"]) - focus_modules: Modules to document in more detail - doc_type: Type of documentation to generate - custom_instructions: Additional instructions for the documentation agent - """ - include_patterns: Optional[List[str]] = None # e.g., ["*.cs"] for C# projects - exclude_patterns: Optional[List[str]] = None # e.g., ["*Tests*", "*Specs*"] - focus_modules: Optional[List[str]] = None # e.g., ["src/core", "src/api"] - doc_type: Optional[str] = None # e.g., "api", "architecture", "user-guide" - custom_instructions: Optional[str] = None # Free-form instructions - - def to_dict(self) -> dict: - """Convert to dictionary, excluding None values.""" - result = {} - if self.include_patterns: - result['include_patterns'] = self.include_patterns - if self.exclude_patterns: - result['exclude_patterns'] = self.exclude_patterns - if self.focus_modules: - result['focus_modules'] = self.focus_modules - if self.doc_type: - result['doc_type'] = self.doc_type - if self.custom_instructions: - result['custom_instructions'] = self.custom_instructions - return result - - @classmethod - def from_dict(cls, data: dict) -> 'AgentInstructions': - """Create AgentInstructions from dictionary.""" - return cls( - include_patterns=data.get('include_patterns'), - exclude_patterns=data.get('exclude_patterns'), - focus_modules=data.get('focus_modules'), - doc_type=data.get('doc_type'), - custom_instructions=data.get('custom_instructions'), - ) - - def is_empty(self) -> bool: - """Check if all fields are empty/None.""" - return not any([ - self.include_patterns, - self.exclude_patterns, - self.focus_modules, - self.doc_type, - self.custom_instructions, - ]) - - def get_prompt_addition(self) -> str: - """Generate prompt additions based on instructions.""" - additions = [] - - if self.doc_type: - doc_type_instructions = { - 'api': "Focus on API documentation: endpoints, parameters, return types, and usage examples.", - 'architecture': "Focus on architecture documentation: system design, component relationships, and data flow.", - 'user-guide': "Focus on user guide documentation: how to use features, step-by-step tutorials.", - 'developer': "Focus on developer documentation: code structure, contribution guidelines, and implementation details.", - } - if self.doc_type.lower() in doc_type_instructions: - additions.append(doc_type_instructions[self.doc_type.lower()]) - else: - additions.append(f"Focus on generating {self.doc_type} documentation.") - - if self.focus_modules: - additions.append(f"Pay special attention to and provide more detailed documentation for these modules: {', '.join(self.focus_modules)}") - - if self.custom_instructions: - additions.append(f"Additional instructions: {self.custom_instructions}") - - return "\n".join(additions) if additions else "" +from code2test.cli.utils.validation import validate_model_name @dataclass class Configuration: """ - CodeWiki configuration data model. - + Code2Test configuration. + Attributes: - base_url: LLM API base URL - main_model: Primary model for documentation generation - cluster_model: Model for module clustering - fallback_model: Fallback model for documentation generation - default_output: Default output directory - max_tokens: Maximum tokens for LLM response (default: 32768) - max_token_per_module: Maximum tokens per module for clustering (default: 36369) - max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000) - max_depth: Maximum depth for hierarchical decomposition (default: 2) - agent_instructions: Custom agent instructions for documentation generation + main_model: pydantic-ai model string used for test generation, + e.g. "anthropic:claude-sonnet-5-5"; empty when not set """ - base_url: str - main_model: str - cluster_model: str - fallback_model: str = "glm-4p5" - default_output: str = "docs" - max_tokens: int = 32768 - max_token_per_module: int = 36369 - max_token_per_leaf_module: int = 16000 - max_depth: int = 2 - agent_instructions: AgentInstructions = field(default_factory=AgentInstructions) - + main_model: str = "" + def validate(self): - """ - Validate all configuration fields. - - Raises: - ConfigurationError: If validation fails - """ - validate_url(self.base_url) - validate_model_name(self.main_model) - validate_model_name(self.cluster_model) - validate_model_name(self.fallback_model) - + """Raise ConfigurationError if a set field is invalid.""" + if self.main_model: + validate_model_name(self.main_model) + def to_dict(self) -> dict: - """Convert to dictionary.""" - result = { - 'base_url': self.base_url, - 'main_model': self.main_model, - 'cluster_model': self.cluster_model, - 'default_output': self.default_output, - 'max_tokens': self.max_tokens, - 'max_token_per_module': self.max_token_per_module, - 'max_token_per_leaf_module': self.max_token_per_leaf_module, - 'max_depth': self.max_depth, - } - if self.agent_instructions and not self.agent_instructions.is_empty(): - result['agent_instructions'] = self.agent_instructions.to_dict() - return result - + return {"main_model": self.main_model} + @classmethod - def from_dict(cls, data: dict) -> 'Configuration': - """ - Create Configuration from dictionary. - - Args: - data: Configuration dictionary - - Returns: - Configuration instance - """ - agent_instructions = AgentInstructions() - if 'agent_instructions' in data: - agent_instructions = AgentInstructions.from_dict(data['agent_instructions']) - - return cls( - base_url=data.get('base_url', ''), - main_model=data.get('main_model', ''), - cluster_model=data.get('cluster_model', ''), - fallback_model=data.get('fallback_model', 'glm-4p5'), - default_output=data.get('default_output', 'docs'), - max_tokens=data.get('max_tokens', 32768), - max_token_per_module=data.get('max_token_per_module', 36369), - max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000), - max_depth=data.get('max_depth', 2), - agent_instructions=agent_instructions, - ) - - def is_complete(self) -> bool: - """Check if all required fields are set.""" - return bool( - self.base_url and - self.main_model and - self.cluster_model and - self.fallback_model - ) - - def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runtime_instructions: AgentInstructions = None): - """ - Convert CLI Configuration to Backend Config. - - This method bridges the gap between persistent user settings (CLI Configuration) - and runtime job configuration (Backend Config). - - Args: - repo_path: Path to the repository to document - output_dir: Output directory for generated documentation - api_key: LLM API key (from keyring) - runtime_instructions: Runtime agent instructions (override persistent settings) - - Returns: - Backend Config instance ready for documentation generation - """ - from code2test.src.config import Config - - # Merge runtime instructions with persistent settings - # Runtime instructions take precedence - final_instructions = self.agent_instructions - if runtime_instructions and not runtime_instructions.is_empty(): - final_instructions = AgentInstructions( - include_patterns=runtime_instructions.include_patterns or self.agent_instructions.include_patterns, - exclude_patterns=runtime_instructions.exclude_patterns or self.agent_instructions.exclude_patterns, - focus_modules=runtime_instructions.focus_modules or self.agent_instructions.focus_modules, - doc_type=runtime_instructions.doc_type or self.agent_instructions.doc_type, - custom_instructions=runtime_instructions.custom_instructions or self.agent_instructions.custom_instructions, - ) - - return Config.from_cli( - repo_path=repo_path, - output_dir=output_dir, - llm_base_url=self.base_url, - llm_api_key=api_key, - main_model=self.main_model, - cluster_model=self.cluster_model, - fallback_model=self.fallback_model, - max_tokens=self.max_tokens, - max_token_per_module=self.max_token_per_module, - max_token_per_leaf_module=self.max_token_per_leaf_module, - max_depth=self.max_depth, - agent_instructions=final_instructions.to_dict() if final_instructions else None - ) + def from_dict(cls, data: dict) -> "Configuration": + # Files written by older versions hold more settings; only main_model is used + return cls(main_model=data.get("main_model", "") or "") + def is_complete(self) -> bool: + """True when a model is set.""" + return bool(self.main_model) diff --git a/code2test/cli/models/job.py b/code2test/cli/models/job.py deleted file mode 100644 index c0c49d1..0000000 --- a/code2test/cli/models/job.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Documentation job data models. -""" - -from dataclasses import dataclass, field, asdict -from datetime import datetime -from typing import List, Optional, Dict, Any -from enum import Enum -import uuid -import json - - -class JobStatus(str, Enum): - """Documentation job status.""" - PENDING = "pending" - RUNNING = "running" - COMPLETED = "completed" - FAILED = "failed" - - -@dataclass -class GenerationOptions: - """Options for documentation generation.""" - create_branch: bool = False - github_pages: bool = False - no_cache: bool = False - custom_output: Optional[str] = None - - -@dataclass -class JobStatistics: - """Statistics for a documentation job.""" - total_files_analyzed: int = 0 - leaf_nodes: int = 0 - max_depth: int = 0 - total_tokens_used: int = 0 - - -@dataclass -class LLMConfig: - """LLM configuration for a job.""" - main_model: str - cluster_model: str - base_url: str - - -@dataclass -class DocumentationJob: - """ - Represents a documentation generation job. - - Attributes: - job_id: Unique job identifier - repository_path: Absolute path to repository - repository_name: Repository name - output_directory: Output directory path - commit_hash: Git commit SHA - branch_name: Git branch name (if applicable) - timestamp_start: Job start time - timestamp_end: Job end time (if completed) - status: Current job status - error_message: Error message (if failed) - files_generated: List of generated files - module_count: Number of modules documented - generation_options: Generation options used - llm_config: LLM configuration used - statistics: Job statistics - """ - job_id: str = field(default_factory=lambda: str(uuid.uuid4())) - repository_path: str = "" - repository_name: str = "" - output_directory: str = "" - commit_hash: str = "" - branch_name: Optional[str] = None - timestamp_start: str = field(default_factory=lambda: datetime.now().isoformat()) - timestamp_end: Optional[str] = None - status: JobStatus = JobStatus.PENDING - error_message: Optional[str] = None - files_generated: List[str] = field(default_factory=list) - module_count: int = 0 - generation_options: GenerationOptions = field(default_factory=GenerationOptions) - llm_config: Optional[LLMConfig] = None - statistics: JobStatistics = field(default_factory=JobStatistics) - - def start(self): - """Mark job as started.""" - self.status = JobStatus.RUNNING - self.timestamp_start = datetime.now().isoformat() - - def complete(self): - """Mark job as completed.""" - self.status = JobStatus.COMPLETED - self.timestamp_end = datetime.now().isoformat() - - def fail(self, error_message: str): - """Mark job as failed.""" - self.status = JobStatus.FAILED - self.error_message = error_message - self.timestamp_end = datetime.now().isoformat() - - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary for JSON serialization.""" - data = { - "job_id": self.job_id, - "repository_path": self.repository_path, - "repository_name": self.repository_name, - "output_directory": self.output_directory, - "commit_hash": self.commit_hash, - "branch_name": self.branch_name, - "timestamp_start": self.timestamp_start, - "timestamp_end": self.timestamp_end, - "status": self.status.value if isinstance(self.status, JobStatus) else self.status, - "error_message": self.error_message, - "files_generated": self.files_generated, - "module_count": self.module_count, - "generation_options": asdict(self.generation_options), - "llm_config": asdict(self.llm_config) if self.llm_config else None, - "statistics": asdict(self.statistics), - } - return data - - def to_json(self) -> str: - """Convert to JSON string.""" - return json.dumps(self.to_dict(), indent=2) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'DocumentationJob': - """Create from dictionary.""" - job = cls( - job_id=data.get('job_id', str(uuid.uuid4())), - repository_path=data.get('repository_path', ''), - repository_name=data.get('repository_name', ''), - output_directory=data.get('output_directory', ''), - commit_hash=data.get('commit_hash', ''), - branch_name=data.get('branch_name'), - timestamp_start=data.get('timestamp_start', datetime.now().isoformat()), - timestamp_end=data.get('timestamp_end'), - status=JobStatus(data.get('status', 'pending')), - error_message=data.get('error_message'), - files_generated=data.get('files_generated', []), - module_count=data.get('module_count', 0), - ) - - # Parse nested objects - if 'generation_options' in data: - opts = data['generation_options'] - job.generation_options = GenerationOptions(**opts) - - if 'llm_config' in data and data['llm_config']: - job.llm_config = LLMConfig(**data['llm_config']) - - if 'statistics' in data: - job.statistics = JobStatistics(**data['statistics']) - - return job - diff --git a/code2test/cli/utils/api_errors.py b/code2test/cli/utils/api_errors.py deleted file mode 100644 index b2ceb2f..0000000 --- a/code2test/cli/utils/api_errors.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -LLM API error handling utilities with fail-fast behavior. -""" - -from typing import Optional -import click - -from code2test.cli.utils.errors import APIError - - -class APIErrorHandler: - """Handler for LLM API errors with fail-fast behavior.""" - - @staticmethod - def handle_api_error( - error: Exception, - context: Optional[str] = None, - fail_fast: bool = True - ) -> APIError: - """ - Handle LLM API error and convert to APIError. - - Args: - error: The original exception - context: Additional context (e.g., module name) - fail_fast: Whether to fail immediately (default: True) - - Returns: - APIError instance - """ - error_message = str(error) - - # Detect specific error types - if "429" in error_message or "rate limit" in error_message.lower(): - message = ( - "LLM API rate limit exceeded.\n\n" - "The API returned a 429 error, indicating too many requests.\n\n" - "Troubleshooting:\n" - " 1. Wait a few minutes before retrying\n" - " 2. Check your API quota at your provider's dashboard\n" - " 3. Consider upgrading your API plan\n" - " 4. For large repositories, generate during off-peak hours" - ) - elif "401" in error_message or "authentication" in error_message.lower(): - message = ( - "LLM API authentication failed.\n\n" - "Your API key appears to be invalid or expired.\n\n" - "Troubleshooting:\n" - " 1. Verify your API key: codewiki config show\n" - " 2. Update your API key: codewiki config set --api-key \n" - " 3. Check that your API key is active in your provider's dashboard" - ) - elif "timeout" in error_message.lower(): - message = ( - "LLM API request timed out.\n\n" - "The API did not respond within the expected time.\n\n" - "Troubleshooting:\n" - " 1. Check your internet connection\n" - " 2. Verify the API service is operational\n" - " 3. Try again in a few moments\n" - " 4. If the issue persists, contact your API provider" - ) - elif "network" in error_message.lower() or "connection" in error_message.lower(): - message = ( - "Network error while connecting to LLM API.\n\n" - "Could not establish connection to the API.\n\n" - "Troubleshooting:\n" - " 1. Check your internet connection\n" - " 2. Verify the base URL: codewiki config show\n" - " 3. Check if you're behind a proxy or firewall\n" - " 4. Try: curl -I to test connectivity" - ) - else: - message = ( - f"LLM API error: {error_message}\n\n" - "An unexpected error occurred while communicating with the LLM API.\n\n" - "Troubleshooting:\n" - " 1. Check your configuration: codewiki config validate\n" - " 2. Verify API service status\n" - " 3. Review the error message above for specific details" - ) - - if context: - message = f"Context: {context}\n\n{message}" - - return APIError(message) - - @staticmethod - def display_api_error(error: APIError, module_name: Optional[str] = None): - """ - Display API error with formatting. - - Args: - error: The API error - module_name: Optional module name for context - """ - click.echo() - click.secho("✗ LLM API Error", fg="red", bold=True) - click.echo() - - if module_name: - click.echo(f"Module: {module_name}") - click.echo() - - click.echo(error.message) - click.echo() - click.secho( - "Documentation generation stopped. No partial results saved.", - fg="yellow" - ) - click.echo() - - -def wrap_api_call(func, *args, fail_fast: bool = True, context: Optional[str] = None, **kwargs): - """ - Wrap an API call with error handling. - - Args: - func: Function to call - *args: Positional arguments - fail_fast: Whether to raise on error (default: True) - context: Optional context for error message - **kwargs: Keyword arguments - - Returns: - Function result - - Raises: - APIError: If API call fails and fail_fast is True - """ - try: - return func(*args, **kwargs) - except Exception as e: - api_error = APIErrorHandler.handle_api_error(e, context=context, fail_fast=fail_fast) - if fail_fast: - raise api_error - else: - APIErrorHandler.display_api_error(api_error) - return None - diff --git a/code2test/cli/utils/instructions.py b/code2test/cli/utils/instructions.py deleted file mode 100644 index 7c2bf05..0000000 --- a/code2test/cli/utils/instructions.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Post-generation instructions generator. -""" - -from pathlib import Path -from typing import Optional -import click - - -def compute_github_pages_url(repo_url: str, repo_name: str) -> str: - """ - Compute expected GitHub Pages URL from repository URL. - - Args: - repo_url: GitHub repository URL - repo_name: Repository name - - Returns: - Expected GitHub Pages URL - """ - # Extract owner from GitHub URL - # e.g., "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/owner/repo" -> "owner" - if "github.com" in repo_url: - parts = repo_url.rstrip('/').split('/') - if len(parts) >= 2: - owner = parts[-2] - repo = parts[-1].replace('.git', '') - return f"https://{owner}.github.io/{repo}/" - - return f"https://YOUR_USERNAME.github.io/{repo_name}/" - - -def get_pr_creation_url(repo_url: str, branch_name: str) -> str: - """ - Get PR creation URL for GitHub. - - Args: - repo_url: GitHub repository URL - branch_name: Branch name - - Returns: - PR creation URL - """ - base_url = repo_url.rstrip('/').replace('.git', '') - return f"{base_url}/compare/{branch_name}" - - -def display_post_generation_instructions( - output_dir: Path, - repo_name: str, - repo_url: Optional[str] = None, - branch_name: Optional[str] = None, - github_pages: bool = False, - files_generated: list = None, - statistics: dict = None -): - """ - Display post-generation instructions. - - Args: - output_dir: Output directory path - repo_name: Repository name - repo_url: GitHub repository URL (optional) - branch_name: Git branch name (optional) - github_pages: Whether GitHub Pages HTML was generated - files_generated: List of generated files - statistics: Generation statistics - """ - click.echo() - click.secho("✓ Documentation generated successfully!", fg="green", bold=True) - click.echo() - - # Output directory - click.secho("Output directory:", fg="cyan", bold=True) - click.echo(f" {output_dir}") - click.echo() - - # Generated files - if files_generated: - click.secho("Generated files:", fg="cyan", bold=True) - for file in files_generated[:10]: # Show first 10 - click.echo(f" - {file}") - if len(files_generated) > 10: - click.echo(f" ... and {len(files_generated) - 10} more") - click.echo() - - # Statistics - if statistics: - click.secho("Statistics:", fg="cyan", bold=True) - if 'module_count' in statistics: - click.echo(f" Total modules: {statistics['module_count']}") - if 'total_files_analyzed' in statistics: - click.echo(f" Files analyzed: {statistics['total_files_analyzed']}") - if 'generation_time' in statistics: - minutes = int(statistics['generation_time'] // 60) - seconds = int(statistics['generation_time'] % 60) - click.echo(f" Generation time: {minutes} minutes {seconds} seconds") - # if 'total_tokens_used' in statistics: - # tokens = statistics['total_tokens_used'] - # click.echo(f" Tokens used: ~{tokens:,}") - click.echo() - - # Next steps - click.secho("Next steps:", fg="cyan", bold=True) - click.echo() - - click.echo("1. Review the generated documentation:") - click.echo(f" cat {output_dir}/overview.md") - if github_pages: - click.echo(f" open {output_dir}/index.html # View in browser") - click.echo() - - if branch_name: - # Git workflow with branch - click.echo("2. Push the documentation branch:") - click.secho(f" git push origin {branch_name}", fg="yellow") - click.echo() - - if repo_url: - pr_url = get_pr_creation_url(repo_url, branch_name) - click.echo("3. Create a Pull Request to merge documentation:") - click.secho(f" {pr_url}", fg="blue") - click.echo() - - click.echo("4. After merge, enable GitHub Pages:") - else: - click.echo("3. Enable GitHub Pages:") - else: - # Direct commit workflow - click.echo("2. Commit the documentation:") - click.secho(" git add docs/", fg="yellow") - click.secho(' git commit -m "Add generated documentation"', fg="yellow") - click.echo() - - click.echo("3. Push to GitHub:") - click.secho(" git push origin main", fg="yellow") - click.echo() - - click.echo("4. Enable GitHub Pages:") - - click.echo(" - Go to repository Settings → Pages") - click.echo(" - Source: Deploy from a branch") - click.echo(" - Branch: main, folder: /docs") - click.echo() - - if repo_url: - github_pages_url = compute_github_pages_url(repo_url, repo_name) - click.echo("5. Your documentation will be available at:") - click.secho(f" {github_pages_url}", fg="blue", bold=True) - click.echo() - - -def display_generation_summary( - success: bool, - error_message: Optional[str] = None, - output_dir: Optional[Path] = None -): - """ - Display generation summary (success or failure). - - Args: - success: Whether generation was successful - error_message: Error message if failed - output_dir: Output directory if successful - """ - if success: - click.echo() - click.secho("✓ Generation completed successfully!", fg="green", bold=True) - if output_dir: - click.echo(f"\nDocumentation saved to: {output_dir}") - click.echo() - else: - click.echo() - click.secho("✗ Generation failed", fg="red", bold=True) - if error_message: - click.echo() - click.echo(error_message) - click.echo() - diff --git a/code2test/cli/utils/logging.py b/code2test/cli/utils/logging.py deleted file mode 100644 index 8086340..0000000 --- a/code2test/cli/utils/logging.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Logging utilities for CLI with colored output and progress tracking. -""" - -import sys -from datetime import datetime -from typing import Optional -import click - - -class CLILogger: - """Logger for CLI with support for verbose and normal modes.""" - - def __init__(self, verbose: bool = False): - """ - Initialize the logger. - - Args: - verbose: Enable verbose output - """ - self.verbose = verbose - self.start_time = datetime.now() - - def debug(self, message: str): - """Log debug message (only in verbose mode).""" - if self.verbose: - timestamp = datetime.now().strftime("%H:%M:%S") - click.secho(f"[{timestamp}] {message}", fg="cyan", dim=True) - - def info(self, message: str): - """Log info message.""" - click.echo(message) - - def success(self, message: str): - """Log success message in green.""" - click.secho(f"✓ {message}", fg="green") - - def warning(self, message: str): - """Log warning message in yellow.""" - click.secho(f"⚠️ {message}", fg="yellow") - - def error(self, message: str): - """Log error message in red.""" - click.secho(f"✗ {message}", fg="red", err=True) - - def step(self, message: str, step: Optional[int] = None, total: Optional[int] = None): - """ - Log a processing step. - - Args: - message: Step description - step: Current step number - total: Total number of steps - """ - if step is not None and total is not None: - prefix = f"[{step}/{total}]" - else: - prefix = "→" - - click.secho(f"{prefix} {message}", fg="blue", bold=True) - - def elapsed_time(self) -> str: - """Get elapsed time since logger was created.""" - elapsed = datetime.now() - self.start_time - minutes = int(elapsed.total_seconds() // 60) - seconds = int(elapsed.total_seconds() % 60) - - if minutes > 0: - return f"{minutes}m {seconds}s" - else: - return f"{seconds}s" - - -def create_logger(verbose: bool = False) -> CLILogger: - """ - Create and return a CLI logger. - - Args: - verbose: Enable verbose output - - Returns: - Configured CLILogger instance - """ - return CLILogger(verbose=verbose) - diff --git a/code2test/cli/utils/model.py b/code2test/cli/utils/model.py index 29737ab..038e157 100644 --- a/code2test/cli/utils/model.py +++ b/code2test/cli/utils/model.py @@ -16,10 +16,7 @@ def resolve_model(display) -> str: """ # Load configuration config_manager = ConfigManager() - configured = config_manager.load() and config_manager.is_configured() - if not configured and not os.environ.get("CODE2TEST_MODEL"): - display.warning("Code2Test is not fully configured.") - display.info("Usage may fail if API keys are missing. Run 'code2test config set' to configure.") + config_manager.load() # Set API key in environment for agents api_key = config_manager.get_api_key() @@ -69,5 +66,7 @@ def resolve_model(display) -> str: else: main_model = "openai:gpt-4o-mini" + display.info("No model configured; using openai:gpt-4o-mini. " + "Set CODE2TEST_MODEL or run 'code2test config set --main-model' to choose another.") return main_model diff --git a/code2test/cli/utils/progress.py b/code2test/cli/utils/progress.py deleted file mode 100644 index f61efc1..0000000 --- a/code2test/cli/utils/progress.py +++ /dev/null @@ -1,222 +0,0 @@ -""" -Progress indicator utilities for CLI. -""" - -import time -from typing import Optional, Callable -from datetime import datetime -import click - - -class ProgressTracker: - """ - Progress tracker with stages and ETA estimation. - - Stages: - 1. Dependency Analysis (40% of time) - 2. Module Clustering (20% of time) - 3. Documentation Generation (30% of time) - 4. HTML Generation (5% of time, optional) - 5. Finalization (5% of time) - """ - - # Stage weights (percentage of total time) - STAGE_WEIGHTS = { - 1: 0.40, # Dependency Analysis - 2: 0.20, # Module Clustering - 3: 0.30, # Documentation Generation - 4: 0.05, # HTML Generation (optional) - 5: 0.05, # Finalization - } - - STAGE_NAMES = { - 1: "Dependency Analysis", - 2: "Module Clustering", - 3: "Documentation Generation", - 4: "HTML Generation", - 5: "Finalization", - } - - def __init__(self, total_stages: int = 5, verbose: bool = False): - """ - Initialize progress tracker. - - Args: - total_stages: Number of stages - verbose: Enable verbose output - """ - self.total_stages = total_stages - self.current_stage = 0 - self.stage_progress = 0.0 - self.start_time = time.time() - self.verbose = verbose - self.current_stage_start = self.start_time - - def start_stage(self, stage: int, description: Optional[str] = None): - """ - Start a new stage. - - Args: - stage: Stage number (1-5) - description: Optional custom description - """ - self.current_stage = stage - self.stage_progress = 0.0 - self.current_stage_start = time.time() - - stage_name = description or self.STAGE_NAMES.get(stage, f"Stage {stage}") - - if self.verbose: - elapsed = self._format_elapsed() - click.secho( - f"\n[{elapsed}] Phase {stage}/{self.total_stages}: {stage_name}", - fg="blue", - bold=True - ) - else: - click.secho( - f"[{stage}/{self.total_stages}] {stage_name}", - fg="blue", - bold=True - ) - - def update_stage(self, progress: float, message: Optional[str] = None): - """ - Update progress within current stage. - - Args: - progress: Progress percentage (0.0 to 1.0) - message: Optional progress message - """ - self.stage_progress = min(1.0, max(0.0, progress)) - - if self.verbose and message: - elapsed = self._format_elapsed() - click.echo(f"[{elapsed}] {message}") - - def complete_stage(self, message: Optional[str] = None): - """ - Complete current stage. - - Args: - message: Optional completion message - """ - self.stage_progress = 1.0 - - if self.verbose: - elapsed = self._format_elapsed() - stage_time = time.time() - self.current_stage_start - stage_name = self.STAGE_NAMES.get(self.current_stage, f"Stage {self.current_stage}") - click.secho( - f"[{elapsed}] {stage_name} complete ({stage_time:.1f}s)", - fg="green" - ) - if message: - click.echo(f"[{elapsed}] {message}") - - def get_overall_progress(self) -> float: - """ - Get overall progress percentage. - - Returns: - Progress (0.0 to 1.0) - """ - completed_weight = sum( - self.STAGE_WEIGHTS.get(s, 0) - for s in range(1, self.current_stage) - ) - - current_weight = self.STAGE_WEIGHTS.get(self.current_stage, 0) * self.stage_progress - - return completed_weight + current_weight - - def _format_elapsed(self) -> str: - """Format elapsed time.""" - elapsed = time.time() - self.start_time - minutes = int(elapsed // 60) - seconds = int(elapsed % 60) - - if minutes > 0: - return f"{minutes:02d}:{seconds:02d}" - else: - return f"00:{seconds:02d}" - - def get_eta(self) -> Optional[str]: - """ - Estimate time remaining. - - Returns: - ETA string or None if cannot estimate - """ - elapsed = time.time() - self.start_time - progress = self.get_overall_progress() - - if progress <= 0.0: - return None - - total_estimated = elapsed / progress - remaining = total_estimated - elapsed - - if remaining < 0: - return "< 1 min" - - minutes = int(remaining // 60) - seconds = int(remaining % 60) - - if minutes > 60: - hours = minutes // 60 - minutes = minutes % 60 - return f"{hours}h {minutes}m" - elif minutes > 0: - return f"{minutes}m {seconds}s" - else: - return f"{seconds}s" - - -class ModuleProgressBar: - """Progress bar for module-by-module generation.""" - - def __init__(self, total_modules: int, verbose: bool = False): - """ - Initialize module progress bar. - - Args: - total_modules: Total number of modules to process - verbose: Enable verbose output - """ - self.total_modules = total_modules - self.current_module = 0 - self.verbose = verbose - self.bar = None - - if not verbose: - self.bar = click.progressbar( - length=total_modules, - label="Generating modules", - show_eta=True, - show_percent=True, - ) - self.bar.__enter__() - - def update(self, module_name: str, cached: bool = False): - """ - Update progress for a module. - - Args: - module_name: Name of the module - cached: Whether the module was loaded from cache - """ - self.current_module += 1 - - if self.verbose: - status = "✓ (cached)" if cached else "⟳ (generating)" - click.echo(f" [{self.current_module}/{self.total_modules}] {module_name}... {status}") - elif self.bar: - self.bar.update(1) - - def finish(self): - """Finish progress bar.""" - if self.bar: - self.bar.__exit__(None, None, None) - self.bar = None - diff --git a/code2test/cli/utils/repo_validator.py b/code2test/cli/utils/repo_validator.py deleted file mode 100644 index fce6827..0000000 --- a/code2test/cli/utils/repo_validator.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Repository validation utilities for documentation generation. -""" - -from pathlib import Path -from typing import Tuple, List -import os - -from code2test.cli.utils.errors import RepositoryError -from code2test.cli.utils.validation import validate_repository_path, detect_supported_languages - - -# Supported file extensions by language -SUPPORTED_EXTENSIONS = { - '.py', # Python - '.java', # Java - '.js', # JavaScript - '.jsx', # JavaScript (React) - '.ts', # TypeScript - '.tsx', # TypeScript (React) - '.c', # C - '.h', # C headers - '.cpp', # C++ - '.hpp', # C++ headers - '.cc', # C++ - '.hh', # C++ headers - '.cxx', # C++ - '.hxx', # C++ headers - '.cs', # C# - '.php', # PHP - '.phtml', # PHP templates - '.inc', # PHP includes -} - - -def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: - """ - Validate repository for documentation generation. - - Checks: - - Path exists and is a directory - - Contains supported code files - - Has sufficient files for meaningful documentation - - Args: - repo_path: Path to repository - - Returns: - Tuple of (validated_path, language_counts) - - Raises: - RepositoryError: If validation fails - """ - # Validate path exists - repo_path = validate_repository_path(repo_path) - - # Detect languages - languages = detect_supported_languages(repo_path) - - if not languages: - raise RepositoryError( - f"No supported code files found in {repo_path}\n\n" - "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#, PHP\n\n" - "Please navigate to a code repository or specify a custom directory:\n" - " cd /path/to/your/project\n" - " codewiki generate" - ) - - return repo_path, languages - - -def check_writable_output(output_dir: Path) -> Path: - """ - Check if output directory is writable. - - Args: - output_dir: Output directory path - - Returns: - Validated output directory path - - Raises: - RepositoryError: If output directory is not writable - """ - output_dir = Path(output_dir).expanduser().resolve() - - # Check if output directory exists - if output_dir.exists(): - if not output_dir.is_dir(): - raise RepositoryError( - f"Output path exists but is not a directory: {output_dir}" - ) - - # Check if writable - if not os.access(output_dir, os.W_OK): - raise RepositoryError( - f"Output directory is not writable: {output_dir}\n\n" - f"Try: chmod u+w {output_dir}" - ) - else: - # Check if parent is writable - parent = output_dir.parent - if not parent.exists(): - raise RepositoryError( - f"Parent directory does not exist: {parent}" - ) - - if not os.access(parent, os.W_OK): - raise RepositoryError( - f"Cannot create output directory (parent not writable): {parent}\n\n" - f"Try: chmod u+w {parent}" - ) - - return output_dir - - -def is_git_repository(repo_path: Path) -> bool: - """ - Check if path is a git repository. - - Args: - repo_path: Path to check - - Returns: - True if git repository, False otherwise - """ - git_dir = repo_path / ".git" - return git_dir.exists() and git_dir.is_dir() - - -def get_git_commit_hash(repo_path: Path) -> str: - """ - Get current git commit hash. - - Args: - repo_path: Repository path - - Returns: - Commit hash or empty string if not a git repo - """ - if not is_git_repository(repo_path): - return "" - - try: - import git - repo = git.Repo(repo_path) - return repo.head.commit.hexsha - except Exception: - return "" - - -def get_git_branch(repo_path: Path) -> str: - """ - Get current git branch name. - - Args: - repo_path: Repository path - - Returns: - Branch name or empty string if not a git repo - """ - if not is_git_repository(repo_path): - return "" - - try: - import git - repo = git.Repo(repo_path) - return repo.active_branch.name - except Exception: - return "" - - -def count_code_files(repo_path: Path) -> int: - """ - Count supported code files in repository. - - Args: - repo_path: Repository path - - Returns: - Number of code files - """ - count = 0 - for ext in SUPPORTED_EXTENSIONS: - count += len(list(repo_path.rglob(f"*{ext}"))) - return count - diff --git a/code2test/cli/utils/validation.py b/code2test/cli/utils/validation.py index 821d25f..8339f78 100644 --- a/code2test/cli/utils/validation.py +++ b/code2test/cli/utils/validation.py @@ -1,55 +1,8 @@ """ -Validation utilities for CLI inputs and configuration. +Input validation for CLI configuration values. """ -import re -from pathlib import Path -from typing import Optional, List, Tuple -from urllib.parse import urlparse - -from code2test.cli.utils.errors import ConfigurationError, RepositoryError - - -def validate_url(url: str, require_https: bool = False, allow_localhost: bool = True) -> str: - """ - Validate URL format. - - Args: - url: URL to validate - require_https: Require HTTPS scheme (except localhost) - allow_localhost: Allow localhost URLs - - Returns: - Validated URL - - Raises: - ConfigurationError: If URL is invalid - """ - try: - parsed = urlparse(url) - - # Check scheme - if not parsed.scheme: - raise ConfigurationError(f"Invalid URL (missing scheme): {url}") - - # Check HTTPS requirement - if require_https and parsed.scheme != 'https': - # Allow HTTP for localhost - if allow_localhost and parsed.hostname in ['localhost', '127.0.0.1', '::1']: - pass - else: - raise ConfigurationError( - f"URL must use HTTPS: {url}\n" - f"HTTP is only allowed for localhost" - ) - - # Check hostname - if not parsed.hostname: - raise ConfigurationError(f"Invalid URL (missing hostname): {url}") - - return url - except ValueError as e: - raise ConfigurationError(f"Invalid URL format: {url}\nError: {e}") +from code2test.cli.utils.errors import ConfigurationError def validate_api_key(api_key: str, min_length: int = 10) -> str: @@ -79,6 +32,7 @@ def validate_api_key(api_key: str, min_length: int = 10) -> str: return api_key + def validate_model_name(model: str) -> str: """ Validate model name format. @@ -98,135 +52,6 @@ def validate_model_name(model: str) -> str: return model.strip() -def validate_output_directory(path: str) -> Path: - """ - Validate output directory path. - - Args: - path: Directory path to validate - - Returns: - Validated Path object - - Raises: - ConfigurationError: If path is invalid - """ - if not path or not path.strip(): - raise ConfigurationError("Output directory cannot be empty") - - try: - resolved_path = Path(path).expanduser().resolve() - - # Check if path is writable (or parent is writable if path doesn't exist) - if resolved_path.exists(): - if not resolved_path.is_dir(): - raise ConfigurationError( - f"Output path exists but is not a directory: {path}" - ) - - return resolved_path - except Exception as e: - raise ConfigurationError(f"Invalid output directory path: {path}\nError: {e}") - - -def validate_repository_path(path: Path) -> Path: - """ - Validate repository path exists and contains code files. - - Args: - path: Repository path to validate - - Returns: - Validated Path object - - Raises: - RepositoryError: If repository is invalid - """ - path = Path(path).expanduser().resolve() - - if not path.exists(): - raise RepositoryError(f"Repository path does not exist: {path}") - - if not path.is_dir(): - raise RepositoryError(f"Repository path is not a directory: {path}") - - return path - - -def detect_supported_languages(directory: Path) -> List[Tuple[str, int]]: - """ - Detect supported programming languages in a directory. - - Args: - directory: Directory to scan - - Returns: - List of (language, file_count) tuples - """ - language_extensions = { - 'Python': ['.py'], - 'Java': ['.java'], - 'JavaScript': ['.js', '.jsx'], - 'TypeScript': ['.ts', '.tsx'], - 'C': ['.c', '.h'], - 'C++': ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], - 'C#': ['.cs'], - 'PHP': ['.php', '.phtml', '.inc'], - } - - # Directories to exclude from counting - excluded_dirs = { - 'node_modules', '__pycache__', '.git', 'build', 'dist', - '.venv', 'venv', 'env', '.env', 'target', 'bin', 'obj', - '.pytest_cache', '.mypy_cache', '.tox', 'coverage', - 'htmlcov', '.eggs', '*.egg-info', 'vendor', 'bower_components', - '.idea', '.vscode', '.gradle', '.mvn' - } - - def should_exclude_file(file_path: Path) -> bool: - """Check if file is in an excluded directory.""" - parts = file_path.parts - return any(excluded_dir in parts for excluded_dir in excluded_dirs) - - language_counts = {} - - for language, extensions in language_extensions.items(): - count = 0 - for ext in extensions: - # Filter out files in excluded directories - count += sum( - 1 for f in directory.rglob(f"*{ext}") - if f.is_file() and not should_exclude_file(f) - ) - - if count > 0: - language_counts[language] = count - - # Sort by count descending - return sorted(language_counts.items(), key=lambda x: x[1], reverse=True) - - -def is_top_tier_model(model: str) -> bool: - """ - Check if a model is considered top-tier for clustering. - - Args: - model: Model name - - Returns: - True if top-tier, False otherwise - """ - top_tier_models = [ - 'claude-opus', - 'claude-sonnet', - 'gpt-4', - 'gpt-5', - 'gemini-2.5', - ] - - model_lower = model.lower() - return any(tier in model_lower for tier in top_tier_models) - def mask_api_key(api_key: str, visible_chars: int = 4) -> str: """ diff --git a/code2test/run_web_app.py b/code2test/run_web_app.py deleted file mode 100644 index 940a1d8..0000000 --- a/code2test/run_web_app.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -""" -Startup script for CodeWiki Web Application -""" - -import os -import sys - -# Add src directory to Python path -src_dir = os.path.join(os.path.dirname(__file__), 'src') -sys.path.insert(0, src_dir) - -from fe.web_app import main - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/code2test/src/be/agent_orchestrator.py b/code2test/src/be/agent_orchestrator.py deleted file mode 100644 index 9455a28..0000000 --- a/code2test/src/be/agent_orchestrator.py +++ /dev/null @@ -1,152 +0,0 @@ -from pydantic_ai import Agent -# import logfire -import logging -import os -import traceback -from typing import Dict, List, Any - -# Configure logging and monitoring - -logger = logging.getLogger(__name__) - -# try: -# # Configure logfire with environment variables for Docker compatibility -# logfire_token = os.getenv('LOGFIRE_TOKEN') -# logfire_project = os.getenv('LOGFIRE_PROJECT_NAME', 'default') -# logfire_service = os.getenv('LOGFIRE_SERVICE_NAME', 'default') - -# if logfire_token: -# # Configure with explicit token (for Docker) -# logfire.configure( -# token=logfire_token, -# project_name=logfire_project, -# service_name=logfire_service, -# ) -# else: -# # Use default configuration (for local development with logfire auth) -# logfire.configure( -# project_name=logfire_project, -# service_name=logfire_service, -# ) - -# logfire.instrument_pydantic_ai() -# logger.debug(f"Logfire configured successfully for project: {logfire_project}") - -# except Exception as e: -# logger.warning(f"Failed to configure logfire: {e}") - -# Local imports -from code2test.src.be.agent_tools.deps import CodeWikiDeps -from code2test.src.be.agent_tools.read_code_components import read_code_components_tool -from code2test.src.be.agent_tools.str_replace_editor import str_replace_editor_tool -from code2test.src.be.agent_tools.generate_sub_module_documentations import generate_sub_module_documentation_tool -from code2test.src.be.llm_services import create_fallback_models -from code2test.src.be.prompt_template import ( - format_user_prompt, - format_system_prompt, - format_leaf_system_prompt, -) -from code2test.src.be.utils import is_complex_module -from code2test.src.config import ( - Config, - MODULE_TREE_FILENAME, - OVERVIEW_FILENAME, -) -from code2test.src.utils import file_manager -from code2test.src.be.dependency_analyzer.models.core import Node - - -class AgentOrchestrator: - """Orchestrates the AI agents for documentation generation.""" - - def __init__(self, config: Config): - self.config = config - self.fallback_models = create_fallback_models(config) - self.custom_instructions = config.get_prompt_addition() if config else None - - def create_agent(self, module_name: str, components: Dict[str, Any], - core_component_ids: List[str]) -> Agent: - """Create an appropriate agent based on module complexity.""" - - if is_complex_module(components, core_component_ids): - return Agent( - self.fallback_models, - name=module_name, - deps_type=CodeWikiDeps, - tools=[ - read_code_components_tool, - str_replace_editor_tool, - generate_sub_module_documentation_tool - ], - system_prompt=format_system_prompt(module_name, self.custom_instructions), - ) - else: - return Agent( - self.fallback_models, - name=module_name, - deps_type=CodeWikiDeps, - tools=[read_code_components_tool, str_replace_editor_tool], - system_prompt=format_leaf_system_prompt(module_name, self.custom_instructions), - ) - - async def process_module(self, module_name: str, components: Dict[str, Node], - core_component_ids: List[str], module_path: List[str], working_dir: str) -> Dict[str, Any]: - """Process a single module and generate its documentation.""" - logger.info(f"Processing module: {module_name}") - - # Load or create module tree - module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - module_tree = file_manager.load_json(module_tree_path) - - # Create agent - agent = self.create_agent(module_name, components, core_component_ids) - - # Create dependencies - deps = CodeWikiDeps( - absolute_docs_path=working_dir, - absolute_repo_path=str(os.path.abspath(self.config.repo_path)), - registry={}, - components=components, - path_to_current_module=module_path, - current_module_name=module_name, - module_tree=module_tree, - max_depth=self.config.max_depth, - current_depth=1, - config=self.config, - custom_instructions=self.custom_instructions - ) - - # check if overview docs already exists - overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) - if os.path.exists(overview_docs_path): - logger.info(f"✓ Overview docs already exists at {overview_docs_path}") - return module_tree - - # check if module docs already exists - docs_path = os.path.join(working_dir, f"{module_name}.md") - if os.path.exists(docs_path): - logger.info(f"✓ Module docs already exists at {docs_path}") - return module_tree - - # Run agent - try: - result = await agent.run( - format_user_prompt( - module_name=module_name, - core_component_ids=core_component_ids, - components=components, - module_tree=deps.module_tree - ), - deps=deps - ) - - # Save updated module tree - file_manager.save_json(deps.module_tree, module_tree_path) - logger.debug(f"Successfully processed module: {module_name}") - - return deps.module_tree - - except Exception as e: - logger.error(f"Error processing module {module_name}: {str(e)}") - logger.error(f"Traceback: {traceback.format_exc()}") - raise \ No newline at end of file diff --git a/code2test/src/be/agent_tools/__init__.py b/code2test/src/be/agent_tools/__init__.py deleted file mode 100644 index b828275..0000000 --- a/code2test/src/be/agent_tools/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Agent tools for backend processing.""" - diff --git a/code2test/src/be/agent_tools/deps.py b/code2test/src/be/agent_tools/deps.py deleted file mode 100644 index 2b05f7d..0000000 --- a/code2test/src/be/agent_tools/deps.py +++ /dev/null @@ -1,17 +0,0 @@ -from dataclasses import dataclass -from code2test.src.be.dependency_analyzer.models.core import Node -from code2test.src.config import Config - -@dataclass -class CodeWikiDeps: - absolute_docs_path: str - absolute_repo_path: str - registry: dict - components: dict[str, Node] - path_to_current_module: list[str] - current_module_name: str - module_tree: dict[str, any] - max_depth: int - current_depth: int - config: Config # LLM configuration - custom_instructions: str = None \ No newline at end of file diff --git a/code2test/src/be/agent_tools/generate_sub_module_documentations.py b/code2test/src/be/agent_tools/generate_sub_module_documentations.py deleted file mode 100644 index 6ab5b85..0000000 --- a/code2test/src/be/agent_tools/generate_sub_module_documentations.py +++ /dev/null @@ -1,92 +0,0 @@ -from pydantic_ai import RunContext, Tool, Agent - -from code2test.src.be.agent_tools.deps import CodeWikiDeps -from code2test.src.be.agent_tools.read_code_components import read_code_components_tool -from code2test.src.be.agent_tools.str_replace_editor import str_replace_editor_tool -from code2test.src.be.llm_services import create_fallback_models -from code2test.src.be.prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt -from code2test.src.be.utils import is_complex_module, count_tokens -from code2test.src.be.cluster_modules import format_potential_core_components - -import logging -logger = logging.getLogger(__name__) - - - -async def generate_sub_module_documentation( - ctx: RunContext[CodeWikiDeps], - sub_module_specs: dict[str, list[str]] -) -> str: - """Generate detailed description of a given sub-module specs to the sub-agents - - Args: - sub_module_specs: The specs of the sub-modules to generate documentation for. E.g. {"sub_module_1": ["core_component_1.1", "core_component_1.2"], "sub_module_2": ["core_component_2.1", "core_component_2.2"], ...} - """ - - deps = ctx.deps - previous_module_name = deps.current_module_name - - # Create fallback models from config - fallback_models = create_fallback_models(deps.config) - - # add the sub-module to the module tree - value = deps.module_tree - for key in deps.path_to_current_module: - value = value[key]["children"] - for sub_module_name, core_component_ids in sub_module_specs.items(): - value[sub_module_name] = {"components": core_component_ids, "children": {}} - - for sub_module_name, core_component_ids in sub_module_specs.items(): - - # Create visual indentation for nested modules - indent = " " * deps.current_depth - arrow = "└─" if deps.current_depth > 0 else "→" - - logger.info(f"{indent}{arrow} Generating documentation for sub-module: {sub_module_name}") - - num_tokens = count_tokens(format_potential_core_components(core_component_ids, ctx.deps.components)[-1]) - - if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module: - sub_agent = Agent( - model=fallback_models, - name=sub_module_name, - deps_type=CodeWikiDeps, - system_prompt=SYSTEM_PROMPT.format(module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions), - tools=[read_code_components_tool, str_replace_editor_tool, generate_sub_module_documentation_tool], - ) - else: - sub_agent = Agent( - model=fallback_models, - name=sub_module_name, - deps_type=CodeWikiDeps, - system_prompt=LEAF_SYSTEM_PROMPT.format(module_name=sub_module_name, custom_instructions=ctx.deps.custom_instructions), - tools=[read_code_components_tool, str_replace_editor_tool], - ) - - deps.current_module_name = sub_module_name - deps.path_to_current_module.append(sub_module_name) - deps.current_depth += 1 - # log the current module tree - # print(f"Current module tree: {json.dumps(deps.module_tree, indent=4)}") - - result = await sub_agent.run( - format_user_prompt( - module_name=deps.current_module_name, - core_component_ids=core_component_ids, - components=ctx.deps.components, - module_tree=ctx.deps.module_tree, - ), - deps=ctx.deps - ) - - # remove the sub-module name from the path to current module and the module tree - deps.path_to_current_module.pop() - deps.current_depth -= 1 - - # restore the previous module name - deps.current_module_name = previous_module_name - - return f"Generate successfully. Documentations: {', '.join([key + '.md' for key in sub_module_specs.keys()])} are saved in the working directory." - - -generate_sub_module_documentation_tool = Tool(function=generate_sub_module_documentation, name="generate_sub_module_documentation", description="Generate detailed description of a given sub-module specs to the sub-agents", takes_ctx=True) \ No newline at end of file diff --git a/code2test/src/be/agent_tools/read_code_components.py b/code2test/src/be/agent_tools/read_code_components.py deleted file mode 100644 index 4335b09..0000000 --- a/code2test/src/be/agent_tools/read_code_components.py +++ /dev/null @@ -1,22 +0,0 @@ -from pydantic_ai import RunContext, Tool -from code2test.src.be.agent_tools.deps import CodeWikiDeps - - -async def read_code_components(ctx: RunContext[CodeWikiDeps], component_ids: list[str]) -> str: - """Read the code of a given component id - - Args: - component_ids: The ids of the components to read, e.g. ["sweagent.types.AgentRunResult", "sweagent.types.AgentRunResult"] where sweagent.types part is the path to the component and AgentRunResult is the name of the component - """ - - results = [] - - for component_id in component_ids: - if component_id not in ctx.deps.components: - results.append(f"# Component {component_id} not found") - else: - results.append(f"# Component {component_id}:\n{ctx.deps.components[component_id].source_code.strip()}\n\n") - - return "\n".join(results) - -read_code_components_tool = Tool(function=read_code_components, name="read_code_components", description="Read the code of a given list of component ids", takes_ctx=True) \ No newline at end of file diff --git a/code2test/src/be/agent_tools/str_replace_editor.py b/code2test/src/be/agent_tools/str_replace_editor.py deleted file mode 100644 index 08d172b..0000000 --- a/code2test/src/be/agent_tools/str_replace_editor.py +++ /dev/null @@ -1,791 +0,0 @@ -#!/usr/bin/env python3 - -"""Source: https://github.com/SWE-agent/SWE-agent/blob/main/tools/edit_anthropic/bin/str_replace_editor -This tool is used to view the given source code and view/edit the documentation files in the separate docs directory. -""" - -import json -import re -import subprocess -import sys -from collections import defaultdict -from pathlib import Path -from typing import List, Optional, Tuple, Literal -import io - -import logging - -# Configure logging and monitoring - -logger = logging.getLogger(__name__) - -from pydantic_ai import RunContext, Tool - -from .deps import CodeWikiDeps -from ..utils import validate_mermaid_diagrams - - -# There are some super strange "ascii can't decode x" errors, -# that can be solved with setting the default encoding for stdout -# Reconfigure in place: replacing sys.stdout with a new wrapper closes the -# original stream when it is garbage-collected (breaks pytest capture). -if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(encoding="utf-8") - -TRUNCATED_MESSAGE: str = "To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for." -MAX_RESPONSE_LEN: int = 16000 - -MAX_WINDOW_EXPANSION_VIEW = 0 -MAX_WINDOW_EXPANSION_EDIT_CONFIRM = 0 -USE_FILEMAP = False -USE_LINTER = False -Command = str -SNIPPET_LINES: int = 4 -LINT_WARNING_TEMPLATE = """ - -Your edits have been applied, but the linter has found syntax errors. - - -{errors} - - -Please review the changes and make sure they are correct. -In addition to the above errors, please also check the following: - -1. The edited file is correctly indented -2. The edited file does not contain duplicate lines -3. The edit does not break existing functionality - -In rare cases, the linter errors might not actually be errors or caused by your edit. Please use your own judgement. - -Edit the file again if necessary. -""" - - -def maybe_truncate(content: str, truncate_after: Optional[int] = MAX_RESPONSE_LEN): - """Truncate content and append a notice if content exceeds the specified length.""" - return ( - content - if not truncate_after or len(content) <= truncate_after - else content[:truncate_after] + TRUNCATED_MESSAGE - ) - - -class Flake8Error: - """A class to represent a single flake8 error""" - - def __init__(self, filename: str, line_number: int, col_number: int, problem: str): - self.filename = filename - self.line_number = line_number - self.col_number = col_number - self.problem = problem - - @classmethod - def from_line(cls, line: str): - try: - prefix, _sep, problem = line.partition(": ") - filename, line_number, col_number = prefix.split(":") - except (ValueError, IndexError) as e: - msg = f"Invalid flake8 error line: {line}" - raise ValueError(msg) from e - return cls(filename, int(line_number), int(col_number), problem) - - def __eq__(self, other): - if not isinstance(other, Flake8Error): - return NotImplemented - return ( - self.filename == other.filename - and self.line_number == other.line_number - and self.col_number == other.col_number - and self.problem == other.problem - ) - - def __repr__(self): - return f"Flake8Error(filename={self.filename}, line_number={self.line_number}, col_number={self.col_number}, problem={self.problem})" - - -def _update_previous_errors( - previous_errors: List[Flake8Error], replacement_window: Tuple[int, int], replacement_n_lines: int -) -> List[Flake8Error]: - """Update the line numbers of the previous errors to what they would be after the edit window. - This is a helper function for `_filter_previous_errors`. - - All previous errors that are inside of the edit window should not be ignored, - so they are removed from the previous errors list. - - Args: - previous_errors: list of errors with old line numbers - replacement_window: the window of the edit/lines that will be replaced - replacement_n_lines: the number of lines that will be used to replace the text - - Returns: - list of errors with updated line numbers - """ - updated = [] - lines_added = replacement_n_lines - (replacement_window[1] - replacement_window[0] + 1) - for error in previous_errors: - if error.line_number < replacement_window[0]: - # no need to adjust the line number - updated.append(error) - continue - if replacement_window[0] <= error.line_number <= replacement_window[1]: - # The error is within the edit window, so let's not ignore it - # either way (we wouldn't know how to adjust the line number anyway) - continue - # We're out of the edit window, so we need to adjust the line number - updated.append(Flake8Error(error.filename, error.line_number + lines_added, error.col_number, error.problem)) - return updated - - -def format_flake8_output( - input_string: str, - show_line_numbers: bool = False, - *, - previous_errors_string: str = "", - replacement_window: Optional[Tuple[int, int]] = None, - replacement_n_lines: Optional[int] = None, -) -> str: - """Filter flake8 output for previous errors and print it for a given file. - - Args: - input_string: The flake8 output as a string - show_line_numbers: Whether to show line numbers in the output - previous_errors_string: The previous errors as a string - replacement_window: The window of the edit (lines that will be replaced) - replacement_n_lines: The number of lines used to replace the text - - Returns: - The filtered flake8 output as a string - """ - # print(f"Replacement window: {replacement_window}") - # print("Replacement n lines:", replacement_n_lines) - # print("Previous errors string:", previous_errors_string) - # print("Input string:", input_string) - errors = [Flake8Error.from_line(line.strip()) for line in input_string.split("\n") if line.strip()] - # print(f"New errors before filtering: {errors=}") - lines = [] - if previous_errors_string: - assert replacement_window is not None - assert replacement_n_lines is not None - previous_errors = [ - Flake8Error.from_line(line.strip()) for line in previous_errors_string.split("\n") if line.strip() - ] - # print(f"Previous errors before updating: {previous_errors=}") - previous_errors = _update_previous_errors(previous_errors, replacement_window, replacement_n_lines) - # print(f"Previous errors after updating: {previous_errors=}") - errors = [error for error in errors if error not in previous_errors] - # Sometimes new errors appear above the replacement window that were 'shadowed' by the previous errors - # they still clearly aren't caused by the edit. - errors = [error for error in errors if error.line_number >= replacement_window[0]] - # print(f"New errors after filtering: {errors=}") - for error in errors: - if not show_line_numbers: - lines.append(f"- {error.problem}") - else: - lines.append(f"- line {error.line_number} col {error.col_number}: {error.problem}") - return "\n".join(lines) - - -def flake8(file_path: str) -> str: - """Run flake8 on a given file and return the output as a string""" - if Path(file_path).suffix != ".py": - return "" - cmd = "flake8 --isolated --select=F821,F822,F831,E111,E112,E113,E999,E902 {file_path}" - # don't use capture_output because it's not compatible with python3.6 - out = subprocess.run(cmd.format(file_path=file_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return out.stdout.decode() - - -class Filemap: - def show_filemap(self, file_contents: str, encoding: str = "utf8"): - import warnings - from tree_sitter_languages import get_language, get_parser - - warnings.simplefilter("ignore", category=FutureWarning) - - parser = get_parser("python") - language = get_language("python") - - tree = parser.parse(bytes(file_contents.encode(encoding, errors="replace"))) - - # See https://tree-sitter.github.io/tree-sitter/using-parsers#pattern-matching-with-queries. - query = language.query(""" - (function_definition - body: (_) @body) - """) - - # TODO: consider special casing docstrings such that they are not elided. This - # could be accomplished by checking whether `body.text.decode('utf8')` starts - # with `"""` or `'''`. - elide_line_ranges = [ - (node.start_point[0], node.end_point[0]) - for node, _ in query.captures(tree.root_node) - # Only elide if it's sufficiently long - if node.end_point[0] - node.start_point[0] >= 5 - ] - # Note that tree-sitter line numbers are 0-indexed, but we display 1-indexed. - elide_lines = {line for start, end in elide_line_ranges for line in range(start, end + 1)} - elide_messages = [(start, f"... eliding lines {start+1}-{end+1} ...") for start, end in elide_line_ranges] - out = [] - for i, line in sorted( - elide_messages + [(i, line) for i, line in enumerate(file_contents.splitlines()) if i not in elide_lines] - ): - out.append(f"{i+1:6d} {line}") - return "\n".join(out) - - -class WindowExpander: - def __init__(self, suffix: str = ""): - """Try to expand viewports to include whole functions, classes, etc. rather than - using fixed line windows. - - Args: - suffix: Filename suffix - """ - self.suffix = suffix - if self.suffix: - assert self.suffix.startswith(".") - - def _find_breakpoints(self, lines: List[str], current_line: int, direction=1, max_added_lines: int = 30) -> int: - """Returns 1-based line number of breakpoint. This line is meant to still be included in the viewport. - - Args: - lines: List of lines of the file - current_line: 1-based line number of the current viewport - direction: 1 for down, -1 for up - max_added_lines: Maximum number of lines to extend - - Returns: - 1-based line number of breakpoint. This line is meant to still be included in the viewport. - """ - assert 1 <= current_line <= len(lines) - assert 0 <= max_added_lines - - # 1. Find line range that we want to search for breakpoints in - - if direction == 1: - # down - if current_line == len(lines): - # already last line, can't extend down - return current_line - iter_lines = range(current_line, 1 + min(current_line + max_added_lines, len(lines))) - elif direction == -1: - # up - if current_line == 1: - # already first line, can't extend up - return current_line - iter_lines = range(current_line, -1 + max(current_line - max_added_lines, 1), -1) - else: - msg = f"Invalid direction {direction}" - raise ValueError(msg) - - # 2. Find the best breakpoint in the line range - - # Every condition gives a score, the best score is the best breakpoint - best_score = 0 - best_breakpoint = current_line - for i_line in iter_lines: - next_line = None - line = lines[i_line - 1] - if i_line + direction in iter_lines: - next_line = lines[i_line + direction - 1] - score = 0 - if line == "": - score = 1 - if next_line == "": - # Double new blank line: - score = 2 - if self.suffix == ".py" and any( - re.match(regex, line) for regex in [r"^\s*def\s+", r"^\s*class\s+", r"^\s*@"] - ): - # We include decorators here, because they are always on top of the function/class definition - score = 3 - if score > best_score: - best_score = score - best_breakpoint = i_line - if direction == 1 and i_line != current_line: - best_breakpoint -= 1 - if i_line == 1 or i_line == len(lines): - score = 3 - if score > best_score: - best_score = score - best_breakpoint = i_line - # print(f"Score {score} for line {i_line} ({line})") - - # print(f"Best score {best_score} for line {best_breakpoint} ({lines[best_breakpoint-1]})") - if direction == 1 and best_breakpoint < current_line or direction == -1 and best_breakpoint > current_line: - # We don't want to shrink the view port, so we return the current line - return current_line - - return best_breakpoint - - def expand_window(self, lines: List[str], start: int, stop: int, max_added_lines: int) -> Tuple[int, int]: - """ - - Args: - lines: All lines of the file - start: 1-based line number of the start of the viewport - stop: 1-based line number of the end of the viewport - max_added_lines: Maximum number of lines to extend (separately for each side) - - Returns: - Tuple of 1-based line numbers of the start and end of the viewport. - Both inclusive. - """ - # print("Input:", start, stop) - assert 1 <= start <= stop <= len(lines), (start, stop, len(lines)) - if max_added_lines <= 0: - # Already at max range, no expansion - return start, stop - new_start = self._find_breakpoints(lines, start, direction=-1, max_added_lines=max_added_lines) - new_stop = self._find_breakpoints(lines, stop, direction=1, max_added_lines=max_added_lines) - # print(f"Expanded window is {new_start} to {new_stop}") - assert new_start <= new_stop, (new_start, new_stop) - assert new_start <= start, (new_start, start) - assert start - new_start <= max_added_lines, (start, new_start) - assert new_stop >= stop, (new_stop, stop) - assert new_stop - stop <= max_added_lines, (new_stop, stop) - return new_start, new_stop - - -class EditTool: - """ - An filesystem editor tool that allows the agent to view, create, and edit files. - The tool parameters are defined by Anthropic and are not editable. - """ - - name = "str_replace_editor" - - def __init__(self, REGISTRY, absolute_docs_path=None): - super().__init__() - self._encoding = None - self.REGISTRY = REGISTRY - self.logs = [] - self.absolute_docs_path = Path(absolute_docs_path) if absolute_docs_path else None - - def _get_display_path(self, path: Path) -> str: - """Get path for display purposes - relative to absolute_docs_path if available""" - if self.absolute_docs_path and path.is_absolute(): - try: - return str(path.relative_to(self.absolute_docs_path)) - except ValueError: - # Path is not under absolute_docs_path, return as-is - return str(path) - return str(path) - - @property - def _file_history(self): - return defaultdict(list, json.loads(self.REGISTRY.get("file_history", "{}"))) - - @_file_history.setter - def _file_history(self, value: dict): - self.REGISTRY["file_history"] = json.dumps(value) - - def __call__( - self, - *, - command: Command, - path: str, - file_text: Optional[str] = None, - view_range: Optional[List[int]] = None, - old_str: Optional[str] = None, - new_str: Optional[str] = None, - insert_line: Optional[int] = None, - **kwargs, - ): - _path = Path(path) - if not self.validate_path(command, _path): - return - if command == "view": - return self.view(_path, view_range) - elif command == "create": - if file_text is None: - self.logs.append("Parameter `file_text` is required for command: create") - return - self.create_file(_path, file_text) - return None - elif command == "str_replace": - if old_str is None: - self.logs.append("Parameter `old_str` is required for command: str_replace") - return - return self.str_replace(_path, old_str, new_str) - elif command == "insert": - if insert_line is None: - self.logs.append("Parameter `insert_line` is required for command: insert") - return - if new_str is None: - self.logs.append("Parameter `new_str` is required for command: insert") - return - return self.insert(_path, insert_line, new_str) - elif command == "undo_edit": - return self.undo_edit(_path) - self.logs.append( - f'Unrecognized command {command}. The allowed commands for the {self.name} tool are: "view", "create", "str_replace", "insert", "undo_edit"' - ) - return - - def validate_path(self, command: str, path: Path): - """ - Check that the path/command combination is valid. - """ - # Check if its an absolute path - if not path.is_absolute(): - suggested_path = Path.cwd() / path - self.logs.append( - f"The path {self._get_display_path(path)} is not an absolute path, it should start with `/`. Maybe you meant {self._get_display_path(suggested_path)}?" - ) - return False - # Check if path exists - if not path.exists() and command != "create": - self.logs.append(f"The path {self._get_display_path(path)} does not exist. Please provide a valid path.") - return False - if path.exists() and command == "create": - self.logs.append(f"File already exists at: {self._get_display_path(path)}. Cannot overwrite files using command `create`.") - return False - # Check if the path points to a directory - if path.is_dir(): - if command != "view": - self.logs.append(f"The path {self._get_display_path(path)} is a directory and only the `view` command can be used on directories") - return False - return True - - def create_file(self, path: Path, file_text: str): - if not path.parent.exists(): - self.logs.append(f"The parent directory {self._get_display_path(path.parent)} does not exist. Please create it first.") - return - self.write_file(path, file_text) - self._file_history[path].append(file_text) - self.logs.append(f"File created successfully at: {self._get_display_path(path)}") - - def view(self, path: Path, view_range: Optional[List[int]] = None): - """Implement the view command""" - if path.is_dir(): - if view_range: - self.logs.append("The `view_range` parameter is not allowed when `path` points to a directory.") - return - - out = subprocess.run( - rf"find {path} -maxdepth 2 -not -path '*/\.*'", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout = out.stdout.decode() - stderr = out.stderr.decode() - - if not stderr: - stdout = stdout.replace(str(path), self._get_display_path(path)) - stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items:\n{stdout}\n" - self.logs.append(stdout) - return - - file_content = self.read_file(path) - if view_range: - if len(view_range) != 2 or not all(isinstance(i, int) for i in view_range): - self.logs.append("Invalid `view_range`. It should be a list of two integers.") - return - file_lines = file_content.split("\n") - n_lines_file = len(file_lines) - init_line, final_line = view_range - if init_line < 1 or init_line > n_lines_file: - self.logs.append( - f"Invalid `view_range`: {view_range}. Its first element `{init_line}` should be within the range of lines of the file: {[1, n_lines_file]}" - ) - return - if final_line > n_lines_file: - self.logs.append( - f"Invalid `view_range`: {view_range}. Its second element `{final_line}` should be smaller than the number of lines in the file: `{n_lines_file}`" - ) - return - if final_line != -1 and final_line < init_line: - self.logs.append( - f"Invalid `view_range`: {view_range}. Its second element `{final_line}` should be larger or equal than its first `{init_line}`" - ) - return - - if final_line == -1: - final_line = n_lines_file - - # Expand the viewport to include the whole function or class - init_line, final_line = WindowExpander(suffix=path.suffix).expand_window( - file_lines, init_line, final_line, max_added_lines=MAX_WINDOW_EXPANSION_VIEW - ) - - file_content = "\n".join(file_lines[init_line - 1 : final_line]) - else: - if path.suffix == ".py" and len(file_content) > MAX_RESPONSE_LEN and USE_FILEMAP: - try: - filemap = Filemap().show_filemap(file_content, encoding=self._encoding or "utf-8") - except Exception: - # If we fail to show the filemap, just show the truncated file content - pass - else: - self.logs.append( - "This file is too large to display entirely. Showing abbreviated version. " - "Please use `str_replace_editor view` with the `view_range` parameter to show selected lines next." - ) - filemap = maybe_truncate(filemap.expandtabs()) - self.logs.append(filemap) - self.logs.append( - "The above file has been abbreviated. Please use `str_replace editor view` with `view_range` to look at relevant files in detail." - ) - return - # Else just show - init_line = 1 - - # init_line is 1-based - self.logs.append(self._make_output(file_content, self._get_display_path(path), init_line=init_line)) - - def str_replace(self, path: Path, old_str: str, new_str: Optional[str]): - """Implement the str_replace command, which replaces old_str with new_str in the file content""" - # Read the file content - file_content = self.read_file(path).expandtabs() - old_str = old_str.expandtabs() - new_str = new_str.expandtabs() if new_str is not None else "" - - # Check if old_str is unique in the file - occurrences = file_content.count(old_str) - if occurrences == 0: - self.logs.append(f"No replacement was performed, old_str `{old_str}` did not appear verbatim in {self._get_display_path(path)}.") - return - elif occurrences > 1: - file_content_lines = file_content.split("\n") - lines = [idx + 1 for idx, line in enumerate(file_content_lines) if old_str in line] - self.logs.append( - f"No replacement was performed. Multiple occurrences of old_str `{old_str}` in lines {lines}. Please ensure it is unique" - ) - return - - if new_str == old_str: - self.logs.append(f"No replacement was performed, old_str `{old_str}` is the same as new_str `{new_str}`.") - return - - pre_edit_lint = "" - if USE_LINTER: - try: - pre_edit_lint = flake8(str(path)) - except Exception as e: - self.logs.append(f"Warning: Failed to run pre-edit linter on {path}: {e}") - - # Replace old_str with new_str - new_file_content = file_content.replace(old_str, new_str) - - # Write the new content to the file - self.write_file(path, new_file_content) - - post_edit_lint = "" - if USE_LINTER: - try: - post_edit_lint = flake8(str(path)) - except Exception as e: - self.logs.append(f"Warning: Failed to run post-edit linter on {path}: {e}") - - epilogue = "" - if post_edit_lint: - ... - replacement_window_start_line = file_content.split(old_str)[0].count("\n") + 1 - replacement_lines = len(new_str.split("\n")) - replacement_window_end_line = replacement_window_start_line + replacement_lines - 1 - replacement_window = (replacement_window_start_line, replacement_window_end_line) - errors = format_flake8_output( - post_edit_lint, - previous_errors_string=pre_edit_lint, - replacement_window=replacement_window, - replacement_n_lines=replacement_lines, - ) - if errors.strip(): - epilogue = LINT_WARNING_TEMPLATE.format(errors=errors) - - # Save the content to history - self._file_history[path].append(file_content) - - # Create a snippet of the edited section - replacement_line = file_content.split(old_str)[0].count("\n") - start_line = max(1, replacement_line - SNIPPET_LINES) - end_line = min(replacement_line + SNIPPET_LINES + new_str.count("\n"), len(new_file_content.splitlines())) - start_line, end_line = WindowExpander(suffix=path.suffix).expand_window( - new_file_content.split("\n"), start_line, end_line, max_added_lines=MAX_WINDOW_EXPANSION_EDIT_CONFIRM - ) - snippet = "\n".join(new_file_content.split("\n")[start_line - 1 : end_line]) - - # Prepare the success message - success_msg = f"The file {self._get_display_path(path)} has been edited. " - success_msg += self._make_output(snippet, f"a snippet of {self._get_display_path(path)}", start_line) - success_msg += "Review the changes and make sure they are as expected. Edit the file again if necessary." - success_msg += epilogue - - self.logs.append(success_msg) - - def insert(self, path: Path, insert_line: int, new_str: str): - """Implement the insert command, which inserts new_str at the specified line in the file content.""" - file_text = self.read_file(path).expandtabs() - new_str = new_str.expandtabs() - file_text_lines = file_text.split("\n") - n_lines_file = len(file_text_lines) - - if insert_line < 0 or insert_line > n_lines_file: - self.logs.append( - f"Invalid `insert_line` parameter: {insert_line}. It should be within the range of lines of the file: {[0, n_lines_file]}" - ) - return - - new_str_lines = new_str.split("\n") - new_file_text_lines = file_text_lines[:insert_line] + new_str_lines + file_text_lines[insert_line:] - snippet_lines = ( - file_text_lines[max(0, insert_line - SNIPPET_LINES) : insert_line] - + new_str_lines - + file_text_lines[insert_line : insert_line + SNIPPET_LINES] - ) - - new_file_text = "\n".join(new_file_text_lines) - snippet = "\n".join(snippet_lines) - - self.write_file(path, new_file_text) - self._file_history[path].append(file_text) - - # todo: Also expand these windows - - success_msg = f"The file {self._get_display_path(path)} has been edited. " - success_msg += self._make_output( - snippet, - "a snippet of the edited file", - max(1, insert_line - SNIPPET_LINES + 1), - ) - success_msg += "Review the changes and make sure they are as expected (correct indentation, no duplicate lines, etc). Edit the file again if necessary." - self.logs.append(success_msg) - - def undo_edit(self, path: Path): - """Implement the undo_edit command.""" - if not self._file_history[path]: - self.logs.append(f"No edit history found for {self._get_display_path(path)}.") - return - - old_text = self._file_history[path].pop() - self.write_file(path, old_text) - - self.logs.append(f"Last edit to {self._get_display_path(path)} undone successfully. {self._make_output(old_text, self._get_display_path(path))}") - - def read_file(self, path: Path): - """Read the content of a file from a given path; raise a ToolError if an error occurs.""" - encodings = [ - (None, None), - ("utf-8", None), - ("latin-1", None), - ("utf-8", "replace"), - ] - exception = None - for self._encoding, errors in encodings: - try: - text = path.read_text(encoding=self._encoding, errors=errors) - except UnicodeDecodeError as e: - exception = e - else: - break - else: - self.logs.append(f"Ran into UnicodeDecodeError {exception} while trying to read {self._get_display_path(path)}") - return - return text - - def write_file(self, path: Path, file: str): - """Write the content of a file to a given path; raise a ToolError if an error occurs.""" - try: - path.write_text(file, encoding=self._encoding or "utf-8") - except Exception as e: - self.logs.append(f"Ran into {e} while trying to write to {self._get_display_path(path)}") - return - - def _make_output( - self, - file_content: str, - file_descriptor: str, - init_line: int = 1, - expand_tabs: bool = True, - ): - """Generate output for the CLI based on the content of a file.""" - file_content = maybe_truncate(file_content) - if expand_tabs: - file_content = file_content.expandtabs() - file_content = "\n".join([f"{i + init_line:6}\t{line}" for i, line in enumerate(file_content.split("\n"))]) - return f"Here's the result of running `cat -n` on {file_descriptor}:\n" + file_content + "\n" - -async def str_replace_editor( - ctx: RunContext[CodeWikiDeps], - working_dir: Literal["repo", "docs"], - command: Literal["view", "create", "str_replace", "insert", "undo_edit"], - path: Optional[str] = None, - file: Optional[str] = None, - file_text: Optional[str] = None, - view_range: Optional[List[int]] = None, - old_str: Optional[str] = None, - new_str: Optional[str] = None, - insert_line: Optional[int] = None, -) -> str: - """ - Custom editing tool for viewing, creating and editing files - * State is persistent across command calls and discussions with the user - * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep. - * The `create` command cannot be used if the specified `path` already exists as a file - * If a `command` generates a long output, it will be truncated and marked with `` - * The `undo_edit` command will revert the last edit made to the file at `path` - * Only `view` command is allowed when `working_dir` is `repo`. - - Args: - working_dir: The working directory to use. Choose `repo` to work with the repository files, or `docs` to work with the generated documentation files. - command: The command to run. Allowed options are: `view`, `create`, `str_replace`, `insert`, `undo_edit`. - path: Path to file or directory, e.g. `./chat_core.md` or `./agents/` - file: Alias for `path` parameter (for compatibility with some models) - file_text: Required parameter of `create` command, with the content of the file to be created. - view_range: Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file. - old_str: Required parameter of `str_replace` command containing the string in `path` to replace. - new_str: Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert. - """ - - # Handle both `path` and `file` parameters for model compatibility - if path is None and file is None: - return "Error: Either `path` or `file` parameter must be provided." - if path is None: - path = file - - tool = EditTool(ctx.deps.registry, ctx.deps.absolute_docs_path) - if working_dir == "docs": - absolute_path = str(Path(ctx.deps.absolute_docs_path) / path) - else: - absolute_path = str(Path(ctx.deps.absolute_repo_path) / path) - - # validate command - if command != "view" and working_dir == "repo": - return "The `view` command is the only allowed command when `working_dir` is `repo`." - - tool( - command=command, - path=absolute_path, - file_text=file_text, - view_range=view_range, - old_str=old_str, - new_str=new_str, - insert_line=insert_line, - ) - - result = "\n".join(tool.logs) - - if command != "view" and path.endswith(".md"): - mermaid_validation = await validate_mermaid_diagrams(absolute_path, path) - result = result + "\n---------- Mermaid validation ----------\n" + mermaid_validation - - return result - - -str_replace_editor_tool = Tool( - function=str_replace_editor, - name="str_replace_editor", - description=""" -Custom editing tool for viewing, creating and editing files - * State is persistent across command calls and discussions with the user - * If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep. - * The `create` command cannot be used if the specified `path` already exists as a file - * If a `command` generates a long output, it will be truncated and marked with `` - * The `undo_edit` command will revert the last edit made to the file at `path` - * Only `view` command is allowed when `working_dir` is `repo`. -""".strip(), - takes_ctx=True -) diff --git a/code2test/src/be/cluster_modules.py b/code2test/src/be/cluster_modules.py deleted file mode 100644 index 01f2c9b..0000000 --- a/code2test/src/be/cluster_modules.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import List, Dict, Any -from collections import defaultdict -import logging -import traceback -logger = logging.getLogger(__name__) - -from code2test.src.be.dependency_analyzer.models.core import Node -from code2test.src.be.llm_services import call_llm -from code2test.src.be.utils import count_tokens -from code2test.src.config import Config -from code2test.src.be.prompt_template import format_cluster_prompt - - -def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]: - """ - Format the potential core components into a string that can be used in the prompt. - """ - # Filter out any invalid leaf nodes that don't exist in components - valid_leaf_nodes = [] - for leaf_node in leaf_nodes: - if leaf_node in components: - valid_leaf_nodes.append(leaf_node) - else: - logger.warning(f"Skipping invalid leaf node '{leaf_node}' - not found in components") - - #group leaf nodes by file - leaf_nodes_by_file = defaultdict(list) - for leaf_node in valid_leaf_nodes: - leaf_nodes_by_file[components[leaf_node].relative_path].append(leaf_node) - - potential_core_components = "" - potential_core_components_with_code = "" - for file, leaf_nodes in dict(sorted(leaf_nodes_by_file.items())).items(): - potential_core_components += f"# {file}\n" - potential_core_components_with_code += f"# {file}\n" - for leaf_node in leaf_nodes: - potential_core_components += f"\t{leaf_node}\n" - potential_core_components_with_code += f"\t{leaf_node}\n" - potential_core_components_with_code += f"{components[leaf_node].source_code}\n" - - return potential_core_components, potential_core_components_with_code - - -def cluster_modules( - leaf_nodes: List[str], - components: Dict[str, Node], - config: Config, - current_module_tree: dict[str, Any] = {}, - current_module_name: str = None, - current_module_path: List[str] = [] -) -> Dict[str, Any]: - """ - Cluster the potential core components into modules. - """ - potential_core_components, potential_core_components_with_code = format_potential_core_components(leaf_nodes, components) - - if count_tokens(potential_core_components_with_code) <= config.max_token_per_module: - logger.debug(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens") - return {} - - prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) - response = call_llm(prompt, config, model=config.cluster_model) - - #parse the response - try: - if "" not in response or "" not in response: - logger.error(f"Invalid LLM response format - missing component tags: {response[:200]}...") - return {} - - response_content = response.split("")[1].split("")[0] - module_tree = eval(response_content) - - if not isinstance(module_tree, dict): - logger.error(f"Invalid module tree format - expected dict, got {type(module_tree)}") - return {} - - except Exception as e: - logger.error(f"Failed to parse LLM response: {e}. Response: {response[:200]}...") - logger.error(f"Traceback: {traceback.format_exc()}") - return {} - - # check if the module tree is valid - if len(module_tree) <= 1: - logger.debug(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules") - return {} - - if current_module_tree == {}: - current_module_tree = module_tree - else: - value = current_module_tree - for key in current_module_path: - value = value[key]["children"] - for module_name, module_info in module_tree.items(): - del module_info["path"] - value[module_name] = module_info - - for module_name, module_info in module_tree.items(): - sub_leaf_nodes = module_info.get("components", []) - - # Filter sub_leaf_nodes to ensure they exist in components - valid_sub_leaf_nodes = [] - for node in sub_leaf_nodes: - if node in components: - valid_sub_leaf_nodes.append(node) - else: - logger.warning(f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components") - - current_module_path.append(module_name) - module_info["children"] = {} - module_info["children"] = cluster_modules(valid_sub_leaf_nodes, components, config, current_module_tree, module_name, current_module_path) - current_module_path.pop() - - return module_tree \ No newline at end of file diff --git a/code2test/src/be/documentation_generator.py b/code2test/src/be/documentation_generator.py deleted file mode 100644 index 007f912..0000000 --- a/code2test/src/be/documentation_generator.py +++ /dev/null @@ -1,292 +0,0 @@ -import logging -import os -import json -from typing import Dict, List, Any -from copy import deepcopy -import traceback - -# Configure logging and monitoring -logger = logging.getLogger(__name__) - -# Local imports -from code2test.src.be.dependency_analyzer import DependencyGraphBuilder -from code2test.src.be.llm_services import call_llm -from code2test.src.be.prompt_template import ( - REPO_OVERVIEW_PROMPT, - MODULE_OVERVIEW_PROMPT, -) -from code2test.src.be.cluster_modules import cluster_modules -from code2test.src.config import ( - Config, - FIRST_MODULE_TREE_FILENAME, - MODULE_TREE_FILENAME, - OVERVIEW_FILENAME -) -from code2test.src.utils import file_manager -from code2test.src.be.agent_orchestrator import AgentOrchestrator - - -class DocumentationGenerator: - """Main documentation generation orchestrator.""" - - def __init__(self, config: Config, commit_id: str = None): - self.config = config - self.commit_id = commit_id - self.graph_builder = DependencyGraphBuilder(config) - self.agent_orchestrator = AgentOrchestrator(config) - - def create_documentation_metadata(self, working_dir: str, components: Dict[str, Any], num_leaf_nodes: int): - """Create a metadata file with documentation generation information.""" - from datetime import datetime - - metadata = { - "generation_info": { - "timestamp": datetime.now().isoformat(), - "main_model": self.config.main_model, - "generator_version": "1.0.1", - "repo_path": self.config.repo_path, - "commit_id": self.commit_id - }, - "statistics": { - "total_components": len(components), - "leaf_nodes": num_leaf_nodes, - "max_depth": self.config.max_depth - }, - "files_generated": [ - "overview.md", - "module_tree.json", - "first_module_tree.json" - ] - } - - # Add generated markdown files to the metadata - try: - for file_path in os.listdir(working_dir): - if file_path.endswith('.md') and file_path not in metadata["files_generated"]: - metadata["files_generated"].append(file_path) - except Exception as e: - logger.warning(f"Could not list generated files: {e}") - - metadata_path = os.path.join(working_dir, "metadata.json") - file_manager.save_json(metadata, metadata_path) - - - def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]: - """Get the processing order using topological sort (leaf modules first).""" - processing_order = [] - - def collect_modules(tree: Dict[str, Any], path: List[str]): - for module_name, module_info in tree.items(): - current_path = path + [module_name] - - # If this module has children, process them first - if module_info.get("children") and isinstance(module_info["children"], dict) and module_info["children"]: - collect_modules(module_info["children"], current_path) - # Add this parent module after its children - processing_order.append((current_path, module_name)) - else: - # This is a leaf module, add it immediately - processing_order.append((current_path, module_name)) - - collect_modules(module_tree, parent_path) - return processing_order - - def is_leaf_module(self, module_info: Dict[str, Any]) -> bool: - """Check if a module is a leaf module (has no children or empty children).""" - children = module_info.get("children", {}) - return not children or (isinstance(children, dict) and len(children) == 0) - - def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str], - working_dir: str) -> Dict[str, Any]: - """Build structure for overview generation with 1-depth children docs and target indicator.""" - - processed_module_tree = deepcopy(module_tree) - module_info = processed_module_tree - for path_part in module_path: - module_info = module_info[path_part] - if path_part != module_path[-1]: - module_info = module_info.get("children", {}) - else: - module_info["is_target_for_overview_generation"] = True - - if "children" in module_info: - module_info = module_info["children"] - - for child_name, child_info in module_info.items(): - if os.path.exists(os.path.join(working_dir, f"{child_name}.md")): - child_info["docs"] = file_manager.load_text(os.path.join(working_dir, f"{child_name}.md")) - else: - logger.warning(f"Module docs not found at {os.path.join(working_dir, f"{child_name}.md")}") - child_info["docs"] = "" - - return processed_module_tree - - async def generate_module_documentation(self, components: Dict[str, Any], leaf_nodes: List[str]) -> str: - """Generate documentation for all modules using dynamic programming approach.""" - # Prepare output directory - working_dir = os.path.abspath(self.config.docs_dir) - file_manager.ensure_directory(working_dir) - - module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) - module_tree = file_manager.load_json(module_tree_path) - first_module_tree = file_manager.load_json(first_module_tree_path) - - # Get processing order (leaf modules first) - processing_order = self.get_processing_order(first_module_tree) - - - # Process modules in dependency order - final_module_tree = module_tree - processed_modules = set() - - if len(module_tree) > 0: - for module_path, module_name in processing_order: - try: - # Get the module info from the tree - module_info = module_tree - for path_part in module_path: - module_info = module_info[path_part] - if path_part != module_path[-1]: # Not the last part - module_info = module_info.get("children", {}) - - # Skip if already processed - module_key = "/".join(module_path) - if module_key in processed_modules: - continue - - # Process the module - if self.is_leaf_module(module_info): - logger.info(f"📄 Processing leaf module: {module_key}") - final_module_tree = await self.agent_orchestrator.process_module( - module_name, components, module_info["components"], module_path, working_dir - ) - else: - logger.info(f"📁 Processing parent module: {module_key}") - final_module_tree = await self.generate_parent_module_docs( - module_path, working_dir - ) - - processed_modules.add(module_key) - - except Exception as e: - logger.error(f"Failed to process module {module_key}: {str(e)}") - logger.error(f"Traceback: {traceback.format_exc()}") - continue - - # Generate repo overview - logger.info(f"📚 Generating repository overview") - final_module_tree = await self.generate_parent_module_docs( - [], working_dir - ) - else: - logger.info(f"Processing whole repo because repo can fit in the context window") - repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) - final_module_tree = await self.agent_orchestrator.process_module( - repo_name, components, leaf_nodes, [], working_dir - ) - - # save final_module_tree to module_tree.json - file_manager.save_json(final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)) - - # rename repo_name.md to overview.md - repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") - if os.path.exists(repo_overview_path): - os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) - - return working_dir - - async def generate_parent_module_docs(self, module_path: List[str], - working_dir: str) -> Dict[str, Any]: - """Generate documentation for a parent module based on its children's documentation.""" - module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path)) - - logger.info(f"Generating parent documentation for: {module_name}") - - # Load module tree - module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - module_tree = file_manager.load_json(module_tree_path) - - # check if overview docs already exists - overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) - if os.path.exists(overview_docs_path): - logger.info(f"✓ Overview docs already exists at {overview_docs_path}") - return module_tree - - # check if parent docs already exists - parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") - if os.path.exists(parent_docs_path): - logger.info(f"✓ Parent docs already exists at {parent_docs_path}") - return module_tree - - # Create repo structure with 1-depth children docs and target indicator - repo_structure = self.build_overview_structure(module_tree, module_path, working_dir) - - prompt = MODULE_OVERVIEW_PROMPT.format( - module_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4) - ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format( - repo_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4) - ) - - try: - parent_docs = call_llm(prompt, self.config) - - # Parse and save parent documentation - parent_content = parent_docs.split("")[1].split("")[0].strip() - # parent_content = prompt - file_manager.save_text(parent_content, parent_docs_path) - - logger.debug(f"Successfully generated parent documentation for: {module_name}") - return module_tree - - except Exception as e: - logger.error(f"Error generating parent documentation for {module_name}: {str(e)}") - logger.error(f"Traceback: {traceback.format_exc()}") - raise - - async def run(self) -> None: - """Run the complete documentation generation process using dynamic programming.""" - try: - # Build dependency graph - components, leaf_nodes = self.graph_builder.build_dependency_graph() - - logger.debug(f"Found {len(leaf_nodes)} leaf nodes") - # logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") - # exit() - - # Cluster modules - working_dir = os.path.abspath(self.config.docs_dir) - file_manager.ensure_directory(working_dir) - first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) - module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - - # Check if module tree exists - if os.path.exists(first_module_tree_path): - logger.debug(f"Module tree found at {first_module_tree_path}") - module_tree = file_manager.load_json(first_module_tree_path) - else: - logger.debug(f"Module tree not found at {module_tree_path}, clustering modules") - module_tree = cluster_modules(leaf_nodes, components, self.config) - file_manager.save_json(module_tree, first_module_tree_path) - - file_manager.save_json(module_tree, module_tree_path) - - logger.debug(f"Grouped components into {len(module_tree)} modules") - - # Generate module documentation using dynamic programming approach - # This processes leaf modules first, then parent modules - working_dir = await self.generate_module_documentation(components, leaf_nodes) - - # Create documentation metadata - self.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - - logger.debug(f"Documentation generation completed successfully using dynamic programming!") - logger.debug(f"Processing order: leaf modules → parent modules → repository overview") - logger.debug(f"Documentation saved to: {working_dir}") - - except Exception as e: - logger.error(f"Documentation generation failed: {str(e)}") - logger.error(f"Traceback: {traceback.format_exc()}") - raise \ No newline at end of file diff --git a/code2test/src/be/llm_services.py b/code2test/src/be/llm_services.py deleted file mode 100644 index e7ac55f..0000000 --- a/code2test/src/be/llm_services.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -LLM service factory for creating configured LLM clients. -""" -from pydantic_ai.models.openai import OpenAIChatModel -from pydantic_ai.providers.openai import OpenAIProvider -from pydantic_ai.models.openai import OpenAIChatModelSettings -from pydantic_ai.models.fallback import FallbackModel -from openai import OpenAI - -from code2test.src.config import Config - - -def create_main_model(config: Config) -> OpenAIChatModel: - """Create the main LLM model from configuration.""" - return OpenAIChatModel( - model_name=config.main_model, - provider=OpenAIProvider( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ), - settings=OpenAIChatModelSettings( - temperature=0.0, - max_tokens=config.max_tokens - ) - ) - - -def create_fallback_model(config: Config) -> OpenAIChatModel: - """Create the fallback LLM model from configuration.""" - return OpenAIChatModel( - model_name=config.fallback_model, - provider=OpenAIProvider( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ), - settings=OpenAIChatModelSettings( - temperature=0.0, - max_tokens=config.max_tokens - ) - ) - - -def create_fallback_models(config: Config) -> FallbackModel: - """Create fallback models chain from configuration.""" - main = create_main_model(config) - fallback = create_fallback_model(config) - return FallbackModel(main, fallback) - - -def create_openai_client(config: Config) -> OpenAI: - """Create OpenAI client from configuration.""" - return OpenAI( - base_url=config.llm_base_url, - api_key=config.llm_api_key - ) - - -def call_llm( - prompt: str, - config: Config, - model: str = None, - temperature: float = 0.0 -) -> str: - """ - Call LLM with the given prompt. - - Args: - prompt: The prompt to send - config: Configuration containing LLM settings - model: Model name (defaults to config.main_model) - temperature: Temperature setting - - Returns: - LLM response text - """ - if model is None: - model = config.main_model - - client = create_openai_client(config) - response = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=temperature, - max_tokens=config.max_tokens - ) - return response.choices[0].message.content \ No newline at end of file diff --git a/code2test/src/be/main.py b/code2test/src/be/main.py deleted file mode 100644 index d6e2d6a..0000000 --- a/code2test/src/be/main.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -CodeWiki - A tool for generating comprehensive documentation from Python codebases. - -This module orchestrates the documentation generation process by: -1. Analyzing code dependencies -2. Clustering related modules -3. Generating documentation using AI agents -4. Creating overview documentation -""" - -import logging -import argparse -import asyncio -import traceback - -# Configure logging and monitoring -from code2test.src.be.dependency_analyzer.utils.logging_config import setup_logging - -# Initialize colored logging -setup_logging(level=logging.INFO) - -logger = logging.getLogger(__name__) - -# Local imports -from code2test.src.be.documentation_generator import DocumentationGenerator -from code2test.src.config import ( - Config, -) - - -def parse_arguments() -> argparse.Namespace: - """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description='Generate comprehensive documentation for Python components in dependency order.' - ) - parser.add_argument( - '--repo-path', - type=str, - required=True, - help='Path to the repository' - ) - - return parser.parse_args() - - -async def main() -> None: - """Main entry point for the documentation generation process.""" - try: - # Parse arguments and create configuration - args = parse_arguments() - config = Config.from_args(args) - - # Create and run documentation generator - doc_generator = DocumentationGenerator(config) - await doc_generator.run() - - except KeyboardInterrupt: - logger.debug("Documentation generation interrupted by user") - except Exception as e: - logger.error(f"Unexpected error: {str(e)}") - logger.error(f"Traceback: {traceback.format_exc()}") - raise - - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file diff --git a/code2test/src/be/prompt_template.py b/code2test/src/be/prompt_template.py deleted file mode 100644 index bdb3a96..0000000 --- a/code2test/src/be/prompt_template.py +++ /dev/null @@ -1,375 +0,0 @@ -SYSTEM_PROMPT = """ - -You are an AI documentation assistant. Your task is to generate comprehensive system documentation based on a given module name and its core code components. - - - -Create documentation that helps developers and maintainers understand: -1. The module's purpose and core functionality -2. Architecture and component relationships -3. How the module fits into the overall system - - - -Generate documentation following this structure: - -1. **Main Documentation File** (`{module_name}.md`): - - Brief introduction and purpose - - Architecture overview with diagrams - - High-level functionality of each sub-module including references to its documentation file - - Link to other module documentation instead of duplicating information - -2. **Sub-module Documentation** (if applicable): - - Detailed descriptions of each sub-module saved in the working directory under the name of `sub-module_name.md` - - Core components and their responsibilities - -3. **Visual Documentation**: - - Mermaid diagrams for architecture, dependencies, and data flow - - Component interaction diagrams - - Process flow diagrams where relevant - - - -1. Analyze the provided code components and module structure, explore the not given dependencies between the components if needed -2. Create the main `{module_name}.md` file with overview and architecture in working directory -3. Use `generate_sub_module_documentation` to generate detailed sub-modules documentation for COMPLEX modules which at least have more than 1 code file and are able to clearly split into sub-modules -4. Include relevant Mermaid diagrams throughout the documentation -5. After all sub-modules are documented, adjust `{module_name}.md` with ONLY ONE STEP to ensure all generated files including sub-modules documentation are properly cross-refered - - - -- `str_replace_editor`: File system operations for creating and editing documentation files -- `read_code_components`: Explore additional code dependencies not included in the provided components -- `generate_sub_module_documentation`: Generate detailed documentation for individual sub-modules via sub-agents - -{custom_instructions} -""".strip() - -LEAF_SYSTEM_PROMPT = """ - -You are an AI documentation assistant. Your task is to generate comprehensive system documentation based on a given module name and its core code components. - - - -Create a comprehensive documentation that helps developers and maintainers understand: -1. The module's purpose and core functionality -2. Architecture and component relationships -3. How the module fits into the overall system - - - -Generate documentation following the following requirements: -1. Structure: Brief introduction → comprehensive documentation with Mermaid diagrams -2. Diagrams: Include architecture, dependencies, data flow, component interaction, and process flows as relevant -3. References: Link to other module documentation instead of duplicating information - - - -1. Analyze provided code components and module structure -2. Explore dependencies between components if needed -3. Generate complete {module_name}.md documentation file - - - -- `str_replace_editor`: File system operations for creating and editing documentation files -- `read_code_components`: Explore additional code dependencies not included in the provided components - -{custom_instructions} -""".strip() - -USER_PROMPT = """ -Generate comprehensive documentation for the {module_name} module using the provided module tree and core components. - - -{module_tree} - -* NOTE: You can refer the other modules in the module tree based on the dependencies between their core components to make the documentation more structured and avoid repeating the same information. Know that all documentation files are saved in the same folder not structured as module tree. e.g. [alt text]([ref_module_name].md) - - -{formatted_core_component_codes} - -""".strip() - -REPO_OVERVIEW_PROMPT = """ -You are an AI documentation assistant. Your task is to generate a brief overview of the {repo_name} repository. - -The overview should be a brief documentation of the repository, including: -- The purpose of the repository -- The end-to-end architecture of the repository visualized by mermaid diagrams -- The references to the core modules documentation - -Provide `{repo_name}` repo structure and its core modules documentation: - -{repo_structure} - - -Please generate the overview of the `{repo_name}` repository in markdown format with the following structure: - -overview_content - -""".strip() - -MODULE_OVERVIEW_PROMPT = """ -You are an AI documentation assistant. Your task is to generate a brief overview of `{module_name}` module. - -The overview should be a brief documentation of the module, including: -- The purpose of the module -- The architecture of the module visualized by mermaid diagrams -- The references to the core components documentation - -Provide repo structure and core components documentation of the `{module_name}` module: - -{repo_structure} - - -Please generate the overview of the `{module_name}` module in markdown format with the following structure: - -overview_content - -""".strip() - -CLUSTER_REPO_PROMPT = """ -Here is list of all potential core components of the repository (It's normal that some components are not essential to the repository): - -{potential_core_components} - - -Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a module. DO NOT include components that are not essential to the repository. -Firstly reason about the components and then group them and return the result in the following format: - -{{ - "module_name_1": {{ - "path": , # the path to the module can be file or directory - "components": [ - , - , - ... - ] - }}, - "module_name_2": {{ - "path": , - "components": [ - , - , - ... - ] - }}, - ... -}} - -""".strip() - -CLUSTER_MODULE_PROMPT = """ -Here is the module tree of a repository: - - -{module_tree} - - -Here is list of all potential core components of the module {module_name} (It's normal that some components are not essential to the module): - -{potential_core_components} - - -Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a smaller module. DO NOT include components that are not essential to the module. - -Firstly reason based on given context and then group them and return the result in the following format: - -{{ - "module_name_1": {{ - "path": , # the path to the module can be file or directory - "components": [ - , - , - ... - ] - }}, - "module_name_2": {{ - "path": , - "components": [ - , - , - ... - ] - }}, - ... -}} - -""".strip() - -FILTER_FOLDERS_PROMPT = """ -Here is the list of relative paths of files, folders in 2-depth of project {project_name}: -``` -{files} -``` - -In order to analyze the core functionality of the project, we need to analyze the files, folders representing the core functionality of the project. - -Please shortlist the files, folders representing the core functionality and ignore the files, folders that are not essential to the core functionality of the project (e.g. test files, documentation files, etc.) from the list above. - -Reasoning at first, then return the list of relative paths in JSON format. -""" - -from typing import Dict, Any -from code2test.src.utils import file_manager - -EXTENSION_TO_LANGUAGE = { - ".py": "python", - ".md": "markdown", - ".sh": "bash", - ".json": "json", - ".yaml": "yaml", - ".java": "java", - ".js": "javascript", - ".ts": "typescript", - ".cpp": "cpp", - ".c": "c", - ".h": "c", - ".hpp": "cpp", - ".tsx": "typescript", - ".cc": "cpp", - ".hpp": "cpp", - ".cxx": "cpp", - ".jsx": "javascript", - ".mjs": "javascript", - ".cjs": "javascript", - ".jsx": "javascript", - ".cs": "csharp", - ".php": "php", - ".phtml": "php", - ".inc": "php" -} - - -def format_user_prompt(module_name: str, core_component_ids: list[str], components: Dict[str, Any], module_tree: dict[str, any]) -> str: - """ - Format the user prompt with module name and organized core component codes. - - Args: - module_name: Name of the module to document - core_component_ids: List of component IDs to include - components: Dictionary mapping component IDs to CodeComponent objects - - Returns: - Formatted user prompt string - """ - - # format module tree - lines = [] - - def _format_module_tree(module_tree: dict[str, any], indent: int = 0): - for key, value in module_tree.items(): - if key == module_name: - lines.append(f"{' ' * indent}{key} (current module)") - else: - lines.append(f"{' ' * indent}{key}") - - lines.append(f"{' ' * (indent + 1)} Core components: {', '.join(value['components'])}") - if isinstance(value["children"], dict) and len(value["children"]) > 0: - lines.append(f"{' ' * (indent + 1)} Children:") - _format_module_tree(value["children"], indent + 2) - - _format_module_tree(module_tree, 0) - formatted_module_tree = "\n".join(lines) - - # print(f"Formatted module tree:\n{formatted_module_tree}") - - # Group core component IDs by their file path - grouped_components: dict[str, list[str]] = {} - for component_id in core_component_ids: - if component_id not in components: - continue - component = components[component_id] - path = component.relative_path - if path not in grouped_components: - grouped_components[path] = [] - grouped_components[path].append(component_id) - - core_component_codes = "" - for path, component_ids_in_file in grouped_components.items(): - core_component_codes += f"# File: {path}\n\n" - core_component_codes += f"## Core Components in this file:\n" - - for component_id in component_ids_in_file: - core_component_codes += f"- {component_id}\n" - - core_component_codes += f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.'+path.split('.')[-1]]}\n" - - # Read content of the file using the first component's file path - try: - core_component_codes += file_manager.load_text(components[component_ids_in_file[0]].file_path) - except (FileNotFoundError, IOError) as e: - core_component_codes += f"# Error reading file: {e}\n" - - core_component_codes += "```\n\n" - - return USER_PROMPT.format(module_name=module_name, formatted_core_component_codes=core_component_codes, module_tree=formatted_module_tree) - - - -def format_cluster_prompt(potential_core_components: str, module_tree: dict[str, any] = {}, module_name: str = None) -> str: - """ - Format the cluster prompt with potential core components and module tree. - """ - - # format module tree - lines = [] - - # print(f"Module tree:\n{json.dumps(module_tree, indent=2)}") - - def _format_module_tree(module_tree: dict[str, any], indent: int = 0): - for key, value in module_tree.items(): - if key == module_name: - lines.append(f"{' ' * indent}{key} (current module)") - else: - lines.append(f"{' ' * indent}{key}") - - lines.append(f"{' ' * (indent + 1)} Core components: {', '.join(value['components'])}") - if ("children" in value) and isinstance(value["children"], dict) and len(value["children"]) > 0: - lines.append(f"{' ' * (indent + 1)} Children:") - _format_module_tree(value["children"], indent + 2) - - _format_module_tree(module_tree, 0) - formatted_module_tree = "\n".join(lines) - - - if module_tree == {}: - return CLUSTER_REPO_PROMPT.format(potential_core_components=potential_core_components) - else: - return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name) - - -def format_system_prompt(module_name: str, custom_instructions: str = None) -> str: - """ - Format the system prompt with module name and optional custom instructions. - - Args: - module_name: Name of the module to document - custom_instructions: Optional custom instructions to append - - Returns: - Formatted system prompt string - """ - custom_section = "" - if custom_instructions: - custom_section = f"\n\n\n{custom_instructions}\n" - - return SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() - - -def format_leaf_system_prompt(module_name: str, custom_instructions: str = None) -> str: - """ - Format the leaf system prompt with module name and optional custom instructions. - - Args: - module_name: Name of the module to document - custom_instructions: Optional custom instructions to append - - Returns: - Formatted leaf system prompt string - """ - custom_section = "" - if custom_instructions: - custom_section = f"\n\n\n{custom_instructions}\n" - - return LEAF_SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() \ No newline at end of file diff --git a/code2test/src/be/utils.py b/code2test/src/be/utils.py deleted file mode 100644 index 6d88d53..0000000 --- a/code2test/src/be/utils.py +++ /dev/null @@ -1,207 +0,0 @@ -import re -from pathlib import Path -from typing import List, Tuple -import logging -import tiktoken -import traceback - - -logger = logging.getLogger(__name__) - -# ------------------------------------------------------------ -# ---------------------- Complexity Check -------------------- -# ------------------------------------------------------------ - -def is_complex_module(components: dict[str, any], core_component_ids: list[str]) -> bool: - files = set() - for component_id in core_component_ids: - if component_id in components: - files.add(components[component_id].file_path) - - result = len(files) > 1 - - return result - - -# ------------------------------------------------------------ -# ---------------------- Token Counting --------------------- -# ------------------------------------------------------------ - -enc = tiktoken.encoding_for_model("gpt-4") - -def count_tokens(text: str) -> int: - """ - Count the number of tokens in a text. - """ - length = len(enc.encode(text)) - # logger.debug(f"Number of tokens: {length}") - return length - - -# ------------------------------------------------------------ -# ---------------------- Mermaid Validation ----------------- -# ------------------------------------------------------------ - -async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> str: - """ - Validate all Mermaid diagrams in a markdown file. - - Args: - md_file_path: Path to the markdown file to check - relative_path: Relative path to the markdown file - Returns: - "All mermaid diagrams are syntax correct" if all diagrams are valid, - otherwise returns error message with details about invalid diagrams - """ - - try: - # Read the markdown file - file_path = Path(md_file_path) - if not file_path.exists(): - return f"Error: File '{md_file_path}' does not exist" - - content = file_path.read_text(encoding='utf-8') - - # Extract all mermaid code blocks - mermaid_blocks = extract_mermaid_blocks(content) - - if not mermaid_blocks: - return "No mermaid diagrams found in the file" - - # Validate each mermaid diagram sequentially to avoid segfaults - errors = [] - for i, (line_start, diagram_content) in enumerate(mermaid_blocks, 1): - error_msg = await validate_single_diagram(diagram_content, i, line_start) - if error_msg: - errors.append("\n") - errors.append(error_msg) - - # if errors: - # logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") - - if errors: - return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) - else: - return "All mermaid diagrams in file: " + relative_path + " are syntax correct" - - except Exception as e: - return f"Error processing file: {str(e)}" - - -def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]: - """ - Extract all mermaid code blocks from markdown content. - - Returns: - List of tuples containing (line_number, diagram_content) - """ - mermaid_blocks = [] - lines = content.split('\n') - i = 0 - - while i < len(lines): - line = lines[i].strip() - - # Look for mermaid code block start - if line == '```mermaid' or line.startswith('```mermaid'): - start_line = i + 1 - diagram_lines = [] - i += 1 - - # Collect lines until we find the closing ``` - while i < len(lines): - if lines[i].strip() == '```': - break - diagram_lines.append(lines[i]) - i += 1 - - if diagram_lines: # Only add non-empty diagrams - diagram_content = '\n'.join(diagram_lines) - mermaid_blocks.append((start_line, diagram_content)) - - i += 1 - - return mermaid_blocks - - -async def validate_single_diagram(diagram_content: str, diagram_num: int, line_start: int) -> str: - """ - Validate a single mermaid diagram. - - Args: - diagram_content: The mermaid diagram content - diagram_num: Diagram number for error reporting - line_start: Starting line number in the file - - Returns: - Error message if invalid, empty string if valid - """ - import sys - import os - from io import StringIO - - core_error = "" - - try: - from mermaid_parser.parser import parse_mermaid_py - # logger.debug("Using mermaid-parser-py to validate mermaid diagrams") - - try: - # Redirect stderr to suppress mermaid parser JavaScript errors - old_stderr = sys.stderr - sys.stderr = open(os.devnull, 'w') - - try: - json_output = await parse_mermaid_py(diagram_content) - finally: - # Restore stderr - sys.stderr.close() - sys.stderr = old_stderr - except Exception as e: - error_str = str(e) - - # Extract the core error information from the exception message - # Look for the pattern that contains "Parse error on line X:" - error_pattern = r"Error:(.*?)(?=Stack Trace:|$)" - match = re.search(error_pattern, error_str, re.DOTALL) - - if match: - core_error = match.group(0).strip() - core_error = core_error - else: - logger.error(f"No match found for error pattern, fallback to mermaid-py\n{error_str}") - logger.error(f"Traceback: {traceback.format_exc()}") - raise Exception(error_str) - - except Exception as e: - logger.warning("Using mermaid-py to validate mermaid diagrams") - try: - import mermaid as md - # Create Mermaid object and check response - render = md.Mermaid(diagram_content) - core_error = render.svg_response.text - - except Exception as e: - return f" Diagram {diagram_num}: Exception during validation - {str(e)}" - - # Check if response indicates a parse error - if core_error: - # Extract line number from parse error and calculate actual line in markdown file - line_match = re.search(r'line (\d+)', core_error) - if line_match: - error_line_in_diagram = int(line_match.group(1)) - actual_line_in_file = line_start + error_line_in_diagram - newline = '\n' - return f"Diagram {diagram_num}: Parse error on line {actual_line_in_file}:{newline}{newline.join(core_error.split(newline)[1:])}" - else: - return f"Diagram {diagram_num}: {core_error}" - - return "" # No error - - -if __name__ == "__main__": - # Test with the provided file - import asyncio - test_file = "output/docs/SWE_agent-docs/agent_hooks.md" - result = asyncio.run(validate_mermaid_diagrams(test_file, "agent_hooks.md")) - print(result) \ No newline at end of file diff --git a/code2test/src/fe/__init__.py b/code2test/src/fe/__init__.py deleted file mode 100644 index 9fa1624..0000000 --- a/code2test/src/fe/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -""" -CodeWiki Frontend Module - -Web interface components for the documentation generation service. -""" - -from .web_app import app, main -from .models import JobStatus, JobStatusResponse, RepositorySubmission, CacheEntry -from .cache_manager import CacheManager -from .background_worker import BackgroundWorker -from .github_processor import GitHubRepoProcessor -from .routes import WebRoutes - -__all__ = [ - 'app', - 'main', - 'JobStatus', - 'JobStatusResponse', - 'RepositorySubmission', - 'CacheEntry', - 'CacheManager', - 'BackgroundWorker', - 'GitHubRepoProcessor', - 'WebRoutes' -] \ No newline at end of file diff --git a/code2test/src/fe/background_worker.py b/code2test/src/fe/background_worker.py deleted file mode 100644 index 671b753..0000000 --- a/code2test/src/fe/background_worker.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -""" -Background worker for processing documentation generation jobs. -""" - -import os -import json -import time -import threading -import subprocess -import asyncio -from datetime import datetime -from pathlib import Path -from queue import Queue -from typing import Dict -from dataclasses import asdict - -from code2test.src.be.documentation_generator import DocumentationGenerator -from code2test.src.config import Config, MAIN_MODEL -from .models import JobStatus -from .cache_manager import CacheManager -from .github_processor import GitHubRepoProcessor -from .config import WebAppConfig -from code2test.src.utils import file_manager - -class BackgroundWorker: - """Background worker for processing documentation generation jobs.""" - - def __init__(self, cache_manager: CacheManager, temp_dir: str = None): - self.cache_manager = cache_manager - self.temp_dir = temp_dir or WebAppConfig.TEMP_DIR - self.running = False - self.processing_queue = Queue(maxsize=WebAppConfig.QUEUE_SIZE) - self.job_status: Dict[str, JobStatus] = {} - self.jobs_file = Path(WebAppConfig.CACHE_DIR) / "jobs.json" - self.load_job_statuses() - - def start(self): - """Start the background worker thread.""" - if not self.running: - self.running = True - thread = threading.Thread(target=self._worker_loop, daemon=True) - thread.start() - print("Background worker started") - - def stop(self): - """Stop the background worker.""" - self.running = False - - def add_job(self, job_id: str, job: JobStatus): - """Add a job to the processing queue.""" - self.job_status[job_id] = job - self.processing_queue.put(job_id) - - def get_job_status(self, job_id: str) -> JobStatus: - """Get job status by ID.""" - return self.job_status.get(job_id) - - def get_all_jobs(self) -> Dict[str, JobStatus]: - """Get all job statuses.""" - return self.job_status - - def load_job_statuses(self): - """Load job statuses from disk.""" - if not self.jobs_file.exists(): - # Try to reconstruct from cache if no job file exists - self._reconstruct_jobs_from_cache() - return - - try: - data = file_manager.load_json(self.jobs_file) - - for job_id, job_data in data.items(): - # Only load completed jobs to avoid inconsistent state - if job_data.get('status') == 'completed': - self.job_status[job_id] = JobStatus( - job_id=job_data['job_id'], - repo_url=job_data['repo_url'], - status=job_data['status'], - created_at=datetime.fromisoformat(job_data['created_at']), - started_at=datetime.fromisoformat(job_data['started_at']) if job_data.get('started_at') else None, - completed_at=datetime.fromisoformat(job_data['completed_at']) if job_data.get('completed_at') else None, - error_message=job_data.get('error_message'), - progress=job_data.get('progress', ''), - docs_path=job_data.get('docs_path') - ) - print(f"Loaded {len([j for j in self.job_status.values() if j.status == 'completed'])} completed jobs from disk") - except Exception as e: - print(f"Error loading job statuses: {e}") - - def _reconstruct_jobs_from_cache(self): - """Reconstruct job statuses from cache entries for backward compatibility.""" - try: - cache_entries = self.cache_manager.cache_index - reconstructed_count = 0 - - for repo_hash, cache_entry in cache_entries.items(): - # Extract repo info to create job_id - from .github_processor import GitHubRepoProcessor - try: - repo_info = GitHubRepoProcessor.get_repo_info(cache_entry.repo_url) - job_id = repo_info['full_name'].replace('/', '--') - - # Only add if job doesn't already exist - if job_id not in self.job_status: - self.job_status[job_id] = JobStatus( - job_id=job_id, - repo_url=cache_entry.repo_url, - status='completed', - created_at=cache_entry.created_at, - completed_at=cache_entry.created_at, - docs_path=cache_entry.docs_path, - progress="Reconstructed from cache" - ) - reconstructed_count += 1 - except Exception as e: - print(f"Failed to reconstruct job for {cache_entry.repo_url}: {e}") - - if reconstructed_count > 0: - print(f"Reconstructed {reconstructed_count} job statuses from cache") - self.save_job_statuses() - - except Exception as e: - print(f"Error reconstructing jobs from cache: {e}") - - def save_job_statuses(self): - """Save job statuses to disk.""" - try: - # Ensure cache directory exists - self.jobs_file.parent.mkdir(parents=True, exist_ok=True) - - data = {} - for job_id, job in self.job_status.items(): - data[job_id] = { - 'job_id': job.job_id, - 'repo_url': job.repo_url, - 'status': job.status, - 'created_at': job.created_at.isoformat(), - 'started_at': job.started_at.isoformat() if job.started_at else None, - 'completed_at': job.completed_at.isoformat() if job.completed_at else None, - 'error_message': job.error_message, - 'progress': job.progress, - 'docs_path': job.docs_path - } - - file_manager.save_json(data, self.jobs_file) - except Exception as e: - print(f"Error saving job statuses: {e}") - - def _worker_loop(self): - """Main worker loop.""" - while self.running: - try: - if not self.processing_queue.empty(): - job_id = self.processing_queue.get(timeout=1) - self._process_job(job_id) - else: - time.sleep(1) - except Exception as e: - print(f"Worker error: {e}") - time.sleep(1) - - def _process_job(self, job_id: str): - """Process a single documentation generation job.""" - if job_id not in self.job_status: - return - - job = self.job_status[job_id] - - try: - # Update job status - job.status = 'processing' - job.started_at = datetime.now() - job.progress = "Starting repository clone..." - job.main_model = MAIN_MODEL - - # Check cache first - cached_docs = self.cache_manager.get_cached_docs(job.repo_url) - if cached_docs and Path(cached_docs).exists(): - job.status = 'completed' - job.completed_at = datetime.now() - job.docs_path = cached_docs - job.progress = "Documentation retrieved from cache" - if not job.main_model: # Only set if not already set - job.main_model = MAIN_MODEL - - # Save job status to disk - self.save_job_statuses() - - print(f"Job {job_id}: Using cached documentation") - return - - # Clone repository - repo_info = GitHubRepoProcessor.get_repo_info(job.repo_url) - # Use repo full name for temp directory (already URL-safe since job_id is URL-safe) - temp_repo_dir = os.path.join(self.temp_dir, job_id) - - job.progress = f"Cloning repository {repo_info['full_name']}..." - - if not GitHubRepoProcessor.clone_repository(repo_info['clone_url'], temp_repo_dir, job.commit_id): - raise Exception("Failed to clone repository") - - # Generate documentation - job.progress = "Analyzing repository structure..." - - # Create config for documentation generation (using env vars) - import argparse - args = argparse.Namespace(repo_path=temp_repo_dir) - config = Config.from_args(args) - # Override docs_dir with job-specific directory - config.docs_dir = os.path.join("output", "docs", f"{job_id}-docs") - - job.progress = "Generating documentation..." - - # Generate documentation - doc_generator = DocumentationGenerator(config, job.commit_id) - - # Run the async documentation generation in a new event loop - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(doc_generator.run()) - finally: - loop.close() - - # Cache the results - docs_path = os.path.abspath(config.docs_dir) - self.cache_manager.add_to_cache(job.repo_url, docs_path) - - # Update job status - job.status = 'completed' - job.completed_at = datetime.now() - job.docs_path = docs_path - job.progress = "Documentation generation completed" - - # Save job status to disk - self.save_job_statuses() - - print(f"Job {job_id}: Documentation generated successfully") - - except Exception as e: - # Update job status with error - job.status = 'failed' - job.completed_at = datetime.now() - job.error_message = str(e) - job.progress = f"Failed: {str(e)}" - - print(f"Job {job_id}: Failed with error: {e}") - - finally: - # Cleanup temporary repository - if 'temp_repo_dir' in locals() and os.path.exists(temp_repo_dir): - try: - subprocess.run(['rm', '-rf', temp_repo_dir], check=True) - except Exception as e: - print(f"Failed to cleanup temp directory: {e}") \ No newline at end of file diff --git a/code2test/src/fe/cache_manager.py b/code2test/src/fe/cache_manager.py deleted file mode 100644 index b6ab49b..0000000 --- a/code2test/src/fe/cache_manager.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -""" -Cache management for documentation generation results. -""" - -import hashlib -from datetime import datetime, timedelta -from pathlib import Path -from typing import Optional, Dict - -from .models import CacheEntry -from .config import WebAppConfig -from code2test.src.utils import file_manager - - -class CacheManager: - """Manages documentation cache.""" - - def __init__(self, cache_dir: str = None, cache_expiry_days: int = None): - self.cache_dir = Path(cache_dir or WebAppConfig.CACHE_DIR) - self.cache_expiry_days = cache_expiry_days or WebAppConfig.CACHE_EXPIRY_DAYS - self.cache_dir.mkdir(parents=True, exist_ok=True) - self.cache_index: Dict[str, CacheEntry] = {} - self.load_cache_index() - - def load_cache_index(self): - """Load cache index from disk.""" - index_file = self.cache_dir / "cache_index.json" - if index_file.exists(): - try: - data = file_manager.load_json(index_file) - for key, value in data.items(): - self.cache_index[key] = CacheEntry( - repo_url=value['repo_url'], - repo_url_hash=value['repo_url_hash'], - docs_path=value['docs_path'], - created_at=datetime.fromisoformat(value['created_at']), - last_accessed=datetime.fromisoformat(value['last_accessed']) - ) - except Exception as e: - print(f"Error loading cache index: {e}") - - def save_cache_index(self): - """Save cache index to disk.""" - index_file = self.cache_dir / "cache_index.json" - try: - data = {} - for key, entry in self.cache_index.items(): - data[key] = { - 'repo_url': entry.repo_url, - 'repo_url_hash': entry.repo_url_hash, - 'docs_path': entry.docs_path, - 'created_at': entry.created_at.isoformat(), - 'last_accessed': entry.last_accessed.isoformat() - } - - file_manager.save_json(data, index_file) - except Exception as e: - print(f"Error saving cache index: {e}") - - def get_repo_hash(self, repo_url: str) -> str: - """Generate hash for repository URL.""" - return hashlib.sha256(repo_url.encode()).hexdigest()[:16] - - def get_cached_docs(self, repo_url: str) -> Optional[str]: - """Get cached documentation path if available.""" - repo_hash = self.get_repo_hash(repo_url) - - if repo_hash in self.cache_index: - entry = self.cache_index[repo_hash] - - # Check if cache is still valid - if datetime.now() - entry.created_at < timedelta(days=self.cache_expiry_days): - # Update last accessed - entry.last_accessed = datetime.now() - self.save_cache_index() - return entry.docs_path - else: - # Cache expired, remove it - self.remove_from_cache(repo_url) - - return None - - def add_to_cache(self, repo_url: str, docs_path: str): - """Add documentation to cache.""" - repo_hash = self.get_repo_hash(repo_url) - now = datetime.now() - - self.cache_index[repo_hash] = CacheEntry( - repo_url=repo_url, - repo_url_hash=repo_hash, - docs_path=docs_path, - created_at=now, - last_accessed=now - ) - - self.save_cache_index() - - def remove_from_cache(self, repo_url: str): - """Remove documentation from cache.""" - repo_hash = self.get_repo_hash(repo_url) - if repo_hash in self.cache_index: - del self.cache_index[repo_hash] - self.save_cache_index() - - def cleanup_expired_cache(self): - """Remove expired cache entries.""" - expired_entries = [] - cutoff = datetime.now() - timedelta(days=self.cache_expiry_days) - - for repo_hash, entry in self.cache_index.items(): - if entry.created_at < cutoff: - expired_entries.append(repo_hash) - - for repo_hash in expired_entries: - del self.cache_index[repo_hash] - - if expired_entries: - self.save_cache_index() \ No newline at end of file diff --git a/code2test/src/fe/config.py b/code2test/src/fe/config.py deleted file mode 100644 index c77d3d4..0000000 --- a/code2test/src/fe/config.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -""" -Configuration settings for the CodeWiki web application. -""" - -import os -from pathlib import Path - - -class WebAppConfig: - """Configuration class for web application settings.""" - - # Directories - CACHE_DIR = "./output/cache" - TEMP_DIR = "./output/temp" - OUTPUT_DIR = "./output" - - # Queue settings - QUEUE_SIZE = 100 - - # Cache settings - CACHE_EXPIRY_DAYS = 365 - - # Job cleanup settings - JOB_CLEANUP_HOURS = 24000 - RETRY_COOLDOWN_MINUTES = 3 - - # Server settings - DEFAULT_HOST = "127.0.0.1" - DEFAULT_PORT = 8000 - - # Git settings - CLONE_TIMEOUT = 300 - CLONE_DEPTH = 1 - - @classmethod - def ensure_directories(cls): - """Ensure all required directories exist.""" - directories = [ - cls.CACHE_DIR, - cls.TEMP_DIR, - cls.OUTPUT_DIR - ] - - for directory in directories: - Path(directory).mkdir(parents=True, exist_ok=True) - - @classmethod - def get_absolute_path(cls, path: str) -> str: - """Get absolute path for a given relative path.""" - return os.path.abspath(path) \ No newline at end of file diff --git a/code2test/src/fe/github_processor.py b/code2test/src/fe/github_processor.py deleted file mode 100644 index bc9084d..0000000 --- a/code2test/src/fe/github_processor.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -""" -GitHub repository processing utilities. -""" - -import os -import subprocess -from typing import Dict -from urllib.parse import urlparse - -from .config import WebAppConfig - - -class GitHubRepoProcessor: - """Handles GitHub repository processing.""" - - @staticmethod - def is_valid_github_url(url: str) -> bool: - """Validate if the URL is a valid GitHub repository URL.""" - try: - parsed = urlparse(url) - if parsed.netloc.lower() not in ['github.com', 'www.github.com']: - return False - - path_parts = parsed.path.strip('/').split('/') - if len(path_parts) < 2: - return False - - # Check if it's a valid repo path (owner/repo) - return len(path_parts) >= 2 and all(part for part in path_parts[:2]) - except Exception: - return False - - @staticmethod - def get_repo_info(url: str) -> Dict[str, str]: - """Extract repository information from GitHub URL.""" - parsed = urlparse(url) - path_parts = parsed.path.strip('/').split('/') - - owner = path_parts[0] - repo = path_parts[1] - - # Remove .git suffix if present - if repo.endswith('.git'): - repo = repo[:-4] - - return { - 'owner': owner, - 'repo': repo, - 'full_name': f"{owner}/{repo}", - 'clone_url': f"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/{owner}/{repo}.git" - } - - @staticmethod - def clone_repository(clone_url: str, target_dir: str, commit_id: str = None) -> bool: - """Clone a GitHub repository to the target directory, optionally checking out a specific commit.""" - try: - # Ensure target directory exists - os.makedirs(os.path.dirname(target_dir), exist_ok=True) - - # If specific commit is requested, don't use shallow clone - if commit_id: - # Clone full repository to access specific commit - result = subprocess.run([ - 'git', 'clone', clone_url, target_dir - ], capture_output=True, text=True, timeout=WebAppConfig.CLONE_TIMEOUT) - - if result.returncode != 0: - print(f"Error cloning repository: {result.stderr}") - return False - - # Checkout specific commit - result = subprocess.run([ - 'git', 'checkout', commit_id - ], cwd=target_dir, capture_output=True, text=True, timeout=30) - - if result.returncode != 0: - print(f"Error checking out commit {commit_id}: {result.stderr}") - return False - else: - # Clone repository with shallow depth (default behavior) - result = subprocess.run([ - 'git', 'clone', '--depth', str(WebAppConfig.CLONE_DEPTH), clone_url, target_dir - ], capture_output=True, text=True, timeout=WebAppConfig.CLONE_TIMEOUT) - - if result.returncode != 0: - print(f"Error cloning repository: {result.stderr}") - return False - - return True - except Exception as e: - print(f"Error cloning repository: {e}") - return False \ No newline at end of file diff --git a/code2test/src/fe/models.py b/code2test/src/fe/models.py deleted file mode 100644 index 253d736..0000000 --- a/code2test/src/fe/models.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -""" -Data models and classes for the CodeWiki web application. -""" - -from datetime import datetime -from typing import Optional -from dataclasses import dataclass -from pydantic import BaseModel, HttpUrl - - -class RepositorySubmission(BaseModel): - """Pydantic model for repository submission form.""" - repo_url: HttpUrl - - -class JobStatusResponse(BaseModel): - """Pydantic model for job status API response.""" - job_id: str - repo_url: str - status: str - created_at: datetime - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - progress: str = "" - docs_path: Optional[str] = None - main_model: Optional[str] = None - commit_id: Optional[str] = None - - -@dataclass -class JobStatus: - """Tracks the status of a documentation generation job.""" - job_id: str - repo_url: str - status: str # 'queued', 'processing', 'completed', 'failed' - created_at: datetime - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - progress: str = "" - docs_path: Optional[str] = None - main_model: Optional[str] = None - commit_id: Optional[str] = None - - -@dataclass -class CacheEntry: - """Represents a cached documentation result.""" - repo_url: str - repo_url_hash: str - docs_path: str - created_at: datetime - last_accessed: datetime \ No newline at end of file diff --git a/code2test/src/fe/routes.py b/code2test/src/fe/routes.py deleted file mode 100644 index 6a5e2e2..0000000 --- a/code2test/src/fe/routes.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 -""" -FastAPI route handlers for the CodeWiki web application. -""" - -from datetime import datetime, timedelta -from pathlib import Path -from dataclasses import asdict - -from traceback import format_exc - -from fastapi import Form, HTTPException, Request, status -from fastapi.responses import HTMLResponse, RedirectResponse - -from .models import JobStatus, JobStatusResponse -from .github_processor import GitHubRepoProcessor -from .background_worker import BackgroundWorker -from .cache_manager import CacheManager -from .templates import WEB_INTERFACE_TEMPLATE -from .template_utils import render_template -from .config import WebAppConfig -from code2test.src.utils import file_manager - - -class WebRoutes: - """Handles all web routes for the application.""" - - def __init__(self, background_worker: BackgroundWorker, cache_manager: CacheManager): - self.background_worker = background_worker - self.cache_manager = cache_manager - - async def index_get(self, request: Request) -> HTMLResponse: - """Main page with form for submitting GitHub repositories.""" - # Clean up old jobs before displaying - # self.cleanup_old_jobs() - - # Get recent jobs (last 10) - all_jobs = self.background_worker.get_all_jobs() - recent_jobs = sorted( - all_jobs.values(), - key=lambda x: x.created_at, - reverse=True - )[:100] - - context = { - "message": None, - "message_type": None, - "repo_url": "", - "commit_id": "", - "recent_jobs": recent_jobs - } - - return HTMLResponse(content=render_template(WEB_INTERFACE_TEMPLATE, context)) - - async def index_post(self, request: Request, repo_url: str = Form(...), commit_id: str = Form("")) -> HTMLResponse: - """Handle repository submission.""" - # Clean up old jobs before processing - self.cleanup_old_jobs() - - message = None - message_type = None - - repo_url = repo_url.strip() - commit_id = commit_id.strip() if commit_id else "" - - if not repo_url: - message = "Please enter a GitHub repository URL" - message_type = "error" - elif not GitHubRepoProcessor.is_valid_github_url(repo_url): - message = "Please enter a valid GitHub repository URL" - message_type = "error" - else: - # Normalize the repo URL for comparison - normalized_repo_url = self._normalize_github_url(repo_url) - - # Get repo info for job ID generation - repo_info = GitHubRepoProcessor.get_repo_info(normalized_repo_url) - job_id = self._repo_full_name_to_job_id(repo_info['full_name']) - - # Check if already in queue, processing, or recently failed - existing_job = self.background_worker.get_job_status(job_id) - recent_cutoff = datetime.now() - timedelta(minutes=WebAppConfig.RETRY_COOLDOWN_MINUTES) - - if existing_job: - if existing_job.status in ['queued', 'processing']: - pass # Will handle below - elif existing_job.status == 'failed' and existing_job.created_at > recent_cutoff: - pass # Will handle below - else: - existing_job = None # Job is old or completed, can reuse - - if existing_job: - if existing_job.status in ['queued', 'processing']: - message = f"Repository is already being processed (Job ID: {existing_job.job_id})" - else: - message = f"Repository recently failed processing. Please wait a few minutes before retrying (Job ID: {existing_job.job_id})" - message_type = "error" - else: - # Check cache - cached_docs = self.cache_manager.get_cached_docs(normalized_repo_url) - if cached_docs and Path(cached_docs).exists(): - message = "Documentation found in cache! Redirecting to view..." - message_type = "success" - # Create a dummy completed job for display - job = JobStatus( - job_id=job_id, - repo_url=normalized_repo_url, # Use normalized URL - status='completed', - created_at=datetime.now(), - completed_at=datetime.now(), - docs_path=cached_docs, - progress="Retrieved from cache", - commit_id=commit_id if commit_id else None - ) - self.background_worker.job_status[job_id] = job - else: - # Add to queue - try: - job = JobStatus( - job_id=job_id, - repo_url=normalized_repo_url, # Use normalized URL - status='queued', - created_at=datetime.now(), - progress="Waiting in queue...", - commit_id=commit_id if commit_id else None - ) - - self.background_worker.add_job(job_id, job) - message = f"Repository added to processing queue! Job ID: {job_id}" - message_type = "success" - repo_url = "" # Clear form - - except Exception as e: - message = f"Failed to add repository to queue: {str(e)}\n{format_exc()}" - message_type = "error" - - # Get recent jobs (last 10) - all_jobs = self.background_worker.get_all_jobs() - recent_jobs = sorted( - all_jobs.values(), - key=lambda x: x.created_at, - reverse=True - ) - - context = { - "message": message, - "message_type": message_type, - "repo_url": repo_url or "", - "commit_id": commit_id or "", - "recent_jobs": recent_jobs - } - - return HTMLResponse(content=render_template(WEB_INTERFACE_TEMPLATE, context)) - - async def get_job_status(self, job_id: str) -> JobStatusResponse: - """API endpoint to get job status.""" - job = self.background_worker.get_job_status(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - return JobStatusResponse(**asdict(job)) - - async def view_docs(self, job_id: str) -> RedirectResponse: - """View generated documentation.""" - job = self.background_worker.get_job_status(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - if job.status != 'completed' or not job.docs_path: - raise HTTPException(status_code=404, detail="Documentation not available") - - docs_path = Path(job.docs_path) - if not docs_path.exists(): - raise HTTPException(status_code=404, detail="Documentation files not found") - - # Redirect to the documentation viewer - return RedirectResponse(url=f"/static-docs/{job_id}/", status_code=status.HTTP_302_FOUND) - - async def serve_generated_docs(self, job_id: str, filename: str = "overview.md") -> HTMLResponse: - """Serve generated documentation files.""" - job = self.background_worker.get_job_status(job_id) - docs_path = None - repo_url = None - - if job: - # Job status exists - use it - if job.status != 'completed' or not job.docs_path: - raise HTTPException(status_code=404, detail="Documentation not available") - docs_path = Path(job.docs_path) - repo_url = job.repo_url - else: - # No job status - try to find documentation in cache by job_id - # Convert job_id back to repo full name and construct potential paths - repo_full_name = self._job_id_to_repo_full_name(job_id) - potential_repo_url = f"/{repo_full_name}" - - # Check if documentation exists in cache - cached_docs = self.cache_manager.get_cached_docs(potential_repo_url) - if cached_docs and Path(cached_docs).exists(): - docs_path = Path(cached_docs) - repo_url = potential_repo_url - - # Recreate job status for consistency - job = JobStatus( - job_id=job_id, - repo_url=potential_repo_url, - status='completed', - created_at=datetime.now(), - completed_at=datetime.now(), - docs_path=cached_docs, - progress="Loaded from cache", - commit_id=None # No commit info available from cache - ) - self.background_worker.job_status[job_id] = job - self.background_worker.save_job_statuses() - else: - raise HTTPException(status_code=404, detail="Documentation not found") - - if not docs_path or not docs_path.exists(): - raise HTTPException(status_code=404, detail="Documentation files not found") - - # Load module tree - module_tree = None - module_tree_file = docs_path / "module_tree.json" - if module_tree_file.exists(): - try: - module_tree = file_manager.load_json(module_tree_file) - except Exception: - pass - - # Load metadata - metadata = None - metadata_file = docs_path / "metadata.json" - if metadata_file.exists(): - try: - metadata = file_manager.load_json(metadata_file) - except Exception: - pass - - # Serve the requested file - file_path = docs_path / filename - if not file_path.exists(): - raise HTTPException(status_code=404, detail=f"File {filename} not found") - - try: - content = file_manager.load_text(file_path) - - # Convert markdown to HTML (reuse from visualise_docs.py) - from .visualise_docs import markdown_to_html, get_file_title - from .templates import DOCS_VIEW_TEMPLATE - - html_content = markdown_to_html(content) - title = get_file_title(file_path) - - context = { - "repo_name": repo_url.split("/")[-1], - "title": title, - "content": html_content, - "navigation": module_tree, - "current_page": filename, - "job_id": job_id, - "metadata": metadata - } - - return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - - except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading {filename}: {e}\n{format_exc()}") - - def _normalize_github_url(self, url: str) -> str: - """Normalize GitHub URL for consistent comparison.""" - try: - # Get repo info to standardize the URL format - repo_info = GitHubRepoProcessor.get_repo_info(url) - return f"/{repo_info['full_name']}" - except Exception: - # Fallback to basic normalization - return url.rstrip('/').lower() - - def _repo_full_name_to_job_id(self, full_name: str) -> str: - """Convert repo full name to URL-safe job ID.""" - return full_name.replace('/', '--') - - def _job_id_to_repo_full_name(self, job_id: str) -> str: - """Convert job ID back to repo full name.""" - return job_id.replace('--', '/') - - def cleanup_old_jobs(self): - """Clean up old job status entries.""" - cutoff = datetime.now() - timedelta(hours=WebAppConfig.JOB_CLEANUP_HOURS) - all_jobs = self.background_worker.get_all_jobs() - expired_jobs = [ - job_id for job_id, job in all_jobs.items() - if job.created_at < cutoff and job.status in ['completed', 'failed'] - ] - - for job_id in expired_jobs: - if job_id in self.background_worker.job_status: - del self.background_worker.job_status[job_id] \ No newline at end of file diff --git a/code2test/src/fe/template_utils.py b/code2test/src/fe/template_utils.py deleted file mode 100644 index 4d9d7e7..0000000 --- a/code2test/src/fe/template_utils.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -""" -Template utilities for FastAPI applications using Jinja2. -""" - -from jinja2 import Environment, BaseLoader, select_autoescape -from typing import Dict, Any - - -class StringTemplateLoader(BaseLoader): - """Custom Jinja2 loader for string templates.""" - - def __init__(self, template_string: str): - self.template_string = template_string - - def get_source(self, environment, template): - return self.template_string, None, lambda: True - - -def render_template(template: str, context: Dict[str, Any]) -> str: - """ - Render template using Jinja2. - - Args: - template: HTML template string with Jinja2 syntax - context: Dictionary of variables to substitute - - Returns: - Rendered HTML string - """ - # Create Jinja2 environment with string template - env = Environment( - loader=StringTemplateLoader(template), - autoescape=select_autoescape(['html', 'xml']), - trim_blocks=True, - lstrip_blocks=True - ) - - # Get template and render - jinja_template = env.get_template('') - return jinja_template.render(**context) - - -def render_navigation(module_tree: Dict[str, Any], current_page: str = "") -> str: - """ - Render navigation HTML from module tree structure. - - Args: - module_tree: Dictionary representing the module tree - current_page: Current page filename for highlighting - - Returns: - HTML string for navigation - """ - if not module_tree: - return "" - - nav_template = """ - {%- for section_key, section_data in module_tree.items() %} -
-

{{ section_key.replace('_', ' ').title() }}

- {%- if section_data.get('components') %} - Overview - {%- endif %} - {%- if section_data.get('children') %} - {%- for child_key, child_data in section_data['children'].items() %} -
- {{ child_key.replace('_', ' ').title() }} -
- {%- endfor %} - {%- endif %} -
- {%- endfor %} - """ - - return render_template(nav_template, { - 'module_tree': module_tree, - 'current_page': current_page - }) - - -def render_job_list(jobs: list) -> str: - """ - Render job list HTML. - - Args: - jobs: List of job objects - - Returns: - HTML string for job list - """ - if not jobs: - return "" - - job_list_template = """ - {%- for job in jobs %} -
-
-
{{ job.repo_url }}
-
{{ job.status.title() }}
-
- {%- if job.progress %} -
{{ job.progress }}
- {%- endif %} - {%- if job.status == 'completed' and job.docs_path %} -
- View Documentation -
- {%- endif %} -
- {%- endfor %} - """ - - return render_template(job_list_template, {'jobs': jobs}) \ No newline at end of file diff --git a/code2test/src/fe/templates.py b/code2test/src/fe/templates.py deleted file mode 100644 index 763562c..0000000 --- a/code2test/src/fe/templates.py +++ /dev/null @@ -1,680 +0,0 @@ -#!/usr/bin/env python3 -""" -HTML templates for the CodeWiki web application. -""" - -# Web interface HTML template -WEB_INTERFACE_TEMPLATE = """ - - - - - - CodeWiki - GitHub Repository Documentation Generator - - - -
-
-

📚 CodeWiki

-

Generate comprehensive documentation for any GitHub repository

-
- -
- {% if message %} -
- {{ message }} -
- {% endif %} - -
-
- - -
- -
- - -
- - -
- - {% if recent_jobs %} -
-

Recent Jobs

- {% for job in recent_jobs %} -
-
-
{{ job.repo_url }}
-
{{ job.status }}
-
-
{{ job.progress }}
- {% if job.main_model %} -
- Generated with: {{ job.main_model }} -
- {% endif %} -
- View Documentation -
-
- {% endfor %} -
- {% endif %} -
-
- - - - -""" - -# HTML template for the documentation pages -DOCS_VIEW_TEMPLATE = """ - - - - - - {{ title }} - - - - -
- - -
-
- {{ content | safe }} -
-
-
- - - - -""" \ No newline at end of file diff --git a/code2test/src/fe/visualise_docs.py b/code2test/src/fe/visualise_docs.py deleted file mode 100644 index 42538ea..0000000 --- a/code2test/src/fe/visualise_docs.py +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple documentation server for hosting documentation folders. - -This server serves documentation folders with the following structure: -- overview.md: The main overview document -- module_tree.json: Hierarchical structure of modules -- Various .md files for different modules - -Usage: - python docs_server.py --docs-folder path/to/docs --port 8080 -""" - -import argparse -import sys -from pathlib import Path -from typing import Dict, Optional - -from fastapi import FastAPI, HTTPException -from fastapi.responses import HTMLResponse -from fastapi.staticfiles import StaticFiles -from markdown_it import MarkdownIt - -from .template_utils import render_template -from .templates import DOCS_VIEW_TEMPLATE -from code2test.src.utils import file_manager - -app = FastAPI(title="Documentation Server", description="Simple documentation server for hosting markdown documentation folders") - -# Global variables to store configuration -DOCS_FOLDER = None -MODULE_TREE = None - -def initialize_globals(): - """Initialize global variables from environment or command line args if not already set.""" - global DOCS_FOLDER, MODULE_TREE - - if DOCS_FOLDER is None: - # Try to get from environment variable or use a default - import os - docs_folder_path = os.environ.get('DOCS_FOLDER') - if docs_folder_path and Path(docs_folder_path).exists(): - DOCS_FOLDER = docs_folder_path - MODULE_TREE = load_module_tree(Path(docs_folder_path)) - else: - # If no environment variable, we need to handle this gracefully - # The FastAPI endpoints will need to check if DOCS_FOLDER is None - pass - -# Markdown parser -md = MarkdownIt() - - -def load_module_tree(docs_folder: Path) -> Optional[Dict]: - """Load the module tree structure from module_tree.json.""" - tree_file = docs_folder / "module_tree.json" - if not tree_file.exists(): - print(f"Warning: module_tree.json not found in {docs_folder}") - return None - - try: - return file_manager.load_json(tree_file) - except Exception as e: - print(f"Error loading module_tree.json: {e}") - return None - - -def markdown_to_html(content: str) -> str: - """Convert markdown content to HTML, with special handling for mermaid diagrams.""" - # First, convert markdown to HTML - html = md.render(content) - - # Post-process to ensure mermaid code blocks are properly formatted - # Look for code blocks with language-mermaid class and convert them to mermaid divs - import re - - # Pattern to match mermaid code blocks - pattern = r'
(.*?)
' - - def replace_mermaid(match): - mermaid_code = match.group(1) - # Decode HTML entities that might have been encoded - import html - mermaid_code = html.unescape(mermaid_code) - return f'
{mermaid_code}
' - - # Replace mermaid code blocks with proper mermaid divs - html = re.sub(pattern, replace_mermaid, html, flags=re.DOTALL) - - return html - - -def get_file_title(file_path: Path) -> str: - """Extract title from markdown file, fallback to filename.""" - try: - content = file_manager.load_text(file_path) - first_line = content.split('\n')[0].strip() - if first_line.startswith('# '): - return first_line[2:].strip() - except Exception: - pass - - # Fallback to filename without extension - return file_path.stem.replace('_', ' ').title() - - -@app.get("/", response_class=HTMLResponse) -async def index(): - """Serve the overview page as the main page.""" - initialize_globals() - - if DOCS_FOLDER is None: - raise HTTPException(status_code=500, detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.") - - overview_file = Path(DOCS_FOLDER) / "overview.md" - - if not overview_file.exists(): - raise HTTPException(status_code=404, detail="overview.md not found in the documentation folder") - - try: - content = file_manager.load_text(overview_file) - - html_content = markdown_to_html(content) - title = get_file_title(overview_file) - - context = { - "title": title, - "content": html_content, - "navigation": MODULE_TREE, - "current_page": "overview.md" - } - - return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - - except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading overview.md: {e}") - - -@app.get("/{filename:path}", response_class=HTMLResponse) -async def serve_doc(filename: str): - """Serve individual documentation files.""" - initialize_globals() - - if DOCS_FOLDER is None: - raise HTTPException(status_code=500, detail="Documentation folder not configured. Please set DOCS_FOLDER environment variable or run with --docs-folder argument.") - - # Security check: ensure we're only serving .md files and they exist in the docs folder - if not filename.endswith('.md'): - raise HTTPException(status_code=404, detail="Only markdown files are supported") - - file_path = Path(DOCS_FOLDER) / filename - - # Ensure the file is within the docs folder (prevent directory traversal) - try: - file_path = file_path.resolve() - docs_folder_resolved = Path(DOCS_FOLDER).resolve() - if not str(file_path).startswith(str(docs_folder_resolved)): - raise HTTPException(status_code=403, detail="Access denied") - except Exception: - raise HTTPException(status_code=403, detail="Invalid file path") - - if not file_path.exists(): - raise HTTPException(status_code=404, detail=f"File {filename} not found") - - try: - content = file_manager.load_text(file_path) - - html_content = markdown_to_html(content) - title = get_file_title(file_path) - - context = { - "title": title, - "content": html_content, - "navigation": MODULE_TREE, - "current_page": filename - } - - return HTMLResponse(content=render_template(DOCS_VIEW_TEMPLATE, context)) - - except Exception as e: - raise HTTPException(status_code=500, detail=f"Error reading {filename}: {e}") - - -# Mount static files -app.mount("/static", StaticFiles(directory="."), name="static") - - -def main(): - """Main function to run the documentation server.""" - parser = argparse.ArgumentParser( - description="Simple documentation server for hosting markdown documentation folders" - ) - parser.add_argument( - "--docs-folder", - type=str, - required=True, - help="Path to the documentation folder containing markdown files and module_tree.json" - ) - parser.add_argument( - "--port", - type=int, - default=8000, - help="Port to run the server on (default: 8000)" - ) - parser.add_argument( - "--host", - type=str, - default="127.0.0.1", - help="Host to bind the server to (default: 127.0.0.1)" - ) - parser.add_argument( - "--debug", - action="store_true", - help="Run the server in debug mode" - ) - - args = parser.parse_args() - - # Validate docs folder - docs_folder = Path(args.docs_folder) - if not docs_folder.exists(): - print(f"Error: Documentation folder '{docs_folder}' does not exist") - sys.exit(1) - - if not docs_folder.is_dir(): - print(f"Error: '{docs_folder}' is not a directory") - sys.exit(1) - - # Check for overview.md - overview_file = docs_folder / "overview.md" - if not overview_file.exists(): - print(f"Warning: overview.md not found in '{docs_folder}'") - - # Set global variables and environment variable for uvicorn reload - global DOCS_FOLDER, MODULE_TREE - DOCS_FOLDER = str(docs_folder.resolve()) - MODULE_TREE = load_module_tree(docs_folder) - - # Set environment variable so uvicorn reload can pick it up - import os - os.environ['DOCS_FOLDER'] = DOCS_FOLDER - - print(f"📚 Starting documentation server...") - print(f"📁 Documentation folder: {DOCS_FOLDER}") - print(f"🌐 Server running at: http://{args.host}:{args.port}") - print(f"📖 Main page: overview.md") - - if MODULE_TREE: - modules_count = len(MODULE_TREE) - print(f"🗂️ Found {modules_count} main modules in module_tree.json") - - print("\nPress Ctrl+C to stop the server") - - try: - import uvicorn - uvicorn.run( - "visualise_docs:app", - host=args.host, - port=args.port, - reload=args.debug, - log_level="debug" if args.debug else "info" - ) - except KeyboardInterrupt: - print("\n👋 Server stopped") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/code2test/src/fe/web_app.py b/code2test/src/fe/web_app.py deleted file mode 100644 index 6f5d846..0000000 --- a/code2test/src/fe/web_app.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -""" -CodeWiki Web Application - -A web interface for users to submit GitHub repositories for documentation generation. -Features: -- Simple web form for GitHub repo URL input -- Background processing queue -- Cache system for generated documentation -- Job status tracking -""" - -import argparse -from fastapi import FastAPI, Request, Form -from fastapi.responses import HTMLResponse - -from .cache_manager import CacheManager -from .background_worker import BackgroundWorker -from .routes import WebRoutes -from .config import WebAppConfig - - -# Initialize FastAPI app -app = FastAPI( - title="CodeWiki", - description="Generate comprehensive documentation for any GitHub repository" -) - -# Initialize components -cache_manager = CacheManager( - cache_dir=WebAppConfig.CACHE_DIR, - cache_expiry_days=WebAppConfig.CACHE_EXPIRY_DAYS -) -background_worker = BackgroundWorker( - cache_manager=cache_manager, - temp_dir=WebAppConfig.TEMP_DIR -) -web_routes = WebRoutes(background_worker=background_worker, cache_manager=cache_manager) - - -# Register routes -@app.get("/", response_class=HTMLResponse) -async def index_get(request: Request): - """Main page with form for submitting GitHub repositories.""" - return await web_routes.index_get(request) - - -@app.post("/", response_class=HTMLResponse) -async def index_post(request: Request, repo_url: str = Form(...), commit_id: str = Form("")): - """Handle repository submission.""" - return await web_routes.index_post(request, repo_url, commit_id) - - -@app.get("/api/job/{job_id}") -async def get_job_status(job_id: str): - """API endpoint to get job status.""" - return await web_routes.get_job_status(job_id) - - -@app.get("/docs/{job_id}") -async def view_docs(job_id: str): - """View generated documentation.""" - return await web_routes.view_docs(job_id) - - -@app.get("/static-docs/{job_id}/") -@app.get("/static-docs/{job_id}/{filename:path}") -async def serve_generated_docs(job_id: str, filename: str = "overview.md"): - """Serve generated documentation files.""" - if not filename: - filename = "overview.md" - return await web_routes.serve_generated_docs(job_id, filename) - - -def main(): - """Main function to run the web application.""" - import uvicorn - - parser = argparse.ArgumentParser( - description="CodeWiki Web Application - Generate documentation for GitHub repositories" - ) - parser.add_argument( - "--host", - type=str, - default=WebAppConfig.DEFAULT_HOST, - help=f"Host to bind the server to (default: {WebAppConfig.DEFAULT_HOST})" - ) - parser.add_argument( - "--port", - type=int, - default=WebAppConfig.DEFAULT_PORT, - help=f"Port to run the server on (default: {WebAppConfig.DEFAULT_PORT})" - ) - parser.add_argument( - "--debug", - action="store_true", - help="Run the server in debug mode" - ) - parser.add_argument( - "--reload", - action="store_true", - help="Enable auto-reload for development" - ) - - args = parser.parse_args() - - # Ensure required directories exist - WebAppConfig.ensure_directories() - - # Start background worker - background_worker.start() - - print(f"🚀 CodeWiki Web Application starting...") - print(f"🌐 Server running at: http://{args.host}:{args.port}") - print(f"📁 Cache directory: {WebAppConfig.get_absolute_path(WebAppConfig.CACHE_DIR)}") - print(f"🗂️ Temp directory: {WebAppConfig.get_absolute_path(WebAppConfig.TEMP_DIR)}") - print("\nPress Ctrl+C to stop the server") - - try: - uvicorn.run( - "fe.web_app:app", - host=args.host, - port=args.port, - reload=args.reload, - log_level="debug" if args.debug else "info" - ) - except KeyboardInterrupt: - print("\n👋 Server stopped") - background_worker.stop() - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/code2test/templates/github_pages/.gitkeep b/code2test/templates/github_pages/.gitkeep deleted file mode 100644 index d5f5da1..0000000 --- a/code2test/templates/github_pages/.gitkeep +++ /dev/null @@ -1,3 +0,0 @@ -# Placeholder file to ensure directory is tracked -# This directory will contain HTML templates for GitHub Pages viewer - diff --git a/code2test/templates/github_pages/viewer_template.html b/code2test/templates/github_pages/viewer_template.html deleted file mode 100644 index 39629ad..0000000 --- a/code2test/templates/github_pages/viewer_template.html +++ /dev/null @@ -1,643 +0,0 @@ - - - - - - {{TITLE}} - - - - - -
- - -
-
- -

Loading documentation...

-
- - -
-
- - - - - diff --git a/docker/.dockerignore b/docker/.dockerignore deleted file mode 100644 index ea5641d..0000000 --- a/docker/.dockerignore +++ /dev/null @@ -1,80 +0,0 @@ -# Git -.git -.gitignore - -# Docker -Dockerfile* -docker-compose* -.dockerignore - -# Environment files -.env -.env.* -!env.example - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# Virtual environments -.ven -env/ -ENV/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Logs -*.log -logs/ - -# Output directories (will be mounted as volumes) -output/ -trajectories/ - -# Test data -data/raw_test_repo/ -data/raw_test_repo_simple/ - -# Documentation build -docs/_build/ -docs/.doctrees/ - -# Cache -.cache/ -.pytest_cache/ -.mypy_cache/ -.coverage -htmlcov/ - -# Temporary files -*.tmp -*.temp -.tmp/ -.temp/ diff --git a/docker/DOCKER_README.md b/docker/DOCKER_README.md deleted file mode 100644 index 286f3ab..0000000 --- a/docker/DOCKER_README.md +++ /dev/null @@ -1,437 +0,0 @@ -# CodeWiki Docker Setup - -This document explains how to run CodeWiki using Docker and Docker Compose. - -## Overview - -The Docker setup provides a containerized environment for running the CodeWiki web application, which allows you to generate documentation for GitHub repositories through a web interface. - -## File Structure - -All Docker-related files are located in the `docker/` directory: - -``` -docker/ -├── Dockerfile # Container image definition -├── docker-compose.yml # Service orchestration -├── env.example # Environment variables template -└── DOCKER_README.md # This file -``` - -The Dockerfile builds from the project root context to include all necessary application code. - ---- - -## Quick Start - -### 1. Clone the Repository - -```bash -git clone -cd CodeWiki -``` - -### 2. Set Up Environment Variables - -```bash -# Copy the example environment file -cp docker/env.example .env - -# Edit .env file with your configuration -nano .env # or use your preferred editor -``` - -Required configuration in `.env`: - -```bash -# LLM API Configuration -MAIN_MODEL=claude-sonnet-4 -FALLBACK_MODEL_1=glm-4p5 -CLUSTER_MODEL=claude-sonnet-4 -LLM_BASE_URL=https://api.anthropic.com # or your LiteLLM proxy -LLM_API_KEY=your-api-key-here - -# Application Port -APP_PORT=8000 - -# Optional: Logfire Configuration (for monitoring) -LOGFIRE_TOKEN= -LOGFIRE_PROJECT_NAME=codewiki -LOGFIRE_SERVICE_NAME=codewiki -``` - -### 3. Create Docker Network - -```bash -docker network create codewiki-network -``` - -### 4. Start the Services - -**Option A: From project root** -```bash -docker-compose -f docker/docker-compose.yml up -d -``` - -### 5. Access the Application - -- Web Application: http://localhost:8000 - -The application will be available at the port specified in your `.env` file (default: 8000). - ---- - -## Docker Compose Configuration - -The `docker-compose.yml` file defines the CodeWiki service with the following features: - -### Service Configuration - -- **Image**: `codewiki:0.0.1` -- **Build Context**: Parent directory (`.` relative to docker/) -- **Container Name**: `codewiki` -- **Port Mapping**: `${APP_PORT:-8000}:8000` -- **Network**: `codewiki-network` (external) - -### Environment Variables - -The container uses environment variables from the `.env` file: -- `PYTHONPATH=/app/src` - Set Python module path -- `PYTHONUNBUFFERED=1` - Enable real-time logging -- All variables from `.env` file - -### Volume Mounts - -The following directories are mounted as volumes: - -```yaml -volumes: - - ./output:/app/output # Persistent storage for generated docs - - ~/.ssh:/root/.ssh:ro # SSH keys for private repos (read-only) -``` - -**Note**: Git credentials can be mounted if needed for private repositories: -```yaml - # Uncomment in docker-compose.yml if needed - - ~/.gitconfig:/root/.gitconfig:ro -``` - -### Health Check - -The service includes a health check that: -- Runs every 30 seconds -- Times out after 10 seconds -- Retries 3 times on failure -- Starts checking after 20 seconds - -### Restart Policy - -The container is set to restart automatically unless explicitly stopped (`restart: unless-stopped`). - ---- - -## Dockerfile Details - -The Dockerfile (`docker/Dockerfile`) builds the CodeWiki image with: - -### Base Image -- Python 3.12 slim image for smaller size - -### System Dependencies -- `git` - For repository cloning -- `curl` - For health checks -- `nodejs` and `npm` - For mermaid diagram validation - -### Application Setup -1. Copies `requirements.txt` first (for better caching) -2. Installs Python dependencies -3. Copies entire application code -4. Creates output directories: - - `output/cache` - - `output/temp` - - `output/docs` - - `output/dependency_graphs` - -### Runtime Configuration -- **Working Directory**: `/app` -- **Exposed Port**: `8000` -- **Entry Point**: `python codewiki/run_web_app.py --host 0.0.0.0 --port 8000` - ---- - -## Common Operations - -### View Logs - -```bash -# From project root -docker-compose -f docker/docker-compose.yml logs -f - -# From docker directory -cd docker -docker-compose logs -f - -# View specific service -docker logs codewiki -f -``` - -### Stop Services - -```bash -# From project root -docker-compose -f docker/docker-compose.yml stop - -# From docker directory -cd docker -docker-compose stop -``` - -### Stop and Remove Containers - -```bash -# From project root -docker-compose -f docker/docker-compose.yml down - -# From docker directory -cd docker -docker-compose down - -# Remove volumes as well -docker-compose down -v -``` - -### Rebuild Image - -If you've made changes to the code or Dockerfile: - -```bash -# From project root -docker-compose -f docker/docker-compose.yml build --no-cache - -# From docker directory -cd docker -docker-compose build --no-cache - -# Rebuild and restart -docker-compose up -d --build -``` - -### Access Container Shell - -```bash -docker exec -it codewiki /bin/bash -``` - ---- - -## Persistent Storage - -### Output Directory - -The `output/` directory is mounted as a volume, ensuring generated documentation persists across container restarts: - -``` -output/ -├── cache/ # Cached dependency graphs and jobs -├── docs/ # Generated documentation -├── dependency_graphs/ # JSON dependency graphs -└── temp/ # Temporary files -``` - -### SSH Keys - -If you need to clone private repositories, ensure your SSH keys are available: - -```bash -# Verify SSH keys are accessible -ls -la ~/.ssh/ - -# The docker-compose.yml mounts ~/.ssh as read-only -``` - ---- - -## Troubleshooting - -### Port Already in Use - -If port 8000 is already in use: - -```bash -# Change APP_PORT in .env file -echo "APP_PORT=8001" >> .env - -# Restart services -docker-compose -f docker/docker-compose.yml down -docker-compose -f docker/docker-compose.yml up -d -``` - -### Container Won't Start - -Check logs for errors: - -```bash -docker logs codewiki -``` - -Common issues: -- **Invalid API key**: Verify `LLM_API_KEY` in `.env` -- **Network not found**: Create network with `docker network create codewiki-network` -- **Port conflict**: Change `APP_PORT` in `.env` - -### Health Check Failing - -```bash -# Check if the application is responding -curl http://localhost:8000/ - -# Check container health status -docker inspect codewiki --format='{{.State.Health.Status}}' - -# View health check logs -docker inspect codewiki --format='{{range .State.Health.Log}}{{.Output}}{{end}}' -``` - -### Permission Issues with Volumes - -If you encounter permission issues with mounted volumes: - -```bash -# On Linux, ensure proper ownership -sudo chown -R $(id -u):$(id -g) output/ - -# Or run container with user mapping -docker-compose -f docker/docker-compose.yml down -# Add to docker-compose.yml under 'codewiki' service: -# user: "${UID}:${GID}" -``` - -### Private Repository Access - -For private repositories: - -1. Ensure SSH keys are properly mounted: - ```yaml - volumes: - - ~/.ssh:/root/.ssh:ro - ``` - -2. Verify key permissions: - ```bash - chmod 600 ~/.ssh/id_rsa - chmod 644 ~/.ssh/id_rsa.pub - ``` - -3. Add GitHub to known_hosts: - ```bash - docker exec -it codewiki ssh-keyscan github.com >> /root/.ssh/known_hosts - ``` - ---- - -## Production Deployment - -### Security Considerations - -1. **Environment Variables**: Never commit `.env` file to version control -2. **API Keys**: Use secrets management in production -3. **Network**: Use isolated Docker networks -4. **Volumes**: Set appropriate permissions on mounted volumes -5. **Updates**: Regularly update base image and dependencies - -### Recommended Production Setup - -```yaml -# Use secrets instead of .env file -services: - codewiki: - secrets: - - llm_api_key - environment: - - LLM_API_KEY_FILE=/run/secrets/llm_api_key - -secrets: - llm_api_key: - external: true -``` - -### Resource Limits - -Add resource limits in production: - -```yaml -services: - codewiki: - deploy: - resources: - limits: - cpus: '2' - memory: 4G - reservations: - cpus: '1' - memory: 2G -``` - -### Reverse Proxy - -Use nginx or traefik as a reverse proxy: - -```yaml -services: - nginx: - image: nginx:alpine - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf:ro - - ./certs:/etc/nginx/certs:ro -``` - ---- - -## Integration with External Services - -### Using LiteLLM Proxy - -If using a LiteLLM proxy for LLM API management: - -```bash -# In .env file -LLM_BASE_URL=http://litellm:4000/ -LLM_API_KEY=sk-your-proxy-key - -# Add LiteLLM service to docker-compose.yml -services: - litellm: - image: ghcr.io/berriai/litellm:latest - ports: - - "4000:4000" - environment: - - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - - OPENAI_API_KEY=${OPENAI_API_KEY} - networks: - - codewiki-network -``` - ---- - -## More Information - -- **Main Documentation**: See [../README.md](../README.md) for complete feature list and usage -- **CLI Tool**: For command-line documentation generation -- **Web Interface**: For GitHub URL-based documentation generation - ---- - -## Support - -For issues related to Docker deployment: -1. Check logs: `docker logs codewiki` -2. Verify configuration: `docker exec codewiki env | grep -E '(LLM|APP)'` -3. Test connectivity: `docker exec codewiki curl -I http://localhost:8000` -4. Report issues: https://github.com/yourusername/codewiki/issues - ---- - -**Happy documenting with Docker! 🐳📚** diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index d0a70d1..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Use Python 3.12 slim image as base -FROM python:3.12-slim - -# Set working directory -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - git \ - curl \ - nodejs \ - npm \ - && rm -rf /var/lib/apt/lists/* - -# Copy requirements first for better caching -COPY requirements.txt . - -# Install Python dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Copy application code -COPY codewiki ./codewiki -COPY img ./img -COPY pyproject.toml . -COPY README.md . - -# Create output directories -RUN mkdir -p output/cache output/temp output/docs output/dependency_graphs - -# Set environment variables -ENV PYTHONPATH=/app -ENV PYTHONUNBUFFERED=1 - -# Expose port -EXPOSE 8000 - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8000/ || exit 1 - -# Default command -CMD ["python", "codewiki/run_web_app.py", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 28084e7..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,34 +0,0 @@ -services: - codewiki: - image: codewiki:0.0.1 - build: - context: .. - dockerfile: docker/Dockerfile - container_name: codewiki - ports: - - "${APP_PORT:-8000}:8000" - environment: - - PYTHONPATH=/app - - PYTHONUNBUFFERED=1 - env_file: - - ../.env - networks: - - net - volumes: - # Persistent storage for cache and output - - ../output:/app/output - # Git credentials (if needed for private repos) - # - ~/.gitconfig:/root/.gitconfig:ro - - ~/.ssh:/root/.ssh:ro - restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 20s - -networks: - net: - external: true - name: codewiki-network diff --git a/docker/env.example b/docker/env.example deleted file mode 100644 index e8d8c9e..0000000 --- a/docker/env.example +++ /dev/null @@ -1,34 +0,0 @@ -# Copy this file to .env and fill in your API keys - -# ============================================================================= -# LLM API Keys -# ============================================================================= - -MAIN_MODEL = claude-sonnet-4 -FALLBACK_MODEL_1 = glm-4p5 -CLUSTER_MODEL = claude-sonnet-4 -LLM_BASE_URL = http://litellm:4000/ -LLM_API_KEY = sk-1234 - - -# ============================================================================= -# Application Configuration -# ============================================================================= - -# Port for the main web application -APP_PORT=8000 - - -# ============================================================================= -# Logfire Configuration -# ============================================================================= - -# Logfire authentication token (get from ~/.logfire/default.toml after running 'logfire auth') -LOGFIRE_TOKEN= - -# Optional: Logfire project name (defaults to auto-detection) -LOGFIRE_PROJECT_NAME=codewiki - -# Optional: Logfire service name -LOGFIRE_SERVICE_NAME=codewiki - diff --git a/docs/architecture.md b/docs/architecture.md index f107a59..9fede69 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,7 +28,7 @@ graph TD | Module | Role | | --- | --- | | `code2test/cli/commands/` | Command-line entry points: `test`, `verify`, `benchmark`, `intent`, `report`, `init`, `config` | -| `code2test/cli/utils/model.py` | Chooses the model (`CODE2TEST_MODEL`, saved configuration, Azure, default) | +| `code2test/cli/utils/model.py` | Chooses the model (`CODE2TEST_MODEL`, saved main model, Azure, Google key, default) | | `code2test/src/be/dependency_analyzer/` | Parses the repository (derived from the CodeWiki analyzer) and builds the dependency graph; Python components are top-level functions and classes | | `code2test/core/intent.py`, `core/intent_analyzers/` | Static intent extraction and confidence scoring from docstrings, type hints, naming conventions, call sites and complexity | | `code2test/agents/intent_agent.py` | Re-infers low-confidence intents with the LLM | @@ -47,6 +47,8 @@ Each component starts from a confidence of 0.5. Static signals add to it: a docs When the score is below the threshold (`--confidence`, default 0.6), the intent agent re-infers the intent from the source, and the more confident result is kept. Intents are stored. An intent edited with `code2test intent edit` gets confidence 0.95 and is never replaced by a later run. +In `--auto` mode, a component whose intent is still below the threshold gets no tests, and a component whose tests were already verified in an earlier run keeps them instead of being regenerated. Both are counted in the run metrics. + ## Generation The test agent's prompt contains: @@ -57,11 +59,13 @@ The test agent's prompt contains: - the public API (signatures, constructors and public methods, no bodies) of the component's module and of the project modules it imports - the signature and up to 16,000 characters of the component's source +The prompt also asks the model to work out each expected value from the source (and to assert a property such as a type, length or exception when it cannot), to build objects exactly as their signatures require, to put any file the code creates under pytest's `tmp_path`, and never to mock the component under test. + The model returns test functions, imports and fixtures as structured output. Each component gets one file, `tests//test__.py`. For a package's `__init__.py`, the file is `tests//test_.py`. ## Verification and repair -The verifier writes the file and runs it with `python -m pytest` from the repository root. It puts the root and `src/` on `PYTHONPATH`, ignores the target project's `addopts`, and turns off bytecode caching, so rewritten files are never served stale. +The verifier writes the file and runs it with `python -m pytest` in a subprocess. The working directory is a new temporary directory, with `--rootdir` set to the repository, so a test that writes to a relative path cannot touch the user's files. The verifier puts the repository root and `src/` on `PYTHONPATH`, ignores the target project's `addopts`, and turns off bytecode caching, so rewritten files are never served stale. Results are read from pytest's JUnit XML and matched to the test functions each generated test defines. An error that prevents the file from being collected counts against every test in it. Failing tests, with their failure output, go back to the test agent for up to two repair rounds. @@ -72,7 +76,7 @@ Tests that still fail are diagnosed against the stored intent: - **Code bug:** the test is kept with `pytest.mark.xfail` and the diagnosis as its reason. - **Wrong test or wrong intent:** the test is dropped. -A file that cannot be collected is never kept. After every file passes on its own, the whole generated suite runs together, and tests that fail only there are dropped. +An error raised by the test itself (an import, syntax or name error) always counts as a wrong test. A file that cannot be collected is never kept. After every file passes on its own, the whole generated suite runs together, and tests that fail only there are dropped, for up to three rounds. The result is the invariant the tool guarantees: every test it writes either passes or is an explicit expected failure. @@ -85,7 +89,11 @@ All three agents use pydantic-ai with typed structured output and up to three ou `code2test benchmark` evaluates the tests already generated. It works in three stages: 1. **Mutate.** It enumerates single-site mutants in the tested components: flipped comparisons, swapped operators, changed constants, and `return None`. Docstrings and type annotations are excluded. -2. **Run.** It samples up to `--mutants` of them and runs each component's tests against each mutant. Before mutating a file it keeps a byte-exact backup in `.code2test/benchmark_backup/`, and it restores the file afterwards, including on SIGTERM. A run that was killed is repaired the next time `benchmark` starts. +2. **Run.** It samples up to `--mutants` of them and runs the whole generated suite against each mutant. It also records whether the mutated component's own tests caught it. Before mutating a file it keeps a byte-exact backup in `.code2test/benchmark_backup/`, and it restores the file afterwards, including on SIGTERM. A run that was killed is repaired the next time `benchmark` starts. 3. **Score diagnosis.** It checks the diagnosis agent against failures whose cause is known: - - tests failing on a mutant, which should be diagnosed as a code bug; - - passing tests whose expected value was changed, which should be diagnosed as a wrong test. + - tests of the mutated component that fail on a mutant, which should be diagnosed as a code bug (the mutated line must be within the source the agent is shown); + - passing tests whose first literal expected value was changed slightly, which should be diagnosed as a wrong test (only when that assertion is what fails). + + The changes leave no marker, such as a `_wrong` suffix, that would give the answer away. + +Results are written to `.code2test/benchmark.json`, including every surviving mutant and every diagnosis case. diff --git a/docs/proposal.md b/docs/proposal.md index 9ab9ce0..d4ee949 100644 --- a/docs/proposal.md +++ b/docs/proposal.md @@ -1,5 +1,7 @@ # CODE2TEST +> This is the original product proposal from January 2026. It describes the planned product, including multi-language support and a business model, which are not part of v1. For what is implemented, see [task.md](task.md); for measured results, see the [README](../README.md#results). The README and the paper measure the first three MVP targets in Section 7.3. + ## Intelligent Test Generation for Legacy Codebases --- diff --git a/docs/quickstart.md b/docs/quickstart.md index d7a577d..3308044 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -51,6 +51,7 @@ Passed on first run 196 (96%) Tests kept 205 of which xfail (possible bugs) 0 Tests dropped 0 +Repair rounds 8 ``` The tests are in `tests/`, one file per component. The same numbers are saved to `.code2test/metrics.json`. diff --git a/docs/task.md b/docs/task.md index d64b8a6..e4760ae 100644 --- a/docs/task.md +++ b/docs/task.md @@ -1,193 +1,59 @@ -# Code2Test Implementation Tasks +# Implementation Status -## Project Overview -Transform CodeWiki (documentation generator) into Code2Test (test generator) using intent-first hierarchical test generation with human-in-the-loop verification. +Code2Test grew out of CodeWiki, a repository documentation generator. Its dependency analyzer was kept and test generation was built on top; CodeWiki's documentation generator and web app have been removed. This page lists what v1 does, what exists in the code but is not used by the v1 pipeline, and what is planned. The original plan and its success criteria are in [proposal.md](proposal.md). ---- +## Scope of v1 -## Phase 1: Foundation (Weeks 1-2) +Python only. Test generation, verification and repair target pytest. Files in other languages are part of the dependency graph, but their components get no intents or tests, and `code2test test` says so when it skips them. -### 1.1 Rename & Rebrand -- [x] Rename package from `codewiki` to `code2test` throughout codebase -- [x] Update `pyproject.toml` with new name, description, and version -- [x] Update CLI entry point from `codewiki` to `code2test` -- [ ] Update README.md with Code2Test branding +## Done -### 1.2 Core Data Models -- [x] Create `code2test/core/models.py` with: - - [x] `Intent` model (component, intent_text, confidence, evidence) - - [x] `TestCase` model (name, intent, assertions, status) - - [x] `TestFile` model (path, test_cases, component) - - [x] `Diagnosis` model (cause, confidence, suggested_fix) - - [x] `VerificationResult` model (passed, failures, diagnosis) +**Pipeline** (see [architecture.md](architecture.md)) +- [x] Dependency analysis of the repository; top-level Python functions and classes become components +- [x] Static intent extraction with confidence scoring (docstrings, type hints, naming conventions, call sites, complexity) +- [x] LLM re-inference of intents below the confidence threshold +- [x] Intent storage in SQLite; edited intents are never overwritten +- [x] Test generation with the intent, import line, module imports, related API and source +- [x] Per-file execution with pytest, isolated from the user's working directory +- [x] Repair from pytest output, up to two rounds +- [x] Diagnosis of remaining failures as wrong test, code bug or wrong intent; suspected bugs kept as `xfail` +- [x] Whole-suite check that drops tests failing only alongside others +- [x] Reuse of already verified tests on re-runs in `--auto` mode +- [x] Provider errors 401, 403 and 404 stop the run -### 1.3 Intent Storage System -- [x] Create `code2test/storage/intent_db.py`: - - [x] SQLite integration for intent persistence - - [x] Intent CRUD operations - - [x] Low-confidence intent filtering -- [x] Create `code2test/storage/test_registry.py`: - - [x] Track generated tests per component - - [x] Verification status tracking +**Commands** +- [x] `test`, interactive and `--auto`, with `--exit-code` for CI and `--report html|json|all` +- [x] `verify`, with `--diagnose` and `--fix` (only for generated files not edited since) +- [x] `intent show|edit|export` +- [x] `benchmark`: mutation score and diagnosis accuracy on failures of known cause +- [x] `report summary`: intents by confidence +- [x] `init`: writes a starter `.code2test/config.json` +- [x] `config set|show|validate`: API key and model; `validate` sends the model one test request -### 1.4 CLI Scaffold Enhancements -- [x] Add new commands to `code2test/cli/commands/`: - - [x] `generate.py` - Interactive test generation (exists, remains as legacy/alias) - - [x] `test.py` - Main interactive test generation command - - [x] `verify.py` - Test verification and diagnosis - - [x] `intent.py` - View/edit inferred intents - - [x] `report.py` - Coverage and quality reports -- [x] Update `main.py` with new command groups +**Measurement and evaluation** +- [x] Run metrics in `.code2test/metrics.json`: acceptance rate, first-run pass rate, repairs, xfails, drops, reused and skipped components +- [x] Evaluation on inflection and TinyDB with two models; results in the [README](../README.md#results) and [research/](../research/) ---- +**Project** +- [x] CI running the test suite on every push and pull request +- [x] CI examples for GitHub Actions and GitLab CI ([examples/ci](../examples/ci)) -## Phase 2: Core Engine (Weeks 3-5) +## In the code but not used by the v1 pipeline -### 2.1 Intent Extraction System -- [x] Create `code2test/core/intent.py`: - - [x] Intent extraction from docstrings - - [x] Intent inference from function signatures/type hints - - [x] Naming convention analysis (e.g., `validate_*`, `is_*`) - - [x] Call site analysis from dependency graph - - [x] Dependency intent aggregation - - [x] Confidence scoring algorithm +- `code2test/adapters/` (pytest, Jest and JUnit adapters) and `code2test/core/templates.py`: the pipeline generates tests with the test agent and runs them with `core/verifier.py` instead. +- `code2test/core/intent_analyzers/` for JavaScript and Java: never reached, because `code2test test` passes only Python components to the pipeline. +- `TestAgent.generate_integration_tests`: not called by any command. +- `--framework unittest`: accepted, but tests are generated in pytest style. -### 2.2 Test Generation Agents -- [x] Create `code2test/agents/intent_agent.py`: - - [x] LLM-powered intent inference - - [x] Multi-signal confidence scoring - - [x] User clarification prompting for low-confidence -- [x] Create `code2test/agents/test_agent.py`: - - [x] Unit test generation from intent - - [x] Integration test generation from module structure - - [x] Test fixture suggestion +## Not done -### 2.3 Verification System -- [x] Create `code2test/core/verifier.py`: - - [x] Test execution orchestration - - [x] Result parsing and aggregation - - [x] Failure categorization -- [x] Create `code2test/agents/diagnosis_agent.py`: - - [x] Failure root cause analysis - - [x] Diagnosis classification (TEST_WRONG, CODE_BUG, INTENT_WRONG) - - [x] Fix suggestion generation +- [ ] `code2test report coverage` (the command says it is not implemented) +- [ ] Markdown run reports +- [ ] Settings from `.code2test/config.json` (written by `init`, not read by other commands) +- [ ] Incremental generation for changed files only (re-runs reuse verified tests per component, whether or not the code changed) +- [ ] An experiment on code with known bugs, to test whether intent-first generation avoids locking bugs into the suite (see the paper's Discussion) -### 2.4 Generator Orchestration -- [x] Create `code2test/core/generator.py`: - - [x] Hierarchical generation flow orchestration - - [x] Intent extraction phase coordination - - [x] Test generation phase coordination - - [x] Verification loop management - ---- - -## Phase 3: Interactive CLI (Weeks 6-7) - -### 3.1 Rich Terminal UI -- [x] Create `code2test/cli/interactive.py`: - - [x] Intent confidence display with color coding - - [x] Test preview rendering - - [x] Verification result formatting - - [x] Diagnosis panel with recommendations -- [x] Create `code2test/cli/display.py`: - - [x] Progress indicators for generation - - [x] Rich tables for test summaries - - [x] Tree view for hierarchical test structure - -### 3.2 User Interaction Flows -- [x] Implement accept/skip/edit/run test flow -- [x] Implement intent editing mode -- [x] Implement diagnosis response flow (fix/flag bug/keep/skip) -- [x] Implement batch confirmation mode - -### 3.3 Low Confidence Handling -- [x] Interactive intent clarification prompts -- [x] Intent editing with live confidence recalculation -- [x] User-provided intent override - ---- - -## Phase 4: Multi-Language Support (Weeks 8-10) - -### 4.1 Test Framework Adapters -- [x] Create `code2test/adapters/python/pytest_adapter.py`: - - [x] pytest test file generation - - [x] Fixture generation - - [x] Test execution and result parsing -- [ ] Create `code2test/adapters/javascript/jest_adapter.py` *(post-v1: adapter code exists but is not wired into generation or verification)*: - - [ ] Jest test file generation - - [ ] Mock generation - - [ ] Test execution and result parsing -- [ ] Create `code2test/adapters/java/junit_adapter.py` *(post-v1: adapter code exists but is not wired into generation or verification)*: - - [ ] JUnit 5 test file generation - - [ ] Test execution and result parsing - -### 4.2 Language-Specific Intent Extraction -- [x] Refactor IntentExtractor for pluggable strategies -- [x] Implement Python analyzer -- [x] Implement JavaScript/TypeScript analyzer -- [x] Implement Java analyzer - -### 4.3 Template System -- [x] Create test file templates per language/framework -- [x] Create fixture/mock templates -- [x] Create assertion pattern library - ---- - -## Phase 5: Enterprise Features (Weeks 11-12) - -### 5.1 CI/CD Integration -- [x] Implement `--auto` mode for non-interactive generation -- [x] Implement confidence threshold filtering -- [x] Create GitHub Actions workflow example -- [x] Create GitLab CI example - -### 5.2 Reporting System -- [x] Create `code2test/reporting/`: - - [x] HTML report generation - - [x] JSON export for integrations - - [x] Markdown summary generation - - [x] Coverage analysis (behavioral vs line) - -### 5.3 Batch Processing -- [x] Parallel module processing -- [x] Progress persistence and resumption -- [x] Incremental test generation (changed files only) - ---- - -## Current Implementation Status - -### ✅ Completed -- Core Models & Storage -- Intent Extraction Engine (Polylingual) -- LLM Agents (Intent, Test, Diagnosis) -- Verification System -- CLI (Refactored & Interactive) -- Pytest adapter and verification (Python) -- Template System -- CI/CD & Reporting -- Batch Processing - -### Scope of v1 -- **Python only.** Test generation, verification and repair target pytest. Other - languages are analysed (dependency graph, intents) but get no tests; the CLI says so. -- Jest/JUnit generation is planned for after v1. - -### MVP metrics (measured, see docs/real_model_run.md) -- `code2test test` reports acceptance rate and first-run pass rate on every run. -- `code2test benchmark` reports mutation score and diagnosis accuracy. - ---- - -## Phase 6: Gap Filling (Weeks 13) - -### 6.1 CLI Completeness -- [x] Implement `code2test init` command - - [x] Generate `.code2test/config.json` - - [x] Auto-detect language and frameworks -- [x] Implement `code2test verify --fix` logic - - [x] Wire up DiagnosisAgent to verify command - - [x] Implement auto-patching of test files based on diagnosis +## Post-v1 +- [ ] Jest test generation and execution for JavaScript and TypeScript +- [ ] JUnit 5 test generation and execution for Java diff --git a/examples/ci/github_actions.yml b/examples/ci/github_actions.yml index fb93f0d..e48ba3d 100644 --- a/examples/ci/github_actions.yml +++ b/examples/ci/github_actions.yml @@ -1,34 +1,43 @@ +# Generate and verify tests for a Python project with Code2Test. +# Copy to .github/workflows/code2test.yml in your project. +# Needs a repository secret with your model provider's API key (here OPENAI_API_KEY); +# see "Choosing a model" in the Code2Test README for other providers. name: Code2Test -on: [push, pull_request] + +on: + pull_request: + push: + branches: [main] jobs: - test-generation: + generate-tests: runs-on: ubuntu-latest + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CODE2TEST_MODEL: openai:gpt-4o-mini steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 with: - python-version: '3.12' - - - name: Install Dependencies + python-version: "3.12" + + - name: Install the project and Code2Test run: | python -m pip install --upgrade pip - pip install . - pip install pytest pytest-cov - - - name: Generate Tests - # Run in auto mode, fail if verification fails - run: | - code2test test \ - --auto \ - --confidence 0.6 \ - --exit-code \ - --output-dir generated_tests \ - code2test/core/ - - - name: Run Generated Tests - if: success() || failure() # Run even if generation had issues but produced files - run: | - pytest generated_tests + pip install -e . + pip install git+https://github.com/Cirkitly/code2test.git + + - name: Generate tests + # --exit-code fails the job if any written test file failed verification + run: code2test test --auto --exit-code --output-dir generated_tests src/ + + - name: Run the generated suite + run: code2test verify generated_tests + + - name: Keep the generated tests + if: always() + uses: actions/upload-artifact@v4 + with: + name: generated-tests + path: generated_tests/ diff --git a/examples/ci/gitlab_ci.yml b/examples/ci/gitlab_ci.yml index b4125d0..cf22dfc 100644 --- a/examples/ci/gitlab_ci.yml +++ b/examples/ci/gitlab_ci.yml @@ -1,45 +1,44 @@ +# Generate and verify tests for a Python project with Code2Test. +# Copy into your project's .gitlab-ci.yml. +# Needs a masked CI/CD variable with your model provider's API key (here OPENAI_API_KEY); +# see "Choosing a model" in the Code2Test README for other providers. stages: - - test-generation - - test-execution + - generate + - verify variables: - PYTHON_VERSION: "3.12" + CODE2TEST_MODEL: "openai:gpt-4o-mini" OUTPUT_DIR: "generated_tests" + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip" default: - image: python:$PYTHON_VERSION - -cache: - paths: - - .venv/ - - .pip/ - -before_script: - - python -m venv .venv - - source .venv/bin/activate - - pip install --upgrade pip - - pip install . - - pip install pytest pytest-cov + image: python:3.12 + cache: + paths: + - .pip/ + before_script: + - pip install --upgrade pip + - pip install -e . + - pip install git+https://github.com/Cirkitly/code2test.git generate_tests: - stage: test-generation + stage: generate script: - - echo "Generating tests with Code2Test..." - - code2test test --auto --confidence 0.7 --exit-code --output-dir $OUTPUT_DIR code2test/core/ + # --exit-code fails the job if any written test file failed verification + - code2test test --auto --exit-code --output-dir $OUTPUT_DIR src/ artifacts: paths: - $OUTPUT_DIR/ expire_in: 1 week rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -run_tests: - stage: test-execution +verify_tests: + stage: verify needs: ["generate_tests"] script: - - echo "Running generated tests..." - - pytest $OUTPUT_DIR + - code2test verify $OUTPUT_DIR rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_BRANCH == "main" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/index.html b/index.html deleted file mode 100644 index 97c2e15..0000000 --- a/index.html +++ /dev/null @@ -1,809 +0,0 @@ - - - - - - - - - - - - - - - - CodeWiki - Automated Repository-Level Documentation at Scale - - - - - - - - -
-

CodeWiki: Evaluating AI's Ability to Generate Holistic Documentation for Large-Scale Codebases

-

The first open-source framework for holistic, structured repository-level documentation across multilingual codebases

- -
- 🐍 Python 3.12+ - 📝 7 Languages Supported - 🚀 Open Source - 📊 Benchmarked -
- -
- - View on GitHub - - - Read Paper - - - Live Demo - -
-
- - -
- -
-

📖 About

-

- Developers spend nearly 58% of their time understanding codebases, yet maintaining comprehensive documentation remains challenging. While recent Large Language Models (LLMs) show promise for function-level documentation, they fail at the repository level, where capturing architectural patterns and cross-module interactions is essential. -

-

- CodeWiki is the first open-source framework for holistic repository-level documentation across seven programming languages, introducing innovations in hierarchical decomposition, recursive agentic processing, and multi-modal synthesis. -

- -
-
-
68.79%
-
Quality Score (Proprietary Models)
-
-
-
64.80%
-
Quality Score (Open-Source Models)
-
-
-
7
-
Programming Languages
-
-
-
1.4M
-
Max LOC Tested
-
-
-
- - -
-

🏗️ Framework Architecture

-
- CodeWiki Framework Architecture -

- Figure 1: CodeWiki Framework operates in three main phases: (1) Repository analysis and hierarchical decomposition, - (2) Recursive documentation generation with dynamic delegation, (3) Hierarchical assembly and synthesis -

-
-
- - -
-

✨ Key Innovations

-
-
-
🔄
-

Hierarchical Decomposition

-

Dynamic programming-inspired strategy that breaks complex repositories into manageable modules while preserving architectural coherence. Handles codebases from 86K to 1.4M lines of code.

-
- -
-
🤖
-

Recursive Agentic System

-

Multi-agent architecture with dynamic delegation capabilities that enables adaptive processing based on module complexity, maintaining quality at repository-level scope.

-
- -
-
📊
-

Multi-Modal Synthesis

-

Generates comprehensive documentation including textual descriptions, architecture diagrams, data flows, and sequence diagrams for holistic understanding.

-
-
- -

🌐 Multilingual Support

-
- 🐍 Python - ☕ Java - 📜 JavaScript - 📘 TypeScript - 🔧 C - ⚙️ C++ - 💎 C# -
-
- - -
-

📊 Experimental Results

-

- Evaluated on CodeWikiBench, the first benchmark specifically designed for repository-level documentation quality assessment across 21 repositories. -

- -

Performance by Language Category

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Language CategoryCodeWiki (Sonnet-4)DeepWikiImprovement
High-Level (Python, JS, TS)79.14%68.67%+10.47%
Managed (C#, Java)68.84%64.80%+4.04%
Systems (C, C++)53.24%56.39%-3.15%
Overall Average68.79%64.06%+4.73%
- -

Results on Representative Repositories

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
RepositoryLanguageLOCCodeWikiDeepWikiImprovement
All-Hands-AI--OpenHandsPython229K82.45%73.04%+9.41%
puppeteer--puppeteerTypeScript136K83.00%64.46%+18.54%
sveltejs--svelteJavaScript125K71.96%68.51%+3.45%
Unity-Technologies--ml-agentsC#86K79.78%74.80%+4.98%
elastic--logstashJava117K57.90%54.80%+3.10%
- -

- View comprehensive results for all 21 repositories in our paper. -

-
- - -
-

🎥 Demo Video

-

- Watch CodeWiki in action as it generates comprehensive documentation for a real repository: -

- - -
- CLI Usage Example -

- CLI Usage Example: Generating documentation with CodeWiki -

-
- - -
- - -
-

🚀 Installation & Quick Start

- -

Prerequisites

-
    -
  • Python 3.12+
    • -
  • Node.js (for Mermaid diagram validation)
    • -
  • LLM API access (Anthropic Claude, OpenAI, etc.)
    • -
- -

Installation

- 
# Install from source
-pip install git+https://github.com/FSoft-AI4Code/CodeWiki.git
-
-# Verify installation
-codewiki --version
- -

Quick Start

-

1. Configure CodeWiki:

- 
codewiki config set \
-  --api-key YOUR_API_KEY \
-  --base-url https://api.anthropic.com \
-  --main-model claude-sonnet-4 \
-  --cluster-model claude-sonnet-4
-
-# Verify configuration
-codewiki config show
-codewiki config validate
- -

2. Generate Documentation:

- 
# Navigate to your project
-cd /path/to/your/project
-
-# Generate documentation (saved to ./docs/)
-codewiki generate
-
-# Generate with GitHub Pages HTML viewer
-codewiki generate --github-pages
-
-# Full-featured generation
-codewiki generate --create-branch --github-pages --verbose
- -

Output Structure

- 
./docs/
-├── overview.md              # Repository overview (start here!)
-├── module1.md               # Module documentation
-├── module2.md               # Additional modules...
-├── module_tree.json         # Hierarchical module structure
-├── first_module_tree.json   # Initial clustering result
-├── metadata.json            # Generation metadata
-└── index.html               # Interactive viewer (with --github-pages)
-
- - -
-

📚 Citation

-

If you use CodeWiki in your research, please cite our paper:

- 
@misc{hoang2025codewikievaluatingaisability,
-      title={CodeWiki: Evaluating AI's Ability to Generate Holistic 
-             Documentation for Large-Scale Codebases}, 
-      author={Anh Nguyen Hoang and Minh Le-Anh and Bach Le 
-              and Nghi D. Q. Bui},
-      year={2025},
-      eprint={2510.24428},
-      archivePrefix={arXiv},
-      primaryClass={cs.SE},
-      url={https://arxiv.org/abs/2510.24428}, 
-}
-
- - -
-

🔗 Resources

-
-
-
📄
-

Research Paper

-

Read the full paper on arXiv

- - arXiv - -
- -
-
💻
-

GitHub Repository

-

Access source code and contribute

- - GitHub - -
- -
-
🎯
-

CodeWikiBench

-

First repository-level doc benchmark

- - Benchmark - -
- -
-
🌐
-

Live Demo

-

View generated documentation examples

- - Demo - -
- -
-
🐳
-

Docker Deployment

-

Containerized deployment guide

- - Docker Guide - -
- -
-
📖
-

Development Guide

-

Contributing and architecture details

- - Dev Guide - -
-
-
-
- - - - - - diff --git a/pyproject.toml b/pyproject.toml index ccfea6a..aa90c53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "code2test" version = "0.1.0" -description = "Intelligent Test Generation for Legacy Codebases - Intent-first hierarchical test generation with human-in-the-loop verification" +description = "Intent-first, execution-verified pytest generation for legacy Python code" readme = "README.md" requires-python = ">=3.12" license = {text = "MIT"} @@ -27,10 +27,8 @@ classifiers = [ dependencies = [ "click>=8.1.0", "keyring>=24.0.0", - "GitPython>=3.1.40", "Jinja2>=3.1.6", "tree-sitter>=0.23.2", - "tree-sitter-language-pack>=0.8.0", "tree-sitter-python>=0.23.6", "tree-sitter-java>=0.23.5", "tree-sitter-javascript>=0.21.4", @@ -40,17 +38,10 @@ dependencies = [ "tree-sitter-c-sharp>=0.23.1", "tree-sitter-php>=0.23.0", "openai>=1.107.0", - "litellm>=1.77.0", "pydantic>=2.11.7", - "pydantic-settings>=2.10.1", "pydantic-ai>=1.0.6", - "requests>=2.32.4", "python-dotenv>=1.1.1", "rich>=14.1.0", - "networkx>=3.5", - "psutil>=7.0.0", - "PyYAML>=6.0.2", - "prompt-toolkit>=3.0.0", "pytest>=7.4.0", ] @@ -68,10 +59,10 @@ dev = [ code2test = "code2test.cli.main:cli" [project.urls] -Homepage = "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/cirkitly/code2test" -Documentation = "/cirkitly/code2test/docs" -Repository = "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/cirkitly/code2test" -Issues = "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/cirkitly/code2test/issues" +Homepage = "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/Cirkitly/code2test" +Documentation = "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/Cirkitly/code2test/tree/master/docs" +Repository = "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/Cirkitly/code2test" +Issues = "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/Cirkitly/code2test/issues" [tool.setuptools] packages = [ @@ -80,23 +71,22 @@ packages = [ "code2test.cli.commands", "code2test.cli.models", "code2test.cli.utils", - "code2test.cli.adapters", "code2test.core", "code2test.core.intent_analyzers", "code2test.agents", "code2test.storage", "code2test.adapters", + "code2test.adapters.java", + "code2test.adapters.javascript", "code2test.adapters.python", "code2test.reporting", "code2test.src", "code2test.src.be", - "code2test.src.be.agent_tools", "code2test.src.be.dependency_analyzer", "code2test.src.be.dependency_analyzer.analysis", "code2test.src.be.dependency_analyzer.analyzers", "code2test.src.be.dependency_analyzer.models", "code2test.src.be.dependency_analyzer.utils", - "code2test.src.fe" ] [tool.setuptools.package-data] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index bed6e91..0000000 --- a/requirements.txt +++ /dev/null @@ -1,164 +0,0 @@ -ag-ui-protocol==0.1.8 -aiodns==3.5.0 -aiohappyeyeballs==2.6.1 -aiohttp==3.12.15 -aiosignal==1.4.0 -annotated-types==0.7.0 -anthropic==0.67.0 -anyio==4.10.0 -appnope==0.1.4 -argcomplete==3.6.2 -asttokens==3.0.0 -attrs==25.3.0 -boto3==1.40.2 -botocore==1.40.2 -Brotli==1.1.0 -cachetools==5.5.2 -certifi==2025.8.3 -cffi==2.0.0 -charset-normalizer==3.4.2 -click==8.2.1 -cohere==5.18.0 -colorama==0.4.6 -comm==0.2.3 -debugpy==1.8.15 -decorator==5.2.1 -distro==1.9.0 -eval_type_backport==0.2.2 -executing==2.2.0 -fastapi==0.116.1 -fastavro==1.12.0 -fastuuid==0.12.0 -filelock==3.18.0 -frozenlist==1.7.0 -fsspec==2025.7.0 -genai-prices==0.0.27 -gitdb==4.0.12 -GitPython==3.1.40 -google-auth==2.40.3 -google-genai==1.36.0 -googleapis-common-protos==1.70.0 -griffe==1.9.0 -groq==0.30.0 -h11==0.16.0 -hf-xet==1.1.5 -httpcore==1.0.9 -httpx==0.28.1 -httpx-sse==0.4.0 -huggingface-hub==0.34.3 -keyring>=24.0.0 -idna==3.10 -importlib_metadata==8.7.0 -invoke==2.2.0 -ipykernel==6.30.1 -ipython==9.4.0 -ipython_pygments_lexers==1.1.1 -jedi==0.19.2 -Jinja2==3.1.6 -jiter==0.10.0 -jmespath==1.0.1 -jsonschema==4.25.0 -jsonschema-specifications==2025.4.1 -jupyter_client==8.6.3 -jupyter_core==5.8.1 -litellm==1.77.0 -logfire==4.1.0 -logfire-api==4.1.0 -loguru==0.7.3 -markdown-it-py==3.0.0 -MarkupSafe==3.0.2 -matplotlib-inline==0.1.7 -mcp==1.12.3 -mdurl==0.1.2 -mermaid-parser-py==0.0.2 -mermaid-py==0.8.0 -mistralai==1.9.10 -multidict==6.6.3 -nest-asyncio==1.6.0 -networkx==3.5 -nexus-rpc==1.1.0 -openai==1.107.1 -openai-agents==0.3.0 -opentelemetry-api==1.36.0 -opentelemetry-exporter-otlp-proto-common==1.36.0 -opentelemetry-exporter-otlp-proto-http==1.36.0 -opentelemetry-instrumentation==0.57b0 -opentelemetry-instrumentation-httpx==0.57b0 -opentelemetry-proto==1.36.0 -opentelemetry-sdk==1.36.0 -opentelemetry-semantic-conventions==0.57b0 -opentelemetry-util-http==0.57b0 -packaging==25.0 -parso==0.8.4 -pathspec==0.12.1 -pexpect==4.9.0 -platformdirs==4.3.8 -pminit==1.2.0 -prompt_toolkit==3.0.51 -propcache==0.3.2 -protobuf==5.29.5 -psutil==7.0.0 -ptyprocess==0.7.0 -pure_eval==0.2.3 -pyasn1==0.6.1 -pyasn1_modules==0.4.2 -pycares==4.11.0 -pycparser==2.23 -pydantic==2.11.7 -pydantic-ai==1.0.6 -pydantic-ai-slim==1.0.6 -pydantic-evals==1.0.6 -pydantic-graph==1.0.6 -pydantic-settings==2.10.1 -pydantic_core==2.33.2 -Pygments==2.19.2 -pyperclip==1.9.0 -python-dateutil==2.9.0.post0 -python-dotenv==1.1.1 -python-multipart==0.0.20 -pythonmonkey==1.2.0 -pytz==2025.2 -PyYAML==6.0.2 -pyzmq==27.0.1 -referencing==0.36.2 -regex==2025.7.34 -requests==2.32.4 -rich==14.1.0 -rpds-py==0.26.0 -rsa==4.9.1 -s3transfer==0.13.1 -six==1.17.0 -smmap==5.0.2 -sniffio==1.3.1 -sse-starlette==3.0.2 -stack-data==0.6.3 -starlette==0.47.2 -temporalio==1.17.0 -tenacity==8.5.0 -tiktoken==0.10.0 -tokenizers==0.21.4 -tornado==6.5.1 -tqdm==4.67.1 -traitlets==5.14.3 -tree-sitter==0.23.2 -tree-sitter-c==0.21.4 -tree-sitter-c-sharp==0.23.1 -tree-sitter-cpp==0.23.4 -tree-sitter-embedded-template==0.23.2 -tree-sitter-java==0.23.5 -tree-sitter-javascript==0.21.4 -tree-sitter-language-pack==0.8.0 -tree-sitter-python==0.23.6 -tree-sitter-typescript==0.21.2 -tree-sitter-yaml==0.7.1 -types-protobuf==6.30.2.20250822 -types-requests==2.32.4.20250611 -typing-inspection==0.4.1 -typing_extensions==4.14.1 -urllib3==2.5.0 -uvicorn==0.35.0 -wcwidth==0.2.13 -websockets==15.0.1 -wrapt==1.17.2 -yarl==1.20.1 -zipp==3.23.0 diff --git a/research/paper/main.pdf b/research/paper/main.pdf index 55ef67a..8a5ddfa 100644 Binary files a/research/paper/main.pdf and b/research/paper/main.pdf differ diff --git a/research/paper/main.tex b/research/paper/main.tex index 5f55446..a5c8a14 100644 --- a/research/paper/main.tex +++ b/research/paper/main.tex @@ -262,7 +262,7 @@ \section{Implementation} The three agents use pydantic-ai with typed structured output and up to three output retries. They work with any model pydantic-ai supports, including OpenAI-compatible endpoints. HTTP 401, 403 and 404 responses stop the run, so a bad key or model name cannot produce an apparently successful run with no tests. -The tool's own test suite (111 tests) drives the whole pipeline offline with scripted models. +The tool's own test suite (over 100 tests) drives the whole pipeline offline with scripted models. % ------------------------------------------------------------------------- \section{Evaluation}\label{sec:evaluation} diff --git a/tests/test_cli.py b/tests/test_cli.py index b851773..dc6ea2c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -185,3 +185,35 @@ def test_benchmark_command_reports_mutation_score(repo, monkeypatch): data = json.loads((repo / ".code2test" / "benchmark.json").read_text()) assert data["mutants_total"] == 2 and data["mutants_killed"] == 2 assert data["generation"]["acceptance_rate"] == 1.0 + + +class _Display: + def __init__(self): + self.messages = [] + + def info(self, message): + self.messages.append(message) + + warning = success = error = info + + +@pytest.fixture +def no_saved_config(tmp_path, monkeypatch): + config_module = importlib.import_module("code2test.cli.config_manager") + monkeypatch.setattr(config_module, "CONFIG_FILE", tmp_path / "none" / "config.json") + monkeypatch.setattr(config_module.ConfigManager, "get_api_key", lambda self: None) + for var in ("CODE2TEST_MODEL", "LLM_BACKEND", "AZURE_OPENAI_DEPLOYMENT", "AZURE_OPENAI_ENDPOINT"): + monkeypatch.delenv(var, raising=False) + + +def test_resolve_model_uses_code2test_model_without_warnings(no_saved_config, monkeypatch): + monkeypatch.setenv("CODE2TEST_MODEL", "anthropic:claude-sonnet-5-5") + display = _Display() + assert model_module.resolve_model(display) == "anthropic:claude-sonnet-5-5" + assert display.messages == [] + + +def test_resolve_model_says_when_it_falls_back_to_the_default(no_saved_config): + display = _Display() + assert model_module.resolve_model(display) == "openai:gpt-4o-mini" + assert len(display.messages) == 1 and "No model configured" in display.messages[0]