diff --git a/.gitignore b/.gitignore index 04a7fff7..b239c5b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,74 @@ config/agent_config.yaml tool/add_header.sh +# Python .venv +venv/ __pycache__/ -*.egg-info +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Tests +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.hypothesis/ +tests/ + +# Jupyter *.ipynb +.ipynb_checkpoints + +# Project specific output/ data/ src/output/ repo_data/ +docs/.cache/ + +# Environment variables .env* *.env **/.env **/.env.* + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Temporary files +*.tmp +*.log +*.bak diff --git a/DOCKER_README.md b/DOCKER_README.md deleted file mode 100644 index 4aa968b6..00000000 --- a/DOCKER_README.md +++ /dev/null @@ -1,30 +0,0 @@ -# CodeWiki Docker Setup - -This document explains how to run CodeWiki using Docker and Docker Compose. - -## Quick Start - -1. **Clone the repository** (if not already done): - ```bash - git clone - cd CodeWiki - ``` - -2. **Set up environment variables**: - ```bash - cp env.example .env - # Edit .env file with your API keys - ``` - -3. **Create network** - ```bash - docker network create codewiki-network - ``` - -3. **Start the services**: - ```bash - docker-compose up -d - ``` - -4. **Access the application**: - - Main web app: http://localhost:8000 diff --git a/README.md b/README.md index b0846545..cb60f1be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# CodeWiki: Automated Repository-Level Documentation Generation +# CodeWiki: Automated Repository-Level Documentation at Scale
@@ -10,7 +10,7 @@ **The first open-source framework for holistic, structured repository-level documentation across multilingual codebases** -[Features](#features) • [Installation](#installation) • [Quick Start](#quick-start) • [Benchmark](#benchmark) • [Documentation](#documentation-structure) • [Demo](https://fsoft-ai4code.github.io/codewiki-demo/) • [Citation](#citation) +[Features](#features) • [Installation](#installation) • [Quick Start](#quick-start) • [Benchmark](#benchmark) • [Demo](https://fsoft-ai4code.github.io/codewiki-demo/) • [Citation](#citation)
@@ -40,14 +40,31 @@ Developers spend **58% of their working time** understanding codebases, yet main ## Features -### Documentation Generation +### 🎯 Two Usage Modes + +CodeWiki offers both **CLI** and **Web Application** interfaces: + +#### CLI Tool +- ✅ **Local Projects** - Generate documentation for codebases on your machine +- ✅ **Git Integration** - Automated branch creation and commit management +- ✅ **Secure Configuration** - API keys stored in system keychain +- ✅ **GitHub Pages** - Generate beautiful, interactive HTML documentation +- ✅ **Progress Tracking** - Real-time progress updates with ETA + +#### Web Application +- ✅ **GitHub URL Input** - Generate docs by repository URL and optional commit ID +- ✅ **Remote Processing** - No need to clone repositories locally +- ✅ **Web Interface** - Easy-to-use browser-based UI + +### 📚 Documentation Generation - ✅ **Repository-Level Documentation** - First framework to generate complete repo-level docs at scale - ✅ **Visual Artifacts** - Automatic generation of architecture diagrams and data flow visualizations - ✅ **Cross-Module References** - Intelligent reference management prevents redundancy - ✅ **Hierarchical Structure** - Multi-level documentation from high-level overviews to detailed APIs +- ✅ **Mermaid Diagrams** - Automatic generation of architecture and data flow diagrams -### Benchmark +### 🔬 Benchmark - ✅ **[CodeWikiBench](https://github.com/FSoft-AI4Code/CodeWikiBench.git)** - First benchmark specifically designed for repository-level documentation @@ -72,9 +89,26 @@ CodeWiki demonstrates significant improvements in high-level and managed languag - Python 3.12+ - Node.js (for mermaid validation) +- Git (optional, for CLI branch management) - Docker (optional, for containerized deployment) -### Standard Installation +### CLI Installation + +Install CodeWiki CLI from source: + +```bash +pip install https://github.com/FSoft-AI4Code/CodeWiki.git +``` + +Verify installation: + +```bash +codewiki --version +``` + +### Web Application Installation + +For the web interface: ```bash # Clone the repository @@ -85,7 +119,7 @@ cd codewiki # macOS brew install node # Linux -sudo apt update && apt install -y nodejs npm +sudo apt update && sudo apt install -y nodejs npm # Create and activate virtual environment python3.12 -m venv .venv @@ -94,27 +128,126 @@ source .venv/bin/activate # On Windows: .venv\Scripts\activate # Install dependencies pip install -r requirements.txt -#Create a `.env` file from the template and edit with your configuration -cp env.example .env +# Create a `.env` file from the template +cp docker/env.example .env +# Edit .env with your API keys and configuration # Start the web application -python run_web_app.py +python codewiki/run_web_app.py -#Access the application at `http://localhost:8000` to generate documentation by github url and commit id (optional) +# Access at http://localhost:8000 ``` ### Docker Installation +For containerized deployment, see [Docker Setup](#docker-deployment) section below. + +--- + +## Quick Start + +### CLI Usage + +#### 1. Configure CodeWiki + +```bash +codewiki config set \ + --api-key YOUR_API_KEY \ + --base-url https://api.anthropic.com \ + --main-model claude-sonnet-4 \ + --cluster-model claude-sonnet-4 +``` + +Verify configuration: + ```bash -# Copy environment configuration -cp env.example .env -# Edit .env with your API keys +codewiki config show +codewiki config validate +``` + +#### 2. Generate Documentation + +```bash +cd /path/to/your/project +codewiki generate +``` + +Documentation will be created in `./docs/` + +#### 3. Generate with GitHub Pages + +```bash +codewiki generate --github-pages +``` + +This creates an interactive HTML viewer at `./docs/index.html` + +--- + +## CLI Commands + +### Configuration Management + +```bash +# Set configuration +codewiki config set --api-key --base-url \ + --main-model --cluster-model + +# Show configuration +codewiki config show + +# Validate configuration +codewiki config validate +``` + +### Documentation Generation + +```bash +# Basic generation +codewiki generate + +# Custom output directory +codewiki generate --output ./documentation -# Create network -docker network create codewiki-network +# Create git branch +codewiki generate --create-branch -# Start services -docker-compose up -d +# Generate GitHub Pages HTML +codewiki generate --github-pages + +# Full-featured +codewiki generate --create-branch --github-pages --verbose +``` + +--- + +## Configuration + +### CLI Configuration + +Configuration is stored in: +- API keys: System keychain (macOS Keychain, Windows Credential Manager, Linux Secret Service) +- Settings: `~/.codewiki/config.json` + +### Web Application Configuration + +Configuration uses environment variables in `.env` file: + +```bash +# LLM API Configuration +MAIN_MODEL=claude-sonnet-4 +FALLBACK_MODEL_1=glm-4p5 +CLUSTER_MODEL=claude-sonnet-4 +LLM_BASE_URL=http://litellm:4000/ +LLM_API_KEY=sk-1234 + +# Application Port +APP_PORT=8000 + +# Optional: Logfire Configuration +LOGFIRE_TOKEN= +LOGFIRE_PROJECT_NAME=codewiki +LOGFIRE_SERVICE_NAME=codewiki ``` --- @@ -164,6 +297,121 @@ Generated documentation includes: - **Sequence Diagrams** - Inter-component communication patterns - **Dependency Graphs** - Module and function dependencies +### Output Structure + +``` +./docs/ +├── overview.md # Start here! +├── module1.md # Module documentation +├── module2.md +├── ... +├── module_tree.json # Module hierarchy +├── first_module_tree.json # Initial clustering +├── metadata.json # Generation details +└── index.html # GitHub Pages viewer (if --github-pages) +``` + +--- + +## Supported Languages + +| Language | Extensions | Support | +|------------|---------------------|---------| +| Python | `.py` | Full | +| Java | `.java` | Full | +| JavaScript | `.js`, `.jsx` | Full | +| TypeScript | `.ts`, `.tsx` | Full | +| C | `.c`, `.h` | Full | +| C++ | `.cpp`, `.hpp`, etc.| Full | +| C# | `.cs` | Full | + +--- + +## Docker Deployment + +### Quick Start with Docker + +1. **Set up environment variables**: + ```bash + # Copy from docker directory + cp docker/env.example .env + # Edit .env with your configuration + ``` + +2. **Create Docker network**: + ```bash + docker network create codewiki-network + ``` + +3. **Start the services**: + ```bash + # From project root + docker-compose -f docker/docker-compose.yml up -d + + # Or from docker directory + cd docker + docker-compose up -d + ``` + +4. **Access the application**: + - Web app: http://localhost:8000 + +### Docker Configuration + +All Docker-related files are in the `docker/` directory: +- `docker/Dockerfile` - Container image definition +- `docker/docker-compose.yml` - Service orchestration +- `docker/env.example` - Environment variables template + +### Stopping Services + +```bash +# From project root +docker-compose -f docker/docker-compose.yml down + +# Or from docker directory +cd docker +docker-compose down +``` + +--- + +## Development + +### Project Structure + +``` +codewiki/ +├── codewiki/ # Main package +│ ├── cli/ # CLI implementation +│ │ ├── commands/ # CLI commands (config, generate) +│ │ ├── models/ # Data models +│ │ ├── utils/ # Utilities +│ │ └── adapters/ # External integrations +│ ├── src/ # Web application +│ │ ├── be/ # Backend (dependency analysis, agents) +│ │ └── fe/ # Frontend (web interface) +│ ├── templates/ # HTML templates +│ └── run_web_app.py # Web app entry point +├── docker/ # Docker configuration +│ ├── Dockerfile +│ ├── docker-compose.yml +│ └── env.example +├── tests/ # Test suite +├── output/ # Generated documentation output +└── README.md # This file +``` + +--- + +## Requirements + +- Python 3.12+ +- Git (optional, for branch management) +- LLM API access (Anthropic Claude, OpenAI, etc.) +- Tree-sitter language parsers (automatically installed) +- System keychain support for CLI (macOS Keychain, Windows Credential Manager, Linux Secret Service) +- Node.js (for mermaid diagram validation) --- @@ -173,29 +421,25 @@ If you use CodeWiki in your research, please cite: ```bibtex @article{codewiki2025, - title={CodeWiki: Automated Repository-Level Documentation Generation with Hierarchical Decomposition and Agentic Processing}, + title={CodeWiki: Automated Repository-Level Documentation at Scale}, author={Your Name}, journal={arXiv preprint arXiv:XXXXX}, year={2025} } ``` - +## License +MIT License - see LICENSE file for details - +- **Live Demo**: [View documentation examples](https://fsoft-ai4code.github.io/codewiki-demo/) +- **Issues**: https://github.com/yourusername/codewiki/issues --- @@ -203,6 +447,6 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file **Made with ❤️ by the CodeWiki Team** -[⬆ Back to Top](#codewiki-automated-repository-level-documentation-generation) +[⬆ Back to Top](#codewiki-automated-repository-level-documentation-at-scale) - \ No newline at end of file + diff --git a/codewiki/__init__.py b/codewiki/__init__.py new file mode 100644 index 00000000..e2e911d2 --- /dev/null +++ b/codewiki/__init__.py @@ -0,0 +1,14 @@ +""" +CodeWiki: Transform codebases into comprehensive documentation using AI-powered analysis. + +This package provides a CLI tool for generating documentation from code repositories. +""" + +__version__ = "1.0.0" +__author__ = "CodeWiki Contributors" +__license__ = "MIT" + +from codewiki.cli.main import cli + +__all__ = ["cli", "__version__"] + diff --git a/codewiki/__main__.py b/codewiki/__main__.py new file mode 100644 index 00000000..bceeeb9a --- /dev/null +++ b/codewiki/__main__.py @@ -0,0 +1,8 @@ +""" +Entry point for running codewiki as a module: python -m codewiki +""" + +from codewiki.cli.main import cli + +if __name__ == "__main__": + cli() \ No newline at end of file diff --git a/codewiki/cli/__init__.py b/codewiki/cli/__init__.py new file mode 100644 index 00000000..e6e485a1 --- /dev/null +++ b/codewiki/cli/__init__.py @@ -0,0 +1,4 @@ +"""CLI module for CodeWiki.""" + +__all__ = [] + diff --git a/codewiki/cli/adapters/__init__.py b/codewiki/cli/adapters/__init__.py new file mode 100644 index 00000000..6204b525 --- /dev/null +++ b/codewiki/cli/adapters/__init__.py @@ -0,0 +1,4 @@ +"""Adapters for integrating with backend modules.""" + +__all__ = [] + diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py new file mode 100644 index 00000000..4a933f05 --- /dev/null +++ b/codewiki/cli/adapters/doc_generator.py @@ -0,0 +1,241 @@ +""" +CLI adapter for documentation generator backend. + +This adapter wraps the existing backend documentation_generator.py +and provides CLI-specific functionality like progress reporting. +""" + +from pathlib import Path +from typing import Dict, Any +import time +import asyncio +import os + + +from codewiki.cli.utils.progress import ProgressTracker +from codewiki.cli.models.job import DocumentationJob, LLMConfig +from codewiki.cli.utils.errors import APIError + +# Import backend modules +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import Config as BackendConfig, set_cli_context + + +class CLIDocumentationGenerator: + """ + CLI adapter for documentation generation with progress reporting. + + This class wraps the backend documentation generator and adds + CLI-specific features like progress tracking and error handling. + """ + + def __init__( + self, + repo_path: Path, + output_dir: Path, + config: Dict[str, Any], + verbose: bool = False, + generate_html: bool = False + ): + """ + Initialize the CLI documentation generator. + + Args: + repo_path: Repository path + output_dir: Output directory + config: LLM configuration + verbose: Enable verbose output + generate_html: Whether to generate HTML viewer + """ + self.repo_path = repo_path + self.output_dir = output_dir + self.config = config + self.verbose = verbose + self.generate_html = generate_html + self.progress_tracker = ProgressTracker(total_stages=5, verbose=verbose) + self.job = DocumentationJob() + + # Setup job metadata + self.job.repository_path = str(repo_path) + self.job.repository_name = repo_path.name + self.job.output_directory = str(output_dir) + self.job.llm_config = LLMConfig( + main_model=config.get('main_model', ''), + cluster_model=config.get('cluster_model', ''), + base_url=config.get('base_url', '') + ) + + def generate(self) -> DocumentationJob: + """ + Generate documentation with progress tracking. + + Returns: + Completed DocumentationJob + + Raises: + APIError: If LLM API call fails + """ + self.job.start() + start_time = time.time() + + try: + # Set CLI context for backend + set_cli_context(True) + + # Create backend config with CLI settings + backend_config = BackendConfig.from_cli( + repo_path=str(self.repo_path), + output_dir=str(self.output_dir), + llm_base_url=self.config.get('base_url'), + llm_api_key=self.config.get('api_key'), + main_model=self.config.get('main_model'), + cluster_model=self.config.get('cluster_model') + ) + + # Run backend documentation generation + asyncio.run(self._run_backend_generation(backend_config)) + + # Stage 4: HTML Generation (optional) + if self.generate_html: + self._run_html_generation() + + # Stage 5: Finalization (metadata already created by backend) + self._finalize_job() + + # Complete job + generation_time = time.time() - start_time + self.job.complete() + + return self.job + + except APIError as e: + self.job.fail(str(e)) + raise + except Exception as e: + self.job.fail(str(e)) + raise + + async def _run_backend_generation(self, backend_config: BackendConfig): + """Run the backend documentation generation with progress tracking.""" + + # Stage 1: Dependency Analysis + self.progress_tracker.start_stage(1, "Dependency Analysis") + if self.verbose: + self.progress_tracker.update_stage(0.2, "Initializing dependency analyzer...") + + # Create documentation generator + doc_generator = DocumentationGenerator(backend_config) + + if self.verbose: + self.progress_tracker.update_stage(0.5, "Parsing source files...") + + # Build dependency graph + try: + components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph() + self.job.statistics.total_files_analyzed = len(components) + self.job.statistics.leaf_nodes = len(leaf_nodes) + + if self.verbose: + self.progress_tracker.update_stage(1.0, f"Found {len(leaf_nodes)} leaf nodes") + except Exception as e: + raise APIError(f"Dependency analysis failed: {e}") + + self.progress_tracker.complete_stage() + + # Stage 2: Module Clustering + self.progress_tracker.start_stage(2, "Module Clustering") + if self.verbose: + self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...") + + # Import clustering function + from codewiki.src.be.cluster_modules import cluster_modules + from codewiki.src.utils import file_manager + from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME + + working_dir = str(self.output_dir.absolute()) + file_manager.ensure_directory(working_dir) + first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) + module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) + + try: + if os.path.exists(first_module_tree_path): + module_tree = file_manager.load_json(first_module_tree_path) + else: + module_tree = cluster_modules(leaf_nodes, components, backend_config) + file_manager.save_json(module_tree, first_module_tree_path) + + file_manager.save_json(module_tree, module_tree_path) + self.job.module_count = len(module_tree) + + if self.verbose: + self.progress_tracker.update_stage(1.0, f"Created {len(module_tree)} modules") + except Exception as e: + raise APIError(f"Module clustering failed: {e}") + + self.progress_tracker.complete_stage() + + # Stage 3: Documentation Generation + self.progress_tracker.start_stage(3, "Documentation Generation") + if self.verbose: + self.progress_tracker.update_stage(0.1, "Generating module documentation...") + + try: + # Run the actual documentation generation + await doc_generator.generate_module_documentation(components, leaf_nodes) + + if self.verbose: + self.progress_tracker.update_stage(0.9, "Creating repository overview...") + + # Create metadata + doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes)) + + # Collect generated files + for file_path in os.listdir(working_dir): + if file_path.endswith('.md') or file_path.endswith('.json'): + self.job.files_generated.append(file_path) + + except Exception as e: + raise APIError(f"Documentation generation failed: {e}") + + self.progress_tracker.complete_stage() + + def _run_html_generation(self): + """Run HTML generation stage.""" + self.progress_tracker.start_stage(4, "HTML Generation") + + from codewiki.cli.html_generator import HTMLGenerator + + # Generate HTML + html_generator = HTMLGenerator() + + if self.verbose: + self.progress_tracker.update_stage(0.3, "Loading module tree and metadata...") + + repo_info = html_generator.detect_repository_info(self.repo_path) + + # Generate HTML with auto-loading of module_tree and metadata from docs_dir + output_path = self.output_dir / "index.html" + html_generator.generate( + output_path=output_path, + title=repo_info['name'], + repository_url=repo_info['url'], + github_pages_url=repo_info['github_pages_url'], + docs_dir=self.output_dir # Auto-load module_tree and metadata from here + ) + + self.job.files_generated.append("index.html") + + if self.verbose: + self.progress_tracker.update_stage(1.0, "Generated index.html") + + self.progress_tracker.complete_stage() + + def _finalize_job(self): + """Finalize the job (metadata already created by backend).""" + # Just verify metadata exists + metadata_path = self.output_dir / "metadata.json" + if not metadata_path.exists(): + # Create our own if backend didn't + with open(metadata_path, 'w') as f: + f.write(self.job.to_json()) + diff --git a/codewiki/cli/commands/__init__.py b/codewiki/cli/commands/__init__.py new file mode 100644 index 00000000..b9cc1b77 --- /dev/null +++ b/codewiki/cli/commands/__init__.py @@ -0,0 +1,4 @@ +"""CLI command implementations.""" + +__all__ = [] + diff --git a/codewiki/cli/commands/config.py b/codewiki/cli/commands/config.py new file mode 100644 index 00000000..87a6873a --- /dev/null +++ b/codewiki/cli/commands/config.py @@ -0,0 +1,381 @@ +""" +Configuration commands for CodeWiki CLI. +""" + +import json +import sys +import click +from typing import Optional + +from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.utils.errors import ( + ConfigurationError, + handle_error, + EXIT_SUCCESS, + EXIT_CONFIG_ERROR +) +from codewiki.cli.utils.validation import ( + validate_url, + validate_api_key, + validate_model_name, + is_top_tier_model, + mask_api_key +) + + +@click.group(name="config") +def config_group(): + """Manage CodeWiki configuration (API credentials and settings).""" + pass + + +@config_group.command(name="set") +@click.option( + "--api-key", + type=str, + help="LLM API key (stored securely in system keychain)" +) +@click.option( + "--base-url", + type=str, + help="LLM API base URL (e.g., https://api.anthropic.com)" +) +@click.option( + "--main-model", + type=str, + help="Primary model for documentation generation" +) +@click.option( + "--cluster-model", + type=str, + help="Model for module clustering (recommend top-tier)" +) +def config_set( + api_key: Optional[str], + base_url: Optional[str], + main_model: Optional[str], + cluster_model: Optional[str] +): + """ + Set configuration values for CodeWiki. + + API keys are stored securely in your system keychain: + • macOS: Keychain Access + • Windows: Credential Manager + • Linux: Secret Service (GNOME Keyring, KWallet) + + Examples: + + \b + # Set all configuration + $ codewiki config set --api-key sk-abc123 --base-url https://api.anthropic.com \\ + --main-model claude-sonnet-4 --cluster-model claude-sonnet-4 + + \b + # Update only API key + $ codewiki config set --api-key sk-new-key + """ + try: + # Check if at least one option is provided + if not any([api_key, base_url, main_model, cluster_model]): + click.echo("No options provided. Use --help for usage information.") + sys.exit(EXIT_CONFIG_ERROR) + + # Validate inputs before saving + validated_data = {} + + if api_key: + validated_data['api_key'] = validate_api_key(api_key) + + if base_url: + validated_data['base_url'] = validate_url(base_url) + + if main_model: + validated_data['main_model'] = validate_model_name(main_model) + + if cluster_model: + validated_data['cluster_model'] = validate_model_name(cluster_model) + + # Create config manager and save + manager = ConfigManager() + manager.load() # Load existing config if present + + manager.save( + api_key=validated_data.get('api_key'), + base_url=validated_data.get('base_url'), + main_model=validated_data.get('main_model'), + cluster_model=validated_data.get('cluster_model') + ) + + # Display success messages + click.echo() + if api_key: + if manager.keyring_available: + click.secho("✓ API key saved to system keychain", fg="green") + else: + click.secho( + "⚠️ System keychain unavailable. API key stored in encrypted file.", + fg="yellow" + ) + + if base_url: + click.secho(f"✓ Base URL: {base_url}", fg="green") + + if main_model: + click.secho(f"✓ Main model: {main_model}", fg="green") + + if cluster_model: + click.secho(f"✓ Cluster model: {cluster_model}", fg="green") + + # Warn if not using top-tier model for clustering + if not is_top_tier_model(cluster_model): + click.secho( + "\n⚠️ Cluster model is not a top-tier LLM. " + "Documentation quality may be suboptimal.", + fg="yellow" + ) + click.echo( + " Recommended models: claude-opus, claude-sonnet-4, gpt-4, gpt-4-turbo" + ) + + click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True)) + + except ConfigurationError as e: + click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) + sys.exit(e.exit_code) + except Exception as e: + sys.exit(handle_error(e)) + + +@config_group.command(name="show") +@click.option( + "--json", + "output_json", + is_flag=True, + help="Output in JSON format" +) +def config_show(output_json: bool): + """ + Display current configuration. + + API keys are masked for security (showing only first and last 4 characters). + + Examples: + + \b + # Display configuration + $ codewiki config show + + \b + # Display as JSON + $ codewiki config show --json + """ + try: + manager = ConfigManager() + + if not manager.load(): + click.secho("\n✗ Configuration not found.", fg="red", err=True) + click.echo("\nPlease run 'codewiki config set' to configure your API credentials:") + click.echo(" codewiki config set --api-key --base-url \\") + click.echo(" --main-model --cluster-model ") + click.echo("\nFor more help: codewiki config set --help") + sys.exit(EXIT_CONFIG_ERROR) + + config = manager.get_config() + api_key = manager.get_api_key() + + if output_json: + # JSON output + output = { + "api_key": mask_api_key(api_key) if api_key else "Not set", + "api_key_storage": "keychain" if manager.keyring_available else "encrypted_file", + "base_url": config.base_url if config else "", + "main_model": config.main_model if config else "", + "cluster_model": config.cluster_model if config else "", + "default_output": config.default_output if config else "docs", + "config_file": str(manager.config_file_path) + } + click.echo(json.dumps(output, indent=2)) + else: + # Human-readable output + click.echo() + click.secho("CodeWiki Configuration", fg="blue", bold=True) + click.echo("━" * 40) + click.echo() + + click.secho("Credentials", fg="cyan", bold=True) + if api_key: + storage = "system keychain" if manager.keyring_available else "encrypted file" + click.echo(f" API Key: {mask_api_key(api_key)} (in {storage})") + else: + click.secho(" API Key: Not set", fg="yellow") + + click.echo() + click.secho("API Settings", fg="cyan", bold=True) + if config: + click.echo(f" Base URL: {config.base_url or 'Not set'}") + click.echo(f" Main Model: {config.main_model or 'Not set'}") + click.echo(f" Cluster Model: {config.cluster_model or 'Not set'}") + else: + click.secho(" Not configured", fg="yellow") + + click.echo() + click.secho("Output Settings", fg="cyan", bold=True) + if config: + click.echo(f" Default Output: {config.default_output}") + + click.echo() + click.echo(f"Configuration file: {manager.config_file_path}") + click.echo() + + except Exception as e: + sys.exit(handle_error(e)) + + +@config_group.command(name="validate") +@click.option( + "--quick", + is_flag=True, + help="Skip API connectivity test" +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed validation steps" +) +def config_validate(quick: bool, verbose: bool): + """ + Validate configuration and test LLM API connectivity. + + Checks: + • Configuration file exists and is valid + • API key is present + • API settings are correctly formatted + • (Optional) API connectivity test + + Examples: + + \b + # Full validation with API test + $ codewiki config validate + + \b + # Quick validation (config only) + $ codewiki config validate --quick + + \b + # Verbose output + $ codewiki config validate --verbose + """ + try: + click.echo() + click.secho("Validating configuration...", fg="blue", bold=True) + click.echo() + + manager = ConfigManager() + + # Step 1: Check config file + if verbose: + click.echo("[1/5] Checking configuration file...") + click.echo(f" Path: {manager.config_file_path}") + + if not manager.load(): + click.secho("✗ Configuration file not found", fg="red") + click.echo() + click.echo("Error: Configuration is incomplete. Run 'codewiki config set --help' for setup instructions.") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(" ✓ File exists", fg="green") + click.secho(" ✓ Valid JSON format", fg="green") + else: + click.secho("✓ Configuration file exists", fg="green") + + # Step 2: Check API key + if verbose: + click.echo() + click.echo("[2/5] Checking API key...") + storage = "system keychain" if manager.keyring_available else "encrypted file" + click.echo(f" Storage: {storage}") + + api_key = manager.get_api_key() + if not api_key: + click.secho("✗ API key missing", fg="red") + click.echo() + click.echo("Error: API key not set. Run 'codewiki config set --api-key '") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(f" ✓ API key retrieved", fg="green") + click.secho(f" ✓ Length: {len(api_key)} characters", fg="green") + else: + click.secho("✓ API key present (stored in keychain)", fg="green") + + # Step 3: Check base URL + config = manager.get_config() + if verbose: + click.echo() + click.echo("[3/5] Checking base URL...") + click.echo(f" URL: {config.base_url}") + + if not config.base_url: + click.secho("✗ Base URL not set", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + try: + validate_url(config.base_url) + if verbose: + click.secho(" ✓ Valid HTTPS URL", fg="green") + else: + click.secho(f"✓ Base URL valid: {config.base_url}", fg="green") + except ConfigurationError as e: + click.secho(f"✗ Invalid base URL: {e.message}", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + # Step 4: Check models + if verbose: + click.echo() + click.echo("[4/5] Checking model configuration...") + click.echo(f" Main model: {config.main_model}") + click.echo(f" Cluster model: {config.cluster_model}") + + if not config.main_model or not config.cluster_model: + click.secho("✗ Models not configured", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + if verbose: + click.secho(" ✓ Models configured", fg="green") + else: + click.secho(f"✓ Main model configured: {config.main_model}", fg="green") + click.secho(f"✓ Cluster model configured: {config.cluster_model}", fg="green") + + # Warn about non-top-tier cluster model + if not is_top_tier_model(config.cluster_model): + click.secho( + "⚠️ Cluster model is not top-tier. Consider using claude-sonnet-4 or gpt-4.", + fg="yellow" + ) + + # Step 5: API connectivity test (unless --quick) + if not quick: + try: + from openai import OpenAI + client = OpenAI(api_key=api_key, base_url=config.base_url) + response = client.models.list() + click.secho("✓ API connectivity test successful", fg="green") + except Exception as e: + click.secho("✗ API connectivity test failed", fg="red") + sys.exit(EXIT_CONFIG_ERROR) + + # Success + click.echo() + click.secho("✓ Configuration is valid!", fg="green", bold=True) + click.echo() + + except ConfigurationError as e: + click.secho(f"\n✗ Configuration error: {e.message}", fg="red", err=True) + sys.exit(e.exit_code) + except Exception as e: + sys.exit(handle_error(e, verbose=verbose)) + diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py new file mode 100644 index 00000000..a4a3e09f --- /dev/null +++ b/codewiki/cli/commands/generate.py @@ -0,0 +1,260 @@ +""" +Generate command for documentation generation. +""" + +import sys +import logging +from pathlib import Path +from typing import Optional +import click +import time + +from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.utils.errors import ( + ConfigurationError, + RepositoryError, + APIError, + handle_error, + EXIT_SUCCESS, +) +from codewiki.cli.utils.repo_validator import ( + validate_repository, + check_writable_output, + is_git_repository, + get_git_commit_hash, + get_git_branch, +) +from codewiki.cli.utils.logging import create_logger +from codewiki.cli.adapters.doc_generator import CLIDocumentationGenerator +from codewiki.cli.utils.instructions import display_post_generation_instructions +from codewiki.cli.models.job import GenerationOptions + + +@click.command(name="generate") +@click.option( + "--output", + "-o", + type=click.Path(), + default="docs", + help="Output directory for generated documentation (default: ./docs)", +) +@click.option( + "--create-branch", + is_flag=True, + help="Create a new git branch for documentation changes", +) +@click.option( + "--github-pages", + is_flag=True, + help="Generate index.html for GitHub Pages deployment", +) +@click.option( + "--no-cache", + is_flag=True, + help="Force full regeneration, ignoring cache", +) +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed progress and debug information", +) +@click.pass_context +def generate_command( + ctx, + output: str, + create_branch: bool, + github_pages: bool, + no_cache: bool, + verbose: bool +): + """ + Generate comprehensive documentation for a code repository. + + Analyzes the current repository and generates documentation using LLM-powered + analysis. Documentation is output to ./docs/ by default. + + Examples: + + \b + # Basic generation + $ codewiki generate + + \b + # With git branch creation and GitHub Pages + $ codewiki generate --create-branch --github-pages + + \b + # Force full regeneration + $ codewiki generate --no-cache + """ + logger = create_logger(verbose=verbose) + start_time = time.time() + + # Suppress httpx INFO logs + logging.getLogger("httpx").setLevel(logging.WARNING) + + try: + # Pre-generation checks + logger.step("Validating configuration...", 1, 4) + + # Load configuration + config_manager = ConfigManager() + if not config_manager.load(): + raise ConfigurationError( + "Configuration not found or invalid.\n\n" + "Please run 'codewiki config set' to configure your LLM API credentials:\n" + " codewiki config set --api-key --base-url \\\n" + " --main-model --cluster-model \n\n" + "For more help: codewiki config --help" + ) + + if not config_manager.is_configured(): + raise ConfigurationError( + "Configuration is incomplete. Please run 'codewiki config validate'" + ) + + config = config_manager.get_config() + api_key = config_manager.get_api_key() + + logger.success("Configuration valid") + + # Validate repository + logger.step("Validating repository...", 2, 4) + + repo_path = Path.cwd() + repo_path, languages = validate_repository(repo_path) + + logger.success(f"Repository valid: {repo_path.name}") + if verbose: + logger.debug(f"Detected languages: {', '.join(f'{lang} ({count} files)' for lang, count in languages)}") + + # Check git repository + if not is_git_repository(repo_path): + if create_branch: + raise RepositoryError( + "Not a git repository.\n\n" + "The --create-branch flag requires a git repository.\n\n" + "To initialize a git repository: git init" + ) + else: + logger.warning("Not a git repository. Git features unavailable.") + + # Validate output directory + output_dir = Path(output).expanduser().resolve() + check_writable_output(output_dir.parent) + + logger.success(f"Output directory: {output_dir}") + + # Check for existing documentation + if output_dir.exists() and list(output_dir.glob("*.md")): + if not click.confirm( + f"\n{output_dir} already contains documentation. Overwrite?", + default=True + ): + logger.info("Generation cancelled by user.") + sys.exit(EXIT_SUCCESS) + + # Git branch creation (if requested) + branch_name = None + if create_branch: + logger.step("Creating git branch...", 3, 4) + + from codewiki.cli.git_manager import GitManager + + git_manager = GitManager(repo_path) + + # Check clean working directory + is_clean, status_msg = git_manager.check_clean_working_directory() + if not is_clean: + raise RepositoryError( + "Working directory has uncommitted changes.\n\n" + f"{status_msg}\n\n" + "Cannot create documentation branch with uncommitted changes.\n" + "Please commit or stash your changes first:\n" + " git add -A && git commit -m \"Your message\"\n" + " # or\n" + " git stash" + ) + + # Create branch + branch_name = git_manager.create_documentation_branch() + logger.success(f"Created branch: {branch_name}") + + # Generate documentation + logger.step("Generating documentation...", 4, 4) + click.echo() + + # Create generation options + generation_options = GenerationOptions( + create_branch=create_branch, + github_pages=github_pages, + no_cache=no_cache, + custom_output=output if output != "docs" else None + ) + + # Create generator + generator = CLIDocumentationGenerator( + repo_path=repo_path, + output_dir=output_dir, + config={ + 'main_model': config.main_model, + 'cluster_model': config.cluster_model, + 'base_url': config.base_url, + 'api_key': api_key, + }, + verbose=verbose, + generate_html=github_pages + ) + + # Run generation + job = generator.generate() + + # Post-generation + generation_time = time.time() - start_time + + # Get repository info + repo_url = None + commit_hash = get_git_commit_hash(repo_path) + current_branch = get_git_branch(repo_path) + + if is_git_repository(repo_path): + try: + import git + repo = git.Repo(repo_path) + if repo.remotes: + repo_url = repo.remotes.origin.url + except: + pass + + # Display instructions + display_post_generation_instructions( + output_dir=output_dir, + repo_name=repo_path.name, + repo_url=repo_url, + branch_name=branch_name, + github_pages=github_pages, + files_generated=job.files_generated, + statistics={ + 'module_count': job.module_count, + 'total_files_analyzed': job.statistics.total_files_analyzed, + 'generation_time': generation_time, + 'total_tokens_used': job.statistics.total_tokens_used, + } + ) + + except ConfigurationError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except RepositoryError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except APIError as e: + logger.error(e.message) + sys.exit(e.exit_code) + except KeyboardInterrupt: + click.echo("\n\nInterrupted by user") + sys.exit(130) + except Exception as e: + sys.exit(handle_error(e, verbose=verbose)) + diff --git a/codewiki/cli/config_manager.py b/codewiki/cli/config_manager.py new file mode 100644 index 00000000..b2b333fa --- /dev/null +++ b/codewiki/cli/config_manager.py @@ -0,0 +1,226 @@ +""" +Configuration manager with keyring integration for secure credential storage. +""" + +import json +from pathlib import Path +from typing import Optional +import keyring +from keyring.errors import KeyringError + +from codewiki.cli.models.config import Configuration +from codewiki.cli.utils.errors import ConfigurationError, FileSystemError +from codewiki.cli.utils.fs import ensure_directory, safe_write, safe_read + + +# Keyring configuration +KEYRING_SERVICE = "codewiki" +KEYRING_API_KEY_ACCOUNT = "api_key" + +# Configuration file location +CONFIG_DIR = Path.home() / ".codewiki" +CONFIG_FILE = CONFIG_DIR / "config.json" +CONFIG_VERSION = "1.0" + + +class ConfigManager: + """ + Manages CodeWiki configuration with secure keyring storage for API keys. + + Storage: + - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager, + Linux Secret Service) + - Other settings: ~/.codewiki/config.json + """ + + def __init__(self): + """Initialize the configuration manager.""" + self._api_key: Optional[str] = None + self._config: Optional[Configuration] = None + self._keyring_available = self._check_keyring_available() + + def _check_keyring_available(self) -> bool: + """Check if system keyring is available.""" + try: + # Try to get/set a test value + keyring.get_password(KEYRING_SERVICE, "__test__") + return True + except KeyringError: + return False + + def load(self) -> bool: + """ + Load configuration from file and keyring. + + Returns: + True if configuration exists, False otherwise + """ + # Load from JSON file + if not CONFIG_FILE.exists(): + return False + + try: + content = safe_read(CONFIG_FILE) + data = json.loads(content) + + # Validate version + if data.get('version') != CONFIG_VERSION: + # Could implement migration here + pass + + self._config = Configuration.from_dict(data) + + # Load API key from keyring + try: + self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + except KeyringError: + # Keyring unavailable, API key will be None + pass + + return True + except (json.JSONDecodeError, FileSystemError) as e: + raise ConfigurationError(f"Failed to load configuration: {e}") + + def save( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + main_model: Optional[str] = None, + cluster_model: Optional[str] = None, + default_output: Optional[str] = None + ): + """ + Save configuration to file and keyring. + + Args: + api_key: API key (stored in keyring) + base_url: LLM API base URL + main_model: Primary model + cluster_model: Clustering model + default_output: Default output directory + """ + # Ensure config directory exists + try: + ensure_directory(CONFIG_DIR) + except FileSystemError as e: + raise ConfigurationError(f"Cannot create config directory: {e}") + + # Load existing config or create new + if self._config is None: + if CONFIG_FILE.exists(): + self.load() + else: + self._config = Configuration( + base_url="", + main_model="", + cluster_model="", + default_output="docs" + ) + + # Update fields if provided + if base_url is not None: + self._config.base_url = base_url + if main_model is not None: + self._config.main_model = main_model + if cluster_model is not None: + self._config.cluster_model = cluster_model + if default_output is not None: + self._config.default_output = default_output + + # Validate configuration + self._config.validate() + + # Save API key to keyring + if api_key is not None: + self._api_key = api_key + try: + keyring.set_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT, api_key) + except KeyringError as e: + # Fallback: warn about keyring unavailability + raise ConfigurationError( + f"System keychain unavailable: {e}\n" + f"Please ensure your system keychain is properly configured." + ) + + # Save non-sensitive config to JSON + config_data = { + "version": CONFIG_VERSION, + **self._config.to_dict() + } + + try: + safe_write(CONFIG_FILE, json.dumps(config_data, indent=2)) + except FileSystemError as e: + raise ConfigurationError(f"Failed to save configuration: {e}") + + def get_api_key(self) -> Optional[str]: + """ + Get API key from keyring. + + Returns: + API key or None if not set + """ + if self._api_key is None: + try: + self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + except KeyringError: + pass + + return self._api_key + + def get_config(self) -> Optional[Configuration]: + """ + Get current configuration. + + Returns: + Configuration object or None if not loaded + """ + return self._config + + def is_configured(self) -> bool: + """ + Check if configuration is complete and valid. + + Returns: + True if configured, False otherwise + """ + if self._config is None: + return False + + # Check if API key is set + if self.get_api_key() is None: + return False + + # Check if config is complete + return self._config.is_complete() + + def delete_api_key(self): + """Delete API key from keyring.""" + try: + keyring.delete_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT) + self._api_key = None + except KeyringError: + pass + + def clear(self): + """Clear all configuration (file and keyring).""" + # Delete API key from keyring + self.delete_api_key() + + # Delete config file + if CONFIG_FILE.exists(): + CONFIG_FILE.unlink() + + self._config = None + self._api_key = None + + @property + def keyring_available(self) -> bool: + """Check if keyring is available.""" + return self._keyring_available + + @property + def config_file_path(self) -> Path: + """Get configuration file path.""" + return CONFIG_FILE + diff --git a/codewiki/cli/git_manager.py b/codewiki/cli/git_manager.py new file mode 100644 index 00000000..e75a9fa0 --- /dev/null +++ b/codewiki/cli/git_manager.py @@ -0,0 +1,227 @@ +""" +Git operations manager for CodeWiki CLI. +""" + +from pathlib import Path +from datetime import datetime +from typing import Optional, Tuple +import git +from git.exc import GitCommandError + +from codewiki.cli.utils.errors import RepositoryError + + +class GitManager: + """ + Manages git operations for documentation generation. + + Handles: + - Status checking + - Branch creation + - Committing documentation + - Remote detection + """ + + def __init__(self, repo_path: Path): + """ + Initialize git manager. + + Args: + repo_path: Path to git repository + + Raises: + RepositoryError: If not a valid git repository + """ + self.repo_path = Path(repo_path).expanduser().resolve() + + try: + self.repo = git.Repo(repo_path, search_parent_directories=True) + except git.InvalidGitRepositoryError: + raise RepositoryError( + f"Not a git repository: {repo_path}\n\n" + "To initialize a git repository: git init" + ) + + def check_clean_working_directory(self) -> Tuple[bool, str]: + """ + Check if working directory is clean (no uncommitted changes). + + Returns: + Tuple of (is_clean, status_message) + """ + if self.repo.is_dirty(untracked_files=True): + status_lines = [] + + # Changed files + changed = [item.a_path for item in self.repo.index.diff(None)] + if changed: + status_lines.append(f"Modified: {', '.join(changed[:3])}") + if len(changed) > 3: + status_lines.append(f"... and {len(changed) - 3} more") + + # Untracked files + untracked = self.repo.untracked_files + if untracked: + status_lines.append(f"Untracked: {', '.join(untracked[:3])}") + if len(untracked) > 3: + status_lines.append(f"... and {len(untracked) - 3} more") + + return False, "\n".join(status_lines) + + return True, "Working directory is clean" + + def create_documentation_branch(self, force: bool = False) -> str: + """ + Create a new documentation branch with timestamp. + + Args: + force: Force creation even if dirty working directory + + Returns: + Branch name + + Raises: + RepositoryError: If working directory is dirty (unless force=True) + """ + # Check working directory + if not force: + is_clean, status_msg = self.check_clean_working_directory() + if not is_clean: + raise RepositoryError( + "Working directory has uncommitted changes.\n\n" + f"{status_msg}\n\n" + "Cannot create documentation branch with uncommitted changes.\n" + "Please commit or stash your changes first:\n" + " git status\n" + " git add -A && git commit -m \"Your message\"\n" + " # or\n" + " git stash\n\n" + "Then re-run: codewiki generate --create-branch" + ) + + # Generate branch name with timestamp + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + branch_name = f"docs/codewiki-{timestamp}" + + # Check if branch already exists (shouldn't happen with timestamp) + existing_branches = [b.name for b in self.repo.branches] + if branch_name in existing_branches: + # Append counter + counter = 1 + while f"{branch_name}-{counter}" in existing_branches: + counter += 1 + branch_name = f"{branch_name}-{counter}" + + try: + # Create and checkout new branch + new_branch = self.repo.create_head(branch_name) + new_branch.checkout() + return branch_name + except GitCommandError as e: + raise RepositoryError(f"Failed to create branch: {e}") + + def commit_documentation( + self, + docs_path: Path, + message: Optional[str] = None + ) -> str: + """ + Commit generated documentation. + + Args: + docs_path: Path to documentation directory + message: Commit message (optional) + + Returns: + Commit hash + + Raises: + RepositoryError: If commit fails + """ + if message is None: + message = "Add generated documentation\n\nGenerated by CodeWiki CLI" + + try: + # Add documentation files + self.repo.index.add([str(docs_path)]) + + # Commit + commit = self.repo.index.commit(message) + + return commit.hexsha + except GitCommandError as e: + raise RepositoryError(f"Failed to commit documentation: {e}") + + def get_remote_url(self, remote_name: str = "origin") -> Optional[str]: + """ + Get remote repository URL. + + Args: + remote_name: Name of remote (default: origin) + + Returns: + Remote URL or None if no remote + """ + try: + remote = self.repo.remote(remote_name) + return remote.url + except ValueError: + return None + + def get_current_branch(self) -> str: + """ + Get current branch name. + + Returns: + Branch name + """ + try: + return self.repo.active_branch.name + except TypeError: + # Detached HEAD + return "HEAD" + + def get_commit_hash(self) -> str: + """ + Get current commit hash. + + Returns: + Commit hash + """ + return self.repo.head.commit.hexsha + + def branch_exists(self, branch_name: str) -> bool: + """ + Check if a branch exists. + + Args: + branch_name: Branch name to check + + Returns: + True if exists, False otherwise + """ + return branch_name in [b.name for b in self.repo.branches] + + def get_github_pr_url(self, branch_name: str) -> Optional[str]: + """ + Get GitHub PR creation URL for a branch. + + Args: + branch_name: Branch name + + Returns: + PR URL or None if not a GitHub repo + """ + remote_url = self.get_remote_url() + if not remote_url or "github.com" not in remote_url: + return None + + # Clean URL + base_url = remote_url.rstrip('/').replace('.git', '') + + # Convert SSH to HTTPS + if base_url.startswith('git@github.com:'): + base_url = base_url.replace('git@github.com:', '/') + + return f"{base_url}/compare/{branch_name}" + diff --git a/codewiki/cli/html_generator.py b/codewiki/cli/html_generator.py new file mode 100644 index 00000000..817c77e7 --- /dev/null +++ b/codewiki/cli/html_generator.py @@ -0,0 +1,284 @@ +""" +HTML generator for GitHub Pages documentation viewer. +""" + +import json +from pathlib import Path +from typing import Optional, Dict, Any + +from codewiki.cli.utils.errors import FileSystemError +from codewiki.cli.utils.fs import safe_write, safe_read + + +class HTMLGenerator: + """ + Generates static HTML documentation viewer for GitHub Pages. + + Creates a self-contained index.html with embedded styles, scripts, + and configuration for client-side markdown rendering. + """ + + def __init__(self, template_dir: Optional[Path] = None): + """ + Initialize HTML generator. + + Args: + template_dir: Path to template directory (default: package templates) + """ + if template_dir is None: + # Use package templates + template_dir = Path(__file__).parent.parent / "templates" / "github_pages" + + self.template_dir = Path(template_dir) + + + def load_module_tree(self, docs_dir: Path) -> Dict[str, Any]: + """ + Load module tree from documentation directory. + + Args: + docs_dir: Documentation directory path + + Returns: + Module tree structure + """ + module_tree_path = docs_dir / "module_tree.json" + if not module_tree_path.exists(): + # Fallback to a simple structure + return { + "Overview": { + "description": "Repository overview", + "components": [], + "children": {} + } + } + + try: + content = safe_read(module_tree_path) + return json.loads(content) + except Exception as e: + raise FileSystemError(f"Failed to load module tree: {e}") + + def load_metadata(self, docs_dir: Path) -> Optional[Dict[str, Any]]: + """ + Load metadata from documentation directory. + + Args: + docs_dir: Documentation directory path + + Returns: + Metadata dictionary or None if not found + """ + metadata_path = docs_dir / "metadata.json" + if not metadata_path.exists(): + return None + + try: + content = safe_read(metadata_path) + return json.loads(content) + except Exception: + # Non-critical, return None + return None + + def generate( + self, + output_path: Path, + title: str, + module_tree: Optional[Dict[str, Any]] = None, + repository_url: Optional[str] = None, + github_pages_url: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + docs_dir: Optional[Path] = None, + metadata: Optional[Dict[str, Any]] = None + ): + """ + Generate HTML documentation viewer. + + Args: + output_path: Output file path (index.html) + title: Documentation title + module_tree: Module tree structure (auto-loaded from docs_dir if not provided) + repository_url: GitHub repository URL + github_pages_url: Expected GitHub Pages URL + config: Additional configuration + docs_dir: Documentation directory (for auto-loading module_tree and metadata) + metadata: Metadata dictionary (auto-loaded from docs_dir if not provided) + """ + # Auto-load module_tree and metadata from docs_dir if not provided + if docs_dir: + if module_tree is None: + module_tree = self.load_module_tree(docs_dir) + if metadata is None: + metadata = self.load_metadata(docs_dir) + + # Default values + if module_tree is None: + module_tree = {} + if config is None: + config = {} + + # Load template + template_path = self.template_dir / "viewer_template.html" + if not template_path.exists(): + raise FileSystemError(f"Template not found: {template_path}") + + template_content = safe_read(template_path) + + # Build info content HTML + info_content = self._build_info_content(metadata) + show_info = "block" if info_content else "none" + + # Build repository link + repo_link = "" + if repository_url: + repo_link = f'🔗 View Repository' + + # Determine docs base path + # For GitHub Pages: relative path to docs folder + # For local: relative path to docs folder + docs_base_path = "" + if docs_dir and output_path.parent != docs_dir: + # Calculate relative path from output to docs + try: + docs_base_path = Path(docs_dir.name).as_posix() + except Exception: + docs_base_path = "." + + # Prepare JSON data for embedding + config_json = json.dumps(config, indent=2) + module_tree_json = json.dumps(module_tree, indent=2) + metadata_json = json.dumps(metadata, indent=2) if metadata else "null" + + # Replace placeholders + html_content = template_content + replacements = { + "{{TITLE}}": self._escape_html(title), + "{{REPO_LINK}}": repo_link, + "{{SHOW_INFO}}": show_info, + "{{INFO_CONTENT}}": info_content, + "{{CONFIG_JSON}}": config_json, + "{{MODULE_TREE_JSON}}": module_tree_json, + "{{METADATA_JSON}}": metadata_json, + "{{DOCS_BASE_PATH}}": docs_base_path, + } + + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + # Write output + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + safe_write(output_path, html_content) + + def _build_info_content(self, metadata: Optional[Dict[str, Any]]) -> str: + """ + Build HTML content for repo info section. + + Args: + metadata: Metadata dictionary + + Returns: + HTML string for info content + """ + if not metadata or not metadata.get('generation_info'): + return "" + + info = metadata.get('generation_info', {}) + stats = metadata.get('statistics', {}) + + html_parts = [] + + if info.get('main_model'): + html_parts.append(f'
Model: {self._escape_html(info["main_model"])}
') + + if info.get('timestamp'): + try: + from datetime import datetime + timestamp = info['timestamp'] + # Parse ISO format timestamp + if isinstance(timestamp, str): + dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + formatted_date = dt.strftime('%Y-%m-%d') + html_parts.append(f'
Generated: {formatted_date}
') + except Exception: + pass + + if info.get('commit_id'): + commit_short = info['commit_id'][:8] + html_parts.append(f'
Commit: {commit_short}
') + + if stats.get('total_components'): + components_str = f"{stats['total_components']:,}" + html_parts.append(f'
Components: {components_str}
') + + if stats.get('max_depth'): + html_parts.append(f'
Max Depth: {stats["max_depth"]}
') + + return '\n '.join(html_parts) + + def _escape_html(self, text: str) -> str: + """ + Escape HTML special characters. + + Args: + text: Text to escape + + Returns: + Escaped text + """ + return (text + .replace('&', '&') + .replace('<', '<') + .replace('>', '>') + .replace('"', '"') + .replace("'", ''')) + + + + def detect_repository_info(self, repo_path: Path) -> Dict[str, Optional[str]]: + """ + Detect repository information from git. + + Args: + repo_path: Repository path + + Returns: + Dictionary with 'name', 'url', 'github_pages_url' + """ + info = { + 'name': repo_path.name, + 'url': None, + 'github_pages_url': None, + } + + try: + import git + repo = git.Repo(repo_path) + + # Get repository name + info['name'] = repo_path.name + + # Get remote URL + if repo.remotes: + remote_url = repo.remotes.origin.url + + # Clean URL + if remote_url.startswith('git@github.com:'): + remote_url = remote_url.replace('git@github.com:', '/') + + remote_url = remote_url.rstrip('/').replace('.git', '') + info['url'] = remote_url + + # Compute GitHub Pages URL + if 'github.com' in remote_url: + parts = remote_url.split('/') + if len(parts) >= 2: + owner = parts[-2] + repo = parts[-1] + info['github_pages_url'] = f"https://{owner}.github.io/{repo}/" + + except Exception: + pass + + return info + diff --git a/codewiki/cli/main.py b/codewiki/cli/main.py new file mode 100644 index 00000000..44b7f751 --- /dev/null +++ b/codewiki/cli/main.py @@ -0,0 +1,56 @@ +""" +Main CLI application for CodeWiki using Click framework. +""" + +import sys +import click +from pathlib import Path + +from codewiki import __version__ + + +@click.group() +@click.version_option(version=__version__, prog_name="CodeWiki CLI") +@click.pass_context +def cli(ctx): + """ + CodeWiki: Transform codebases into comprehensive documentation. + + Generate AI-powered documentation for your code repositories with support + for Python, Java, JavaScript, TypeScript, C, C++, and C#. + """ + # Ensure context object exists + ctx.ensure_object(dict) + + +@cli.command() +def version(): + """Display version information.""" + click.echo(f"CodeWiki CLI v{__version__}") + click.echo("Python-based documentation generator using AI analysis") + + +# Import commands +from codewiki.cli.commands.config import config_group +from codewiki.cli.commands.generate import generate_command + +# Register command groups +cli.add_command(config_group) +cli.add_command(generate_command, name="generate") + + +def main(): + """Entry point for the CLI.""" + try: + cli(obj={}) + except KeyboardInterrupt: + click.echo("\n\nInterrupted by user", err=True) + sys.exit(130) + except Exception as e: + click.secho(f"\n✗ Unexpected error: {e}", fg="red", err=True) + sys.exit(1) + + +if __name__ == "__main__": + main() + diff --git a/codewiki/cli/models/__init__.py b/codewiki/cli/models/__init__.py new file mode 100644 index 00000000..07db84ac --- /dev/null +++ b/codewiki/cli/models/__init__.py @@ -0,0 +1,4 @@ +"""Data models for CLI.""" + +__all__ = [] + diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py new file mode 100644 index 00000000..94ca211b --- /dev/null +++ b/codewiki/cli/models/config.py @@ -0,0 +1,102 @@ +""" +Configuration data models for CodeWiki CLI. + +This module contains the Configuration class which represents persistent +user settings stored in ~/.codewiki/config.json. These settings are converted +to the backend Config class when running documentation generation. +""" + +from dataclasses import dataclass, asdict +from typing import Optional +from pathlib import Path + +from codewiki.cli.utils.validation import ( + validate_url, + validate_api_key, + validate_model_name, +) + + +@dataclass +class Configuration: + """ + CodeWiki configuration data model. + + Attributes: + base_url: LLM API base URL + main_model: Primary model for documentation generation + cluster_model: Model for module clustering + default_output: Default output directory + """ + base_url: str + main_model: str + cluster_model: str + default_output: str = "docs" + + def validate(self): + """ + Validate all configuration fields. + + Raises: + ConfigurationError: If validation fails + """ + validate_url(self.base_url) + validate_model_name(self.main_model) + validate_model_name(self.cluster_model) + + def to_dict(self) -> dict: + """Convert to dictionary.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict) -> 'Configuration': + """ + Create Configuration from dictionary. + + Args: + data: Configuration dictionary + + Returns: + Configuration instance + """ + return cls( + base_url=data.get('base_url', ''), + main_model=data.get('main_model', ''), + cluster_model=data.get('cluster_model', ''), + default_output=data.get('default_output', 'docs'), + ) + + def is_complete(self) -> bool: + """Check if all required fields are set.""" + return bool( + self.base_url and + self.main_model and + self.cluster_model + ) + + def to_backend_config(self, repo_path: str, output_dir: str, api_key: str): + """ + Convert CLI Configuration to Backend Config. + + This method bridges the gap between persistent user settings (CLI Configuration) + and runtime job configuration (Backend Config). + + Args: + repo_path: Path to the repository to document + output_dir: Output directory for generated documentation + api_key: LLM API key (from keyring) + + Returns: + Backend Config instance ready for documentation generation + """ + from codewiki.src.config import Config + + return Config.from_cli( + repo_path=repo_path, + output_dir=output_dir, + llm_base_url=self.base_url, + llm_api_key=api_key, + main_model=self.main_model, + cluster_model=self.cluster_model + ) + diff --git a/codewiki/cli/models/job.py b/codewiki/cli/models/job.py new file mode 100644 index 00000000..c0c49d12 --- /dev/null +++ b/codewiki/cli/models/job.py @@ -0,0 +1,156 @@ +""" +Documentation job data models. +""" + +from dataclasses import dataclass, field, asdict +from datetime import datetime +from typing import List, Optional, Dict, Any +from enum import Enum +import uuid +import json + + +class JobStatus(str, Enum): + """Documentation job status.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class GenerationOptions: + """Options for documentation generation.""" + create_branch: bool = False + github_pages: bool = False + no_cache: bool = False + custom_output: Optional[str] = None + + +@dataclass +class JobStatistics: + """Statistics for a documentation job.""" + total_files_analyzed: int = 0 + leaf_nodes: int = 0 + max_depth: int = 0 + total_tokens_used: int = 0 + + +@dataclass +class LLMConfig: + """LLM configuration for a job.""" + main_model: str + cluster_model: str + base_url: str + + +@dataclass +class DocumentationJob: + """ + Represents a documentation generation job. + + Attributes: + job_id: Unique job identifier + repository_path: Absolute path to repository + repository_name: Repository name + output_directory: Output directory path + commit_hash: Git commit SHA + branch_name: Git branch name (if applicable) + timestamp_start: Job start time + timestamp_end: Job end time (if completed) + status: Current job status + error_message: Error message (if failed) + files_generated: List of generated files + module_count: Number of modules documented + generation_options: Generation options used + llm_config: LLM configuration used + statistics: Job statistics + """ + job_id: str = field(default_factory=lambda: str(uuid.uuid4())) + repository_path: str = "" + repository_name: str = "" + output_directory: str = "" + commit_hash: str = "" + branch_name: Optional[str] = None + timestamp_start: str = field(default_factory=lambda: datetime.now().isoformat()) + timestamp_end: Optional[str] = None + status: JobStatus = JobStatus.PENDING + error_message: Optional[str] = None + files_generated: List[str] = field(default_factory=list) + module_count: int = 0 + generation_options: GenerationOptions = field(default_factory=GenerationOptions) + llm_config: Optional[LLMConfig] = None + statistics: JobStatistics = field(default_factory=JobStatistics) + + def start(self): + """Mark job as started.""" + self.status = JobStatus.RUNNING + self.timestamp_start = datetime.now().isoformat() + + def complete(self): + """Mark job as completed.""" + self.status = JobStatus.COMPLETED + self.timestamp_end = datetime.now().isoformat() + + def fail(self, error_message: str): + """Mark job as failed.""" + self.status = JobStatus.FAILED + self.error_message = error_message + self.timestamp_end = datetime.now().isoformat() + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + data = { + "job_id": self.job_id, + "repository_path": self.repository_path, + "repository_name": self.repository_name, + "output_directory": self.output_directory, + "commit_hash": self.commit_hash, + "branch_name": self.branch_name, + "timestamp_start": self.timestamp_start, + "timestamp_end": self.timestamp_end, + "status": self.status.value if isinstance(self.status, JobStatus) else self.status, + "error_message": self.error_message, + "files_generated": self.files_generated, + "module_count": self.module_count, + "generation_options": asdict(self.generation_options), + "llm_config": asdict(self.llm_config) if self.llm_config else None, + "statistics": asdict(self.statistics), + } + return data + + def to_json(self) -> str: + """Convert to JSON string.""" + return json.dumps(self.to_dict(), indent=2) + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'DocumentationJob': + """Create from dictionary.""" + job = cls( + job_id=data.get('job_id', str(uuid.uuid4())), + repository_path=data.get('repository_path', ''), + repository_name=data.get('repository_name', ''), + output_directory=data.get('output_directory', ''), + commit_hash=data.get('commit_hash', ''), + branch_name=data.get('branch_name'), + timestamp_start=data.get('timestamp_start', datetime.now().isoformat()), + timestamp_end=data.get('timestamp_end'), + status=JobStatus(data.get('status', 'pending')), + error_message=data.get('error_message'), + files_generated=data.get('files_generated', []), + module_count=data.get('module_count', 0), + ) + + # Parse nested objects + if 'generation_options' in data: + opts = data['generation_options'] + job.generation_options = GenerationOptions(**opts) + + if 'llm_config' in data and data['llm_config']: + job.llm_config = LLMConfig(**data['llm_config']) + + if 'statistics' in data: + job.statistics = JobStatistics(**data['statistics']) + + return job + diff --git a/codewiki/cli/utils/__init__.py b/codewiki/cli/utils/__init__.py new file mode 100644 index 00000000..a7a69c23 --- /dev/null +++ b/codewiki/cli/utils/__init__.py @@ -0,0 +1,4 @@ +"""Utility functions and helpers for CLI.""" + +__all__ = [] + diff --git a/codewiki/cli/utils/api_errors.py b/codewiki/cli/utils/api_errors.py new file mode 100644 index 00000000..286db4cf --- /dev/null +++ b/codewiki/cli/utils/api_errors.py @@ -0,0 +1,140 @@ +""" +LLM API error handling utilities with fail-fast behavior. +""" + +from typing import Optional +import click + +from codewiki.cli.utils.errors import APIError + + +class APIErrorHandler: + """Handler for LLM API errors with fail-fast behavior.""" + + @staticmethod + def handle_api_error( + error: Exception, + context: Optional[str] = None, + fail_fast: bool = True + ) -> APIError: + """ + Handle LLM API error and convert to APIError. + + Args: + error: The original exception + context: Additional context (e.g., module name) + fail_fast: Whether to fail immediately (default: True) + + Returns: + APIError instance + """ + error_message = str(error) + + # Detect specific error types + if "429" in error_message or "rate limit" in error_message.lower(): + message = ( + "LLM API rate limit exceeded.\n\n" + "The API returned a 429 error, indicating too many requests.\n\n" + "Troubleshooting:\n" + " 1. Wait a few minutes before retrying\n" + " 2. Check your API quota at your provider's dashboard\n" + " 3. Consider upgrading your API plan\n" + " 4. For large repositories, generate during off-peak hours" + ) + elif "401" in error_message or "authentication" in error_message.lower(): + message = ( + "LLM API authentication failed.\n\n" + "Your API key appears to be invalid or expired.\n\n" + "Troubleshooting:\n" + " 1. Verify your API key: codewiki config show\n" + " 2. Update your API key: codewiki config set --api-key \n" + " 3. Check that your API key is active in your provider's dashboard" + ) + elif "timeout" in error_message.lower(): + message = ( + "LLM API request timed out.\n\n" + "The API did not respond within the expected time.\n\n" + "Troubleshooting:\n" + " 1. Check your internet connection\n" + " 2. Verify the API service is operational\n" + " 3. Try again in a few moments\n" + " 4. If the issue persists, contact your API provider" + ) + elif "network" in error_message.lower() or "connection" in error_message.lower(): + message = ( + "Network error while connecting to LLM API.\n\n" + "Could not establish connection to the API.\n\n" + "Troubleshooting:\n" + " 1. Check your internet connection\n" + " 2. Verify the base URL: codewiki config show\n" + " 3. Check if you're behind a proxy or firewall\n" + " 4. Try: curl -I to test connectivity" + ) + else: + message = ( + f"LLM API error: {error_message}\n\n" + "An unexpected error occurred while communicating with the LLM API.\n\n" + "Troubleshooting:\n" + " 1. Check your configuration: codewiki config validate\n" + " 2. Verify API service status\n" + " 3. Review the error message above for specific details" + ) + + if context: + message = f"Context: {context}\n\n{message}" + + return APIError(message) + + @staticmethod + def display_api_error(error: APIError, module_name: Optional[str] = None): + """ + Display API error with formatting. + + Args: + error: The API error + module_name: Optional module name for context + """ + click.echo() + click.secho("✗ LLM API Error", fg="red", bold=True) + click.echo() + + if module_name: + click.echo(f"Module: {module_name}") + click.echo() + + click.echo(error.message) + click.echo() + click.secho( + "Documentation generation stopped. No partial results saved.", + fg="yellow" + ) + click.echo() + + +def wrap_api_call(func, *args, fail_fast: bool = True, context: Optional[str] = None, **kwargs): + """ + Wrap an API call with error handling. + + Args: + func: Function to call + *args: Positional arguments + fail_fast: Whether to raise on error (default: True) + context: Optional context for error message + **kwargs: Keyword arguments + + Returns: + Function result + + Raises: + APIError: If API call fails and fail_fast is True + """ + try: + return func(*args, **kwargs) + except Exception as e: + api_error = APIErrorHandler.handle_api_error(e, context=context, fail_fast=fail_fast) + if fail_fast: + raise api_error + else: + APIErrorHandler.display_api_error(api_error) + return None + diff --git a/codewiki/cli/utils/errors.py b/codewiki/cli/utils/errors.py new file mode 100644 index 00000000..667c4a30 --- /dev/null +++ b/codewiki/cli/utils/errors.py @@ -0,0 +1,113 @@ +""" +Error handling utilities and exit codes for CLI. + +Exit Codes: + 0: Success + 1: General error + 2: Configuration error (missing/invalid credentials) + 3: Repository error (not a git repo, no code files) + 4: LLM API error (including rate limits) + 5: File system error (permissions, disk space) +""" + +import sys +import click +from typing import Optional + + +# Exit codes +EXIT_SUCCESS = 0 +EXIT_GENERAL_ERROR = 1 +EXIT_CONFIG_ERROR = 2 +EXIT_REPOSITORY_ERROR = 3 +EXIT_API_ERROR = 4 +EXIT_FILESYSTEM_ERROR = 5 + + +class CodeWikiError(Exception): + """Base exception for CodeWiki CLI errors.""" + + def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): + self.message = message + self.exit_code = exit_code + super().__init__(self.message) + + +class ConfigurationError(CodeWikiError): + """Configuration-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_CONFIG_ERROR) + + +class RepositoryError(CodeWikiError): + """Repository-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_REPOSITORY_ERROR) + + +class APIError(CodeWikiError): + """LLM API-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_API_ERROR) + + +class FileSystemError(CodeWikiError): + """File system-related errors.""" + + def __init__(self, message: str): + super().__init__(message, EXIT_FILESYSTEM_ERROR) + + +def handle_error(error: Exception, verbose: bool = False) -> int: + """ + Handle errors and return appropriate exit code. + + Args: + error: The exception to handle + verbose: Whether to show detailed error information + + Returns: + Exit code for the error + """ + if isinstance(error, CodeWikiError): + click.secho(f"\n✗ Error: {error.message}", fg="red", err=True) + return error.exit_code + else: + click.secho(f"\n✗ Unexpected error: {error}", fg="red", err=True) + if verbose: + import traceback + click.echo(traceback.format_exc(), err=True) + return EXIT_GENERAL_ERROR + + +def error_with_suggestion(message: str, suggestion: str, exit_code: int = EXIT_GENERAL_ERROR): + """ + Display error message with actionable suggestion and exit. + + Args: + message: The error message + suggestion: Suggested action to resolve the error + exit_code: Exit code to use + """ + click.secho(f"\n✗ Error: {message}", fg="red", err=True) + click.echo(f"\n{suggestion}", err=True) + sys.exit(exit_code) + + +def warning(message: str): + """Display a warning message.""" + click.secho(f"⚠️ {message}", fg="yellow") + + +def success(message: str): + """Display a success message.""" + click.secho(f"✓ {message}", fg="green") + + +def info(message: str): + """Display an info message.""" + click.echo(message) + diff --git a/codewiki/cli/utils/fs.py b/codewiki/cli/utils/fs.py new file mode 100644 index 00000000..88f4cb31 --- /dev/null +++ b/codewiki/cli/utils/fs.py @@ -0,0 +1,190 @@ +""" +File system utilities for CLI operations. +""" + +import os +import shutil +from pathlib import Path +from typing import Optional, List + +from codewiki.cli.utils.errors import FileSystemError + + +def ensure_directory(path: Path, mode: int = 0o700) -> Path: + """ + Ensure directory exists, create if necessary. + + Args: + path: Directory path + mode: Directory permissions (default: 0o700 - user only) + + Returns: + Path to the directory + + Raises: + FileSystemError: If directory cannot be created + """ + try: + path = Path(path).expanduser().resolve() + path.mkdir(parents=True, exist_ok=True, mode=mode) + return path + except PermissionError: + raise FileSystemError( + f"Permission denied: Cannot create directory {path}\n" + f"Try: chmod u+w {path.parent}" + ) + except OSError as e: + raise FileSystemError(f"Cannot create directory {path}: {e}") + + +def check_writable(path: Path) -> bool: + """ + Check if a path is writable. + + Args: + path: Path to check + + Returns: + True if writable, False otherwise + """ + path = Path(path).expanduser().resolve() + + if path.exists(): + return os.access(path, os.W_OK) + else: + # Check parent directory if path doesn't exist + parent = path.parent + return parent.exists() and os.access(parent, os.W_OK) + + +def safe_write(path: Path, content: str, encoding: str = "utf-8"): + """ + Safely write content to a file using atomic write (temp file + rename). + + Args: + path: File path + content: Content to write + encoding: File encoding + + Raises: + FileSystemError: If write fails + """ + path = Path(path).expanduser().resolve() + temp_path = path.with_suffix(path.suffix + ".tmp") + + try: + # Write to temp file + with open(temp_path, "w", encoding=encoding) as f: + f.write(content) + + # Atomic rename + temp_path.replace(path) + except Exception as e: + # Clean up temp file if it exists + if temp_path.exists(): + temp_path.unlink() + raise FileSystemError(f"Cannot write to {path}: {e}") + + +def safe_read(path: Path, encoding: str = "utf-8") -> str: + """ + Safely read content from a file. + + Args: + path: File path + encoding: File encoding + + Returns: + File content + + Raises: + FileSystemError: If read fails + """ + path = Path(path).expanduser().resolve() + + try: + with open(path, "r", encoding=encoding) as f: + return f.read() + except FileNotFoundError: + raise FileSystemError(f"File not found: {path}") + except PermissionError: + raise FileSystemError(f"Permission denied: Cannot read {path}") + except Exception as e: + raise FileSystemError(f"Cannot read {path}: {e}") + + +def get_file_size(path: Path) -> int: + """ + Get file size in bytes. + + Args: + path: File path + + Returns: + File size in bytes + """ + return Path(path).stat().st_size + + +def find_files( + directory: Path, + extensions: Optional[List[str]] = None, + recursive: bool = True +) -> List[Path]: + """ + Find files in directory matching extensions. + + Args: + directory: Directory to search + extensions: List of file extensions (e.g., ['.py', '.java']) + recursive: Search recursively + + Returns: + List of matching file paths + """ + directory = Path(directory).expanduser().resolve() + + if not directory.exists(): + return [] + + pattern = "**/*" if recursive else "*" + files = [] + + for path in directory.glob(pattern): + if not path.is_file(): + continue + + if extensions is None or path.suffix in extensions: + files.append(path) + + return files + + +def cleanup_directory(path: Path, keep_hidden: bool = True): + """ + Clean up a directory by removing its contents. + + Args: + path: Directory to clean + keep_hidden: Keep hidden files/directories (starting with .) + + Raises: + FileSystemError: If cleanup fails + """ + path = Path(path).expanduser().resolve() + + if not path.exists(): + return + + try: + for item in path.iterdir(): + if keep_hidden and item.name.startswith('.'): + continue + + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + except Exception as e: + raise FileSystemError(f"Cannot clean directory {path}: {e}") + diff --git a/codewiki/cli/utils/instructions.py b/codewiki/cli/utils/instructions.py new file mode 100644 index 00000000..7c2bf058 --- /dev/null +++ b/codewiki/cli/utils/instructions.py @@ -0,0 +1,179 @@ +""" +Post-generation instructions generator. +""" + +from pathlib import Path +from typing import Optional +import click + + +def compute_github_pages_url(repo_url: str, repo_name: str) -> str: + """ + Compute expected GitHub Pages URL from repository URL. + + Args: + repo_url: GitHub repository URL + repo_name: Repository name + + Returns: + Expected GitHub Pages URL + """ + # Extract owner from GitHub URL + # e.g., "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/owner/repo" -> "owner" + if "github.com" in repo_url: + parts = repo_url.rstrip('/').split('/') + if len(parts) >= 2: + owner = parts[-2] + repo = parts[-1].replace('.git', '') + return f"https://{owner}.github.io/{repo}/" + + return f"https://YOUR_USERNAME.github.io/{repo_name}/" + + +def get_pr_creation_url(repo_url: str, branch_name: str) -> str: + """ + Get PR creation URL for GitHub. + + Args: + repo_url: GitHub repository URL + branch_name: Branch name + + Returns: + PR creation URL + """ + base_url = repo_url.rstrip('/').replace('.git', '') + return f"{base_url}/compare/{branch_name}" + + +def display_post_generation_instructions( + output_dir: Path, + repo_name: str, + repo_url: Optional[str] = None, + branch_name: Optional[str] = None, + github_pages: bool = False, + files_generated: list = None, + statistics: dict = None +): + """ + Display post-generation instructions. + + Args: + output_dir: Output directory path + repo_name: Repository name + repo_url: GitHub repository URL (optional) + branch_name: Git branch name (optional) + github_pages: Whether GitHub Pages HTML was generated + files_generated: List of generated files + statistics: Generation statistics + """ + click.echo() + click.secho("✓ Documentation generated successfully!", fg="green", bold=True) + click.echo() + + # Output directory + click.secho("Output directory:", fg="cyan", bold=True) + click.echo(f" {output_dir}") + click.echo() + + # Generated files + if files_generated: + click.secho("Generated files:", fg="cyan", bold=True) + for file in files_generated[:10]: # Show first 10 + click.echo(f" - {file}") + if len(files_generated) > 10: + click.echo(f" ... and {len(files_generated) - 10} more") + click.echo() + + # Statistics + if statistics: + click.secho("Statistics:", fg="cyan", bold=True) + if 'module_count' in statistics: + click.echo(f" Total modules: {statistics['module_count']}") + if 'total_files_analyzed' in statistics: + click.echo(f" Files analyzed: {statistics['total_files_analyzed']}") + if 'generation_time' in statistics: + minutes = int(statistics['generation_time'] // 60) + seconds = int(statistics['generation_time'] % 60) + click.echo(f" Generation time: {minutes} minutes {seconds} seconds") + # if 'total_tokens_used' in statistics: + # tokens = statistics['total_tokens_used'] + # click.echo(f" Tokens used: ~{tokens:,}") + click.echo() + + # Next steps + click.secho("Next steps:", fg="cyan", bold=True) + click.echo() + + click.echo("1. Review the generated documentation:") + click.echo(f" cat {output_dir}/overview.md") + if github_pages: + click.echo(f" open {output_dir}/index.html # View in browser") + click.echo() + + if branch_name: + # Git workflow with branch + click.echo("2. Push the documentation branch:") + click.secho(f" git push origin {branch_name}", fg="yellow") + click.echo() + + if repo_url: + pr_url = get_pr_creation_url(repo_url, branch_name) + click.echo("3. Create a Pull Request to merge documentation:") + click.secho(f" {pr_url}", fg="blue") + click.echo() + + click.echo("4. After merge, enable GitHub Pages:") + else: + click.echo("3. Enable GitHub Pages:") + else: + # Direct commit workflow + click.echo("2. Commit the documentation:") + click.secho(" git add docs/", fg="yellow") + click.secho(' git commit -m "Add generated documentation"', fg="yellow") + click.echo() + + click.echo("3. Push to GitHub:") + click.secho(" git push origin main", fg="yellow") + click.echo() + + click.echo("4. Enable GitHub Pages:") + + click.echo(" - Go to repository Settings → Pages") + click.echo(" - Source: Deploy from a branch") + click.echo(" - Branch: main, folder: /docs") + click.echo() + + if repo_url: + github_pages_url = compute_github_pages_url(repo_url, repo_name) + click.echo("5. Your documentation will be available at:") + click.secho(f" {github_pages_url}", fg="blue", bold=True) + click.echo() + + +def display_generation_summary( + success: bool, + error_message: Optional[str] = None, + output_dir: Optional[Path] = None +): + """ + Display generation summary (success or failure). + + Args: + success: Whether generation was successful + error_message: Error message if failed + output_dir: Output directory if successful + """ + if success: + click.echo() + click.secho("✓ Generation completed successfully!", fg="green", bold=True) + if output_dir: + click.echo(f"\nDocumentation saved to: {output_dir}") + click.echo() + else: + click.echo() + click.secho("✗ Generation failed", fg="red", bold=True) + if error_message: + click.echo() + click.echo(error_message) + click.echo() + diff --git a/codewiki/cli/utils/logging.py b/codewiki/cli/utils/logging.py new file mode 100644 index 00000000..80863404 --- /dev/null +++ b/codewiki/cli/utils/logging.py @@ -0,0 +1,85 @@ +""" +Logging utilities for CLI with colored output and progress tracking. +""" + +import sys +from datetime import datetime +from typing import Optional +import click + + +class CLILogger: + """Logger for CLI with support for verbose and normal modes.""" + + def __init__(self, verbose: bool = False): + """ + Initialize the logger. + + Args: + verbose: Enable verbose output + """ + self.verbose = verbose + self.start_time = datetime.now() + + def debug(self, message: str): + """Log debug message (only in verbose mode).""" + if self.verbose: + timestamp = datetime.now().strftime("%H:%M:%S") + click.secho(f"[{timestamp}] {message}", fg="cyan", dim=True) + + def info(self, message: str): + """Log info message.""" + click.echo(message) + + def success(self, message: str): + """Log success message in green.""" + click.secho(f"✓ {message}", fg="green") + + def warning(self, message: str): + """Log warning message in yellow.""" + click.secho(f"⚠️ {message}", fg="yellow") + + def error(self, message: str): + """Log error message in red.""" + click.secho(f"✗ {message}", fg="red", err=True) + + def step(self, message: str, step: Optional[int] = None, total: Optional[int] = None): + """ + Log a processing step. + + Args: + message: Step description + step: Current step number + total: Total number of steps + """ + if step is not None and total is not None: + prefix = f"[{step}/{total}]" + else: + prefix = "→" + + click.secho(f"{prefix} {message}", fg="blue", bold=True) + + def elapsed_time(self) -> str: + """Get elapsed time since logger was created.""" + elapsed = datetime.now() - self.start_time + minutes = int(elapsed.total_seconds() // 60) + seconds = int(elapsed.total_seconds() % 60) + + if minutes > 0: + return f"{minutes}m {seconds}s" + else: + return f"{seconds}s" + + +def create_logger(verbose: bool = False) -> CLILogger: + """ + Create and return a CLI logger. + + Args: + verbose: Enable verbose output + + Returns: + Configured CLILogger instance + """ + return CLILogger(verbose=verbose) + diff --git a/codewiki/cli/utils/progress.py b/codewiki/cli/utils/progress.py new file mode 100644 index 00000000..f61efc18 --- /dev/null +++ b/codewiki/cli/utils/progress.py @@ -0,0 +1,222 @@ +""" +Progress indicator utilities for CLI. +""" + +import time +from typing import Optional, Callable +from datetime import datetime +import click + + +class ProgressTracker: + """ + Progress tracker with stages and ETA estimation. + + Stages: + 1. Dependency Analysis (40% of time) + 2. Module Clustering (20% of time) + 3. Documentation Generation (30% of time) + 4. HTML Generation (5% of time, optional) + 5. Finalization (5% of time) + """ + + # Stage weights (percentage of total time) + STAGE_WEIGHTS = { + 1: 0.40, # Dependency Analysis + 2: 0.20, # Module Clustering + 3: 0.30, # Documentation Generation + 4: 0.05, # HTML Generation (optional) + 5: 0.05, # Finalization + } + + STAGE_NAMES = { + 1: "Dependency Analysis", + 2: "Module Clustering", + 3: "Documentation Generation", + 4: "HTML Generation", + 5: "Finalization", + } + + def __init__(self, total_stages: int = 5, verbose: bool = False): + """ + Initialize progress tracker. + + Args: + total_stages: Number of stages + verbose: Enable verbose output + """ + self.total_stages = total_stages + self.current_stage = 0 + self.stage_progress = 0.0 + self.start_time = time.time() + self.verbose = verbose + self.current_stage_start = self.start_time + + def start_stage(self, stage: int, description: Optional[str] = None): + """ + Start a new stage. + + Args: + stage: Stage number (1-5) + description: Optional custom description + """ + self.current_stage = stage + self.stage_progress = 0.0 + self.current_stage_start = time.time() + + stage_name = description or self.STAGE_NAMES.get(stage, f"Stage {stage}") + + if self.verbose: + elapsed = self._format_elapsed() + click.secho( + f"\n[{elapsed}] Phase {stage}/{self.total_stages}: {stage_name}", + fg="blue", + bold=True + ) + else: + click.secho( + f"[{stage}/{self.total_stages}] {stage_name}", + fg="blue", + bold=True + ) + + def update_stage(self, progress: float, message: Optional[str] = None): + """ + Update progress within current stage. + + Args: + progress: Progress percentage (0.0 to 1.0) + message: Optional progress message + """ + self.stage_progress = min(1.0, max(0.0, progress)) + + if self.verbose and message: + elapsed = self._format_elapsed() + click.echo(f"[{elapsed}] {message}") + + def complete_stage(self, message: Optional[str] = None): + """ + Complete current stage. + + Args: + message: Optional completion message + """ + self.stage_progress = 1.0 + + if self.verbose: + elapsed = self._format_elapsed() + stage_time = time.time() - self.current_stage_start + stage_name = self.STAGE_NAMES.get(self.current_stage, f"Stage {self.current_stage}") + click.secho( + f"[{elapsed}] {stage_name} complete ({stage_time:.1f}s)", + fg="green" + ) + if message: + click.echo(f"[{elapsed}] {message}") + + def get_overall_progress(self) -> float: + """ + Get overall progress percentage. + + Returns: + Progress (0.0 to 1.0) + """ + completed_weight = sum( + self.STAGE_WEIGHTS.get(s, 0) + for s in range(1, self.current_stage) + ) + + current_weight = self.STAGE_WEIGHTS.get(self.current_stage, 0) * self.stage_progress + + return completed_weight + current_weight + + def _format_elapsed(self) -> str: + """Format elapsed time.""" + elapsed = time.time() - self.start_time + minutes = int(elapsed // 60) + seconds = int(elapsed % 60) + + if minutes > 0: + return f"{minutes:02d}:{seconds:02d}" + else: + return f"00:{seconds:02d}" + + def get_eta(self) -> Optional[str]: + """ + Estimate time remaining. + + Returns: + ETA string or None if cannot estimate + """ + elapsed = time.time() - self.start_time + progress = self.get_overall_progress() + + if progress <= 0.0: + return None + + total_estimated = elapsed / progress + remaining = total_estimated - elapsed + + if remaining < 0: + return "< 1 min" + + minutes = int(remaining // 60) + seconds = int(remaining % 60) + + if minutes > 60: + hours = minutes // 60 + minutes = minutes % 60 + return f"{hours}h {minutes}m" + elif minutes > 0: + return f"{minutes}m {seconds}s" + else: + return f"{seconds}s" + + +class ModuleProgressBar: + """Progress bar for module-by-module generation.""" + + def __init__(self, total_modules: int, verbose: bool = False): + """ + Initialize module progress bar. + + Args: + total_modules: Total number of modules to process + verbose: Enable verbose output + """ + self.total_modules = total_modules + self.current_module = 0 + self.verbose = verbose + self.bar = None + + if not verbose: + self.bar = click.progressbar( + length=total_modules, + label="Generating modules", + show_eta=True, + show_percent=True, + ) + self.bar.__enter__() + + def update(self, module_name: str, cached: bool = False): + """ + Update progress for a module. + + Args: + module_name: Name of the module + cached: Whether the module was loaded from cache + """ + self.current_module += 1 + + if self.verbose: + status = "✓ (cached)" if cached else "⟳ (generating)" + click.echo(f" [{self.current_module}/{self.total_modules}] {module_name}... {status}") + elif self.bar: + self.bar.update(1) + + def finish(self): + """Finish progress bar.""" + if self.bar: + self.bar.__exit__(None, None, None) + self.bar = None + diff --git a/codewiki/cli/utils/repo_validator.py b/codewiki/cli/utils/repo_validator.py new file mode 100644 index 00000000..d0b22ef1 --- /dev/null +++ b/codewiki/cli/utils/repo_validator.py @@ -0,0 +1,184 @@ +""" +Repository validation utilities for documentation generation. +""" + +from pathlib import Path +from typing import Tuple, List +import os + +from codewiki.cli.utils.errors import RepositoryError +from codewiki.cli.utils.validation import validate_repository_path, detect_supported_languages + + +# Supported file extensions by language +SUPPORTED_EXTENSIONS = { + '.py', # Python + '.java', # Java + '.js', # JavaScript + '.jsx', # JavaScript (React) + '.ts', # TypeScript + '.tsx', # TypeScript (React) + '.c', # C + '.h', # C headers + '.cpp', # C++ + '.hpp', # C++ headers + '.cc', # C++ + '.hh', # C++ headers + '.cxx', # C++ + '.hxx', # C++ headers + '.cs', # C# +} + + +def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: + """ + Validate repository for documentation generation. + + Checks: + - Path exists and is a directory + - Contains supported code files + - Has sufficient files for meaningful documentation + + Args: + repo_path: Path to repository + + Returns: + Tuple of (validated_path, language_counts) + + Raises: + RepositoryError: If validation fails + """ + # Validate path exists + repo_path = validate_repository_path(repo_path) + + # Detect languages + languages = detect_supported_languages(repo_path) + + if not languages: + raise RepositoryError( + f"No supported code files found in {repo_path}\n\n" + "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#\n\n" + "Please navigate to a code repository or specify a custom directory:\n" + " cd /path/to/your/project\n" + " codewiki generate" + ) + + return repo_path, languages + + +def check_writable_output(output_dir: Path) -> Path: + """ + Check if output directory is writable. + + Args: + output_dir: Output directory path + + Returns: + Validated output directory path + + Raises: + RepositoryError: If output directory is not writable + """ + output_dir = Path(output_dir).expanduser().resolve() + + # Check if output directory exists + if output_dir.exists(): + if not output_dir.is_dir(): + raise RepositoryError( + f"Output path exists but is not a directory: {output_dir}" + ) + + # Check if writable + if not os.access(output_dir, os.W_OK): + raise RepositoryError( + f"Output directory is not writable: {output_dir}\n\n" + f"Try: chmod u+w {output_dir}" + ) + else: + # Check if parent is writable + parent = output_dir.parent + if not parent.exists(): + raise RepositoryError( + f"Parent directory does not exist: {parent}" + ) + + if not os.access(parent, os.W_OK): + raise RepositoryError( + f"Cannot create output directory (parent not writable): {parent}\n\n" + f"Try: chmod u+w {parent}" + ) + + return output_dir + + +def is_git_repository(repo_path: Path) -> bool: + """ + Check if path is a git repository. + + Args: + repo_path: Path to check + + Returns: + True if git repository, False otherwise + """ + git_dir = repo_path / ".git" + return git_dir.exists() and git_dir.is_dir() + + +def get_git_commit_hash(repo_path: Path) -> str: + """ + Get current git commit hash. + + Args: + repo_path: Repository path + + Returns: + Commit hash or empty string if not a git repo + """ + if not is_git_repository(repo_path): + return "" + + try: + import git + repo = git.Repo(repo_path) + return repo.head.commit.hexsha + except Exception: + return "" + + +def get_git_branch(repo_path: Path) -> str: + """ + Get current git branch name. + + Args: + repo_path: Repository path + + Returns: + Branch name or empty string if not a git repo + """ + if not is_git_repository(repo_path): + return "" + + try: + import git + repo = git.Repo(repo_path) + return repo.active_branch.name + except Exception: + return "" + + +def count_code_files(repo_path: Path) -> int: + """ + Count supported code files in repository. + + Args: + repo_path: Repository path + + Returns: + Number of code files + """ + count = 0 + for ext in SUPPORTED_EXTENSIONS: + count += len(list(repo_path.rglob(f"*{ext}"))) + return count + diff --git a/codewiki/cli/utils/validation.py b/codewiki/cli/utils/validation.py new file mode 100644 index 00000000..da76500d --- /dev/null +++ b/codewiki/cli/utils/validation.py @@ -0,0 +1,231 @@ +""" +Validation utilities for CLI inputs and configuration. +""" + +import re +from pathlib import Path +from typing import Optional, List, Tuple +from urllib.parse import urlparse + +from codewiki.cli.utils.errors import ConfigurationError, RepositoryError + + +def validate_url(url: str, require_https: bool = True, allow_localhost: bool = True) -> str: + """ + Validate URL format. + + Args: + url: URL to validate + require_https: Require HTTPS scheme (except localhost) + allow_localhost: Allow localhost URLs + + Returns: + Validated URL + + Raises: + ConfigurationError: If URL is invalid + """ + try: + parsed = urlparse(url) + + # Check scheme + if not parsed.scheme: + raise ConfigurationError(f"Invalid URL (missing scheme): {url}") + + # Check HTTPS requirement + if require_https and parsed.scheme != 'https': + # Allow HTTP for localhost + if allow_localhost and parsed.hostname in ['localhost', '127.0.0.1', '::1']: + pass + else: + raise ConfigurationError( + f"URL must use HTTPS: {url}\n" + f"HTTP is only allowed for localhost" + ) + + # Check hostname + if not parsed.hostname: + raise ConfigurationError(f"Invalid URL (missing hostname): {url}") + + return url + except ValueError as e: + raise ConfigurationError(f"Invalid URL format: {url}\nError: {e}") + + +def validate_api_key(api_key: str, min_length: int = 10) -> str: + """ + Validate API key format. + + Args: + api_key: API key to validate + min_length: Minimum key length + + Returns: + Validated API key + + Raises: + ConfigurationError: If API key is invalid + """ + if not api_key or not api_key.strip(): + raise ConfigurationError("API key cannot be empty") + + api_key = api_key.strip() + + if len(api_key) < min_length: + raise ConfigurationError( + f"API key too short (minimum {min_length} characters)" + ) + + return api_key + + +def validate_model_name(model: str) -> str: + """ + Validate model name format. + + Args: + model: Model name to validate + + Returns: + Validated model name + + Raises: + ConfigurationError: If model name is invalid + """ + if not model or not model.strip(): + raise ConfigurationError("Model name cannot be empty") + + return model.strip() + + +def validate_output_directory(path: str) -> Path: + """ + Validate output directory path. + + Args: + path: Directory path to validate + + Returns: + Validated Path object + + Raises: + ConfigurationError: If path is invalid + """ + if not path or not path.strip(): + raise ConfigurationError("Output directory cannot be empty") + + try: + resolved_path = Path(path).expanduser().resolve() + + # Check if path is writable (or parent is writable if path doesn't exist) + if resolved_path.exists(): + if not resolved_path.is_dir(): + raise ConfigurationError( + f"Output path exists but is not a directory: {path}" + ) + + return resolved_path + except Exception as e: + raise ConfigurationError(f"Invalid output directory path: {path}\nError: {e}") + + +def validate_repository_path(path: Path) -> Path: + """ + Validate repository path exists and contains code files. + + Args: + path: Repository path to validate + + Returns: + Validated Path object + + Raises: + RepositoryError: If repository is invalid + """ + path = Path(path).expanduser().resolve() + + if not path.exists(): + raise RepositoryError(f"Repository path does not exist: {path}") + + if not path.is_dir(): + raise RepositoryError(f"Repository path is not a directory: {path}") + + return path + + +def detect_supported_languages(directory: Path) -> List[Tuple[str, int]]: + """ + Detect supported programming languages in a directory. + + Args: + directory: Directory to scan + + Returns: + List of (language, file_count) tuples + """ + language_extensions = { + 'Python': ['.py'], + 'Java': ['.java'], + 'JavaScript': ['.js', '.jsx'], + 'TypeScript': ['.ts', '.tsx'], + 'C': ['.c', '.h'], + 'C++': ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], + 'C#': ['.cs'], + } + + language_counts = {} + + for language, extensions in language_extensions.items(): + count = 0 + for ext in extensions: + count += len(list(directory.rglob(f"*{ext}"))) + + if count > 0: + language_counts[language] = count + + # Sort by count descending + return sorted(language_counts.items(), key=lambda x: x[1], reverse=True) + + +def is_top_tier_model(model: str) -> bool: + """ + Check if a model is considered top-tier for clustering. + + Args: + model: Model name + + Returns: + True if top-tier, False otherwise + """ + top_tier_models = [ + 'claude-opus', + 'claude-sonnet-4', + 'gpt-4', + 'gpt-4-turbo', + 'gemini-1.5-pro', + ] + + model_lower = model.lower() + return any(tier in model_lower for tier in top_tier_models) + + +def mask_api_key(api_key: str, visible_chars: int = 4) -> str: + """ + Mask API key for display, showing only first and last few characters. + + Args: + api_key: API key to mask + visible_chars: Number of visible characters at start and end + + Returns: + Masked API key (e.g., "sk-1234...5678") + """ + if not api_key: + return "Not set" + + if len(api_key) <= visible_chars * 2: + # Key too short, mask everything except edges + return f"{api_key[:2]}...{api_key[-2:]}" + + return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" + diff --git a/codewiki/py.typed b/codewiki/py.typed new file mode 100644 index 00000000..484056b3 --- /dev/null +++ b/codewiki/py.typed @@ -0,0 +1,2 @@ +# PEP 561 marker file for type checking support + diff --git a/run_web_app.py b/codewiki/run_web_app.py similarity index 100% rename from run_web_app.py rename to codewiki/run_web_app.py diff --git a/codewiki/src/__init__.py b/codewiki/src/__init__.py new file mode 100644 index 00000000..9a6e5272 --- /dev/null +++ b/codewiki/src/__init__.py @@ -0,0 +1,2 @@ +"""CodeWiki backend and frontend modules.""" + diff --git a/codewiki/src/be/__init__.py b/codewiki/src/be/__init__.py new file mode 100644 index 00000000..f7696180 --- /dev/null +++ b/codewiki/src/be/__init__.py @@ -0,0 +1,2 @@ +"""CodeWiki backend modules for documentation generation.""" + diff --git a/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py similarity index 60% rename from src/be/agent_orchestrator.py rename to codewiki/src/be/agent_orchestrator.py index 7b7455e3..78ea3ac2 100644 --- a/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -1,5 +1,5 @@ from pydantic_ai import Agent -import logfire +# import logfire import logging import os from typing import Dict, List, Any @@ -8,50 +8,51 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -try: - # Configure logfire with environment variables for Docker compatibility - logfire_token = os.getenv('LOGFIRE_TOKEN') - logfire_project = os.getenv('LOGFIRE_PROJECT_NAME', 'default') - logfire_service = os.getenv('LOGFIRE_SERVICE_NAME', 'default') +# try: +# # Configure logfire with environment variables for Docker compatibility +# logfire_token = os.getenv('LOGFIRE_TOKEN') +# logfire_project = os.getenv('LOGFIRE_PROJECT_NAME', 'default') +# logfire_service = os.getenv('LOGFIRE_SERVICE_NAME', 'default') - if logfire_token: - # Configure with explicit token (for Docker) - logfire.configure( - token=logfire_token, - project_name=logfire_project, - service_name=logfire_service, - ) - else: - # Use default configuration (for local development with logfire auth) - logfire.configure( - project_name=logfire_project, - service_name=logfire_service, - ) +# if logfire_token: +# # Configure with explicit token (for Docker) +# logfire.configure( +# token=logfire_token, +# project_name=logfire_project, +# service_name=logfire_service, +# ) +# else: +# # Use default configuration (for local development with logfire auth) +# logfire.configure( +# project_name=logfire_project, +# service_name=logfire_service, +# ) - logfire.instrument_pydantic_ai() - logger.info(f"Logfire configured successfully for project: {logfire_project}") +# logfire.instrument_pydantic_ai() +# logger.debug(f"Logfire configured successfully for project: {logfire_project}") -except Exception as e: - logger.warning(f"Failed to configure logfire: {e}") +# except Exception as e: +# logger.warning(f"Failed to configure logfire: {e}") # Local imports -from .agent_tools.deps import CodeWikiDeps -from .agent_tools.read_code_components import read_code_components_tool -from .agent_tools.str_replace_editor import str_replace_editor_tool -from .agent_tools.generate_sub_module_documentations import generate_sub_module_documentation_tool -from .llm_services import fallback_models -from .prompt_template import ( +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool +from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool +from codewiki.src.be.agent_tools.generate_sub_module_documentations import generate_sub_module_documentation_tool +from codewiki.src.be.llm_services import create_fallback_models +from codewiki.src.be.prompt_template import ( SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt, ) -from .utils import is_complex_module -from config import ( +from codewiki.src.be.utils import is_complex_module +from codewiki.src.config import ( Config, MODULE_TREE_FILENAME, + OVERVIEW_FILENAME, ) -from utils import file_manager -from .dependency_analyzer.models.core import Node +from codewiki.src.utils import file_manager +from codewiki.src.be.dependency_analyzer.models.core import Node class AgentOrchestrator: @@ -59,13 +60,14 @@ class AgentOrchestrator: def __init__(self, config: Config): self.config = config + self.fallback_models = create_fallback_models(config) def create_agent(self, module_name: str, components: Dict[str, Any], core_component_ids: List[str]) -> Agent: """Create an appropriate agent based on module complexity.""" if is_complex_module(components, core_component_ids): return Agent( - fallback_models, + self.fallback_models, name=module_name, deps_type=CodeWikiDeps, tools=[ @@ -77,7 +79,7 @@ def create_agent(self, module_name: str, components: Dict[str, Any], ) else: return Agent( - fallback_models, + self.fallback_models, name=module_name, deps_type=CodeWikiDeps, tools=[read_code_components_tool, str_replace_editor_tool], @@ -87,7 +89,7 @@ def create_agent(self, module_name: str, components: Dict[str, Any], async def process_module(self, module_name: str, components: Dict[str, Node], core_component_ids: List[str], module_path: List[str], working_dir: str) -> Dict[str, Any]: """Process a single module and generate its documentation.""" - logger.info(f"Processing module: {module_name}") + logger.debug(f"Processing module: {module_name}") # Load or create module tree module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) @@ -106,9 +108,16 @@ async def process_module(self, module_name: str, components: Dict[str, Node], current_module_name=module_name, module_tree=module_tree, max_depth=self.config.max_depth, - current_depth=1 + current_depth=1, + config=self.config ) + # check if overview docs already exists + overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) + if os.path.exists(overview_docs_path): + logger.info(f"Overview docs already exists at {overview_docs_path}") + return module_tree + # check if module docs already exists docs_path = os.path.join(working_dir, f"{module_name}.md") if os.path.exists(docs_path): @@ -129,7 +138,7 @@ async def process_module(self, module_name: str, components: Dict[str, Node], # Save updated module tree file_manager.save_json(deps.module_tree, module_tree_path) - logger.info(f"Successfully processed module: {module_name}") + logger.debug(f"Successfully processed module: {module_name}") return deps.module_tree diff --git a/codewiki/src/be/agent_tools/__init__.py b/codewiki/src/be/agent_tools/__init__.py new file mode 100644 index 00000000..b828275e --- /dev/null +++ b/codewiki/src/be/agent_tools/__init__.py @@ -0,0 +1,2 @@ +"""Agent tools for backend processing.""" + diff --git a/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py similarity index 63% rename from src/be/agent_tools/deps.py rename to codewiki/src/be/agent_tools/deps.py index 471a7aa3..e5e6c4ff 100644 --- a/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from ..dependency_analyzer.models.core import Node +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.config import Config @dataclass class CodeWikiDeps: @@ -11,4 +12,5 @@ class CodeWikiDeps: current_module_name: str module_tree: dict[str, any] max_depth: int - current_depth: int \ No newline at end of file + current_depth: int + config: Config # LLM configuration \ No newline at end of file diff --git a/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py similarity index 82% rename from src/be/agent_tools/generate_sub_module_documentations.py rename to codewiki/src/be/agent_tools/generate_sub_module_documentations.py index 4bc275ae..ec1431e0 100644 --- a/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -1,13 +1,13 @@ from pydantic_ai import RunContext, Tool, Agent -from .deps import CodeWikiDeps -from .read_code_components import read_code_components_tool -from .str_replace_editor import str_replace_editor_tool -from ..llm_services import fallback_models -from ..prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt -from ..utils import is_complex_module, count_tokens -from ..cluster_modules import format_potential_core_components -from config import MAX_TOKEN_PER_LEAF_MODULE +from codewiki.src.be.agent_tools.deps import CodeWikiDeps +from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool +from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool +from codewiki.src.be.llm_services import create_fallback_models +from codewiki.src.be.prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt +from codewiki.src.be.utils import is_complex_module, count_tokens +from codewiki.src.be.cluster_modules import format_potential_core_components +from codewiki.src.config import MAX_TOKEN_PER_LEAF_MODULE @@ -23,6 +23,9 @@ async def generate_sub_module_documentation( deps = ctx.deps previous_module_name = deps.current_module_name + + # Create fallback models from config + fallback_models = create_fallback_models(deps.config) # add the sub-module to the module tree value = deps.module_tree diff --git a/src/be/agent_tools/read_code_components.py b/codewiki/src/be/agent_tools/read_code_components.py similarity index 94% rename from src/be/agent_tools/read_code_components.py rename to codewiki/src/be/agent_tools/read_code_components.py index 10005624..0125cbb2 100644 --- a/src/be/agent_tools/read_code_components.py +++ b/codewiki/src/be/agent_tools/read_code_components.py @@ -1,5 +1,5 @@ from pydantic_ai import RunContext, Tool -from .deps import CodeWikiDeps +from codewiki.src.be.agent_tools.deps import CodeWikiDeps async def read_code_components(ctx: RunContext[CodeWikiDeps], component_ids: list[str]) -> str: diff --git a/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py similarity index 100% rename from src/be/agent_tools/str_replace_editor.py rename to codewiki/src/be/agent_tools/str_replace_editor.py diff --git a/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py similarity index 83% rename from src/be/cluster_modules.py rename to codewiki/src/be/cluster_modules.py index a927219c..69eebf79 100644 --- a/src/be/cluster_modules.py +++ b/codewiki/src/be/cluster_modules.py @@ -3,11 +3,11 @@ import logging logger = logging.getLogger(__name__) -from .dependency_analyzer.models.core import Node -from .llm_services import call_llm -from .utils import count_tokens -from config import MAX_TOKEN_PER_MODULE, CLUSTER_MODEL -from .prompt_template import format_cluster_prompt +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.llm_services import call_llm +from codewiki.src.be.utils import count_tokens +from codewiki.src.config import MAX_TOKEN_PER_MODULE, Config +from codewiki.src.be.prompt_template import format_cluster_prompt def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]: @@ -43,6 +43,7 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str def cluster_modules( leaf_nodes: List[str], components: Dict[str, Node], + config: Config, current_module_tree: dict[str, Any] = {}, current_module_name: str = None, current_module_path: List[str] = [] @@ -53,11 +54,11 @@ def cluster_modules( potential_core_components, potential_core_components_with_code = format_potential_core_components(leaf_nodes, components) if count_tokens(potential_core_components_with_code) <= MAX_TOKEN_PER_MODULE: - logger.info(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens") + logger.debug(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens") return {} prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name) - response = call_llm(prompt, model=CLUSTER_MODEL) + response = call_llm(prompt, config, model=config.cluster_model) #parse the response try: @@ -78,7 +79,7 @@ def cluster_modules( # check if the module tree is valid if len(module_tree) <= 1: - logger.info(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules") + logger.debug(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules") return {} if current_module_tree == {}: @@ -104,7 +105,7 @@ def cluster_modules( current_module_path.append(module_name) module_info["children"] = {} - module_info["children"] = cluster_modules(valid_sub_leaf_nodes, components, current_module_tree, module_name, current_module_path) + module_info["children"] = cluster_modules(valid_sub_leaf_nodes, components, config, current_module_tree, module_name, current_module_path) current_module_path.pop() return module_tree \ No newline at end of file diff --git a/codewiki/src/be/dependency_analyzer/__init__.py b/codewiki/src/be/dependency_analyzer/__init__.py new file mode 100644 index 00000000..283d901d --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates +""" +Dependency analyzer module for building and processing import dependency graphs +between Python code components. +""" + +from codewiki.src.be.dependency_analyzer.models.core import Node +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.topo_sort import topological_sort, resolve_cycles, build_graph_from_components, dependency_first_dfs, get_leaf_nodes +from codewiki.src.be.dependency_analyzer.dependency_graphs_builder import DependencyGraphBuilder + +__all__ = [ + 'Node', + 'DependencyParser', + 'topological_sort', + 'resolve_cycles', + 'build_graph_from_components', + 'dependency_first_dfs', + 'get_leaf_nodes', + 'DependencyGraphBuilder' +] \ No newline at end of file diff --git a/src/be/dependency_analyzer/analysis/__init__.py b/codewiki/src/be/dependency_analyzer/analysis/__init__.py similarity index 100% rename from src/be/dependency_analyzer/analysis/__init__.py rename to codewiki/src/be/dependency_analyzer/analysis/__init__.py diff --git a/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py similarity index 86% rename from src/be/dependency_analyzer/analysis/analysis_service.py rename to codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 75d18cb7..eb9d27b5 100644 --- a/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -9,12 +9,12 @@ import logging from typing import Dict, List, Optional, Any from pathlib import Path -from ..utils.security import safe_open_text, assert_safe_path -from ..analysis.repo_analyzer import RepoAnalyzer -from ..analysis.call_graph_analyzer import CallGraphAnalyzer -from ..analysis.cloning import clone_repository, cleanup_repository, parse_github_url -from ..models.analysis import AnalysisResult -from ..models.core import Repository +from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text, assert_safe_path +from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer +from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer +from codewiki.src.be.dependency_analyzer.analysis.cloning import clone_repository, cleanup_repository, parse_github_url +from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult +from codewiki.src.be.dependency_analyzer.models.core import Repository logger = logging.getLogger(__name__) @@ -55,7 +55,7 @@ def analyze_local_repository( Dict with analysis results including nodes and relationships """ try: - logger.info(f"Analyzing local repository at {repo_path}") + logger.debug(f"Analyzing local repository at {repo_path}") # Get repo analyzer to find files repo_analyzer = RepoAnalyzer() @@ -71,9 +71,9 @@ def analyze_local_repository( # Limit number of files if len(code_files) > max_files: code_files = code_files[:max_files] - logger.info(f"Limited analysis to {max_files} files") + logger.debug(f"Limited analysis to {max_files} files") - logger.info(f"Analyzing {len(code_files)} files") + logger.debug(f"Analyzing {len(code_files)} files") # Analyze files result = self.call_graph_analyzer.analyze_code_files(code_files, repo_path) @@ -115,18 +115,18 @@ def analyze_repository_full( """ temp_dir = None try: - logger.info(f"Starting full analysis of {github_url}") + logger.debug(f"Starting full analysis of {github_url}") temp_dir = self._clone_repository(github_url) repo_info = self._parse_repository_info(github_url) - logger.info("Analyzing repository file structure...") + logger.debug("Analyzing repository file structure...") structure_result = self._analyze_structure(temp_dir, include_patterns, exclude_patterns) - logger.info(f"Found {structure_result['summary']['total_files']} files to analyze.") + logger.debug(f"Found {structure_result['summary']['total_files']} files to analyze.") - logger.info("Starting call graph analysis...") + logger.debug("Starting call graph analysis...") call_graph_result = self._analyze_call_graph(structure_result["file_tree"], temp_dir) - logger.info( + logger.debug( f"Call graph analysis complete. Found {call_graph_result['call_graph']['total_functions']} functions." ) @@ -152,10 +152,10 @@ def analyze_repository_full( readme_content=readme_content, ) - logger.info(f"Cleaning up temporary repository directory: {temp_dir}") + logger.debug(f"Cleaning up temporary repository directory: {temp_dir}") self._cleanup_repository(temp_dir) - logger.info( + logger.debug( f"Analysis completed: {analysis_result.summary['total_functions']} functions found" ) return analysis_result @@ -185,7 +185,7 @@ def analyze_repository_structure_only( """ temp_dir = None try: - logger.info(f"Starting structure analysis of {github_url}") + logger.debug(f"Starting structure analysis of {github_url}") temp_dir = self._clone_repository(github_url) repo_info = self._parse_repository_info(github_url) @@ -203,7 +203,7 @@ def analyze_repository_structure_only( self._cleanup_repository(temp_dir) - logger.info( + logger.debug( f"Structure analysis completed: {result['file_summary']['total_files']} files found" ) return result @@ -216,9 +216,9 @@ def analyze_repository_structure_only( def _clone_repository(self, github_url: str) -> str: """Clone repository and return temp dir path.""" - logger.info(f"Cloning {github_url}...") + logger.debug(f"Cloning {github_url}...") temp_dir = clone_repository(github_url) - logger.info(f"Repository cloned to {temp_dir}") + logger.debug(f"Repository cloned to {temp_dir}") self._temp_directories.append(temp_dir) return temp_dir @@ -233,7 +233,7 @@ def _analyze_structure( exclude_patterns: Optional[List[str]], ) -> Dict[str, Any]: """Analyze repository file structure with filtering.""" - logger.info( + logger.debug( f"Initializing RepoAnalyzer with include: {include_patterns}, exclude: {exclude_patterns}" ) repo_analyzer = RepoAnalyzer(include_patterns, exclude_patterns) @@ -246,12 +246,12 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: # readme_path = Path(repo_dir) / name # if readme_path.exists(): # try: - # logger.info(f"Found README file at {readme_path}") + # logger.debug(f"Found README file at {readme_path}") # return readme_path.read_text(encoding="utf-8") # except Exception as e: # logger.warning(f"Could not read README file at {readme_path}: {e}") # return None - # logger.info("No README file found in repository root.") + # logger.debug("No README file found in repository root.") # return None base = Path(repo_dir) possible_readme_names = ["README.md", "README", "readme.md", "README.txt"] @@ -260,12 +260,12 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: if p.exists(): try: assert_safe_path(base, p) - logger.info(f"Found README file at {p}") + logger.debug(f"Found README file at {p}") return safe_open_text(base, p, encoding="utf-8") except Exception as e: logger.warning(f"Skipping unsafe/ unreadable README at {p}: {e}") return None - logger.info("No README file found in repository root.") + logger.debug("No README file found in repository root.") return None def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[str, Any]: @@ -277,12 +277,12 @@ def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[ - JavaScript/TypeScript AST analysis (planned) - Additional language support (future) """ - logger.info("Extracting code files from file tree...") + logger.debug("Extracting code files from file tree...") code_files = self.call_graph_analyzer.extract_code_files(file_tree) - logger.info(f"Found {len(code_files)} total code files. Filtering for supported languages.") + logger.debug(f"Found {len(code_files)} total code files. Filtering for supported languages.") supported_files = self._filter_supported_languages(code_files) - logger.info(f"Analyzing {len(supported_files)} supported files.") + logger.debug(f"Analyzing {len(supported_files)} supported files.") result = self.call_graph_analyzer.analyze_code_files(supported_files, repo_dir) @@ -321,7 +321,7 @@ def _get_supported_languages(self) -> List[str]: def _cleanup_repository(self, temp_dir: str): """Clean up cloned repository.""" - logger.info(f"Attempting to clean up {temp_dir}") + logger.debug(f"Attempting to clean up {temp_dir}") cleanup_repository(temp_dir) if temp_dir in self._temp_directories: self._temp_directories.remove(temp_dir) diff --git a/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py similarity index 95% rename from src/be/dependency_analyzer/analysis/call_graph_analyzer.py rename to codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index db71790f..e21eeff9 100644 --- a/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -9,9 +9,9 @@ from typing import Dict, List import logging from pathlib import Path -from ..models.core import Node, CallRelationship -from ..utils.patterns import CODE_EXTENSIONS -from ..utils.security import safe_open_text +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS +from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text logger = logging.getLogger(__name__) @@ -155,7 +155,7 @@ def _analyze_python_file(self, file_path: str, content: str, base_dir: str): content: File content string base_dir: Repository base directory path """ - from ..analyzers.python import analyze_python_file + from codewiki.src.be.dependency_analyzer.analyzers.python import analyze_python_file try: functions, relationships = analyze_python_file( @@ -182,7 +182,7 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): try: logger.debug(f"Starting tree-sitter JavaScript analysis for {file_path}") - from ..analyzers.javascript import analyze_javascript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.javascript import analyze_javascript_file_treesitter functions, relationships = analyze_javascript_file_treesitter( file_path, content, repo_path=repo_dir @@ -212,7 +212,7 @@ def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): try: logger.debug(f"Starting tree-sitter TypeScript analysis for {file_path}") - from ..analyzers.typescript import analyze_typescript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.typescript import analyze_typescript_file_treesitter functions, relationships = analyze_typescript_file_treesitter( file_path, content, repo_path=repo_dir @@ -242,7 +242,7 @@ def _analyze_c_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.c import analyze_c_file + from codewiki.src.be.dependency_analyzer.analyzers.c import analyze_c_file functions, relationships = analyze_c_file(file_path, content, repo_path=repo_dir) @@ -260,7 +260,7 @@ def _analyze_cpp_file(self, file_path: str, content: str, repo_dir: str): file_path: Relative path to the C++ file content: File content string """ - from ..analyzers.cpp import analyze_cpp_file + from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file functions, relationships = analyze_cpp_file( file_path, content, repo_path=repo_dir @@ -281,7 +281,7 @@ def _analyze_java_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.java import analyze_java_file + from codewiki.src.be.dependency_analyzer.analyzers.java import analyze_java_file try: functions, relationships = analyze_java_file(file_path, content, repo_path=repo_dir) @@ -305,7 +305,7 @@ def _analyze_csharp_file(self, file_path: str, content: str, repo_dir: str): content: File content string repo_dir: Repository base directory """ - from ..analyzers.csharp import analyze_csharp_file + from codewiki.src.be.dependency_analyzer.analyzers.csharp import analyze_csharp_file try: functions, relationships = analyze_csharp_file(file_path, content, repo_path=repo_dir) diff --git a/src/be/dependency_analyzer/analysis/cloning.py b/codewiki/src/be/dependency_analyzer/analysis/cloning.py similarity index 100% rename from src/be/dependency_analyzer/analysis/cloning.py rename to codewiki/src/be/dependency_analyzer/analysis/cloning.py diff --git a/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py similarity index 97% rename from src/be/dependency_analyzer/analysis/repo_analyzer.py rename to codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index 37c9eab3..887aaa5d 100644 --- a/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -10,7 +10,7 @@ import json from pathlib import Path from typing import Dict, List, Optional, Union -from ..utils.patterns import DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS +from codewiki.src.be.dependency_analyzer.utils.patterns import DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS class RepoAnalyzer: diff --git a/src/be/dependency_analyzer/analyzers/__init__.py b/codewiki/src/be/dependency_analyzer/analyzers/__init__.py similarity index 100% rename from src/be/dependency_analyzer/analyzers/__init__.py rename to codewiki/src/be/dependency_analyzer/analyzers/__init__.py diff --git a/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py similarity index 98% rename from src/be/dependency_analyzer/analyzers/c.py rename to codewiki/src/be/dependency_analyzer/analyzers/c.py index 778d160d..5d19d5b6 100644 --- a/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_c -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/cpp.py b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/cpp.py rename to codewiki/src/be/dependency_analyzer/analyzers/cpp.py index 3f77dd52..dd89d1b3 100644 --- a/src/be/dependency_analyzer/analyzers/cpp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/cpp.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_cpp -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/csharp.py b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/csharp.py rename to codewiki/src/be/dependency_analyzer/analyzers/csharp.py index 813aff9d..50500aa4 100644 --- a/src/be/dependency_analyzer/analyzers/csharp.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/csharp.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_c_sharp -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/java.py b/codewiki/src/be/dependency_analyzer/analyzers/java.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/java.py rename to codewiki/src/be/dependency_analyzer/analyzers/java.py index 8afb0ca7..26f586a1 100644 --- a/src/be/dependency_analyzer/analyzers/java.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/java.py @@ -6,7 +6,7 @@ from tree_sitter import Parser, Language import tree_sitter_java -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/javascript.py b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/javascript.py rename to codewiki/src/be/dependency_analyzer/analyzers/javascript.py index 3babe768..2cd9f120 100644 --- a/src/be/dependency_analyzer/analyzers/javascript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/javascript.py @@ -9,7 +9,7 @@ import tree_sitter_javascript import tree_sitter_typescript -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/python.py b/codewiki/src/be/dependency_analyzer/analyzers/python.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/python.py rename to codewiki/src/be/dependency_analyzer/analyzers/python.py index 12c282ef..c50b1a7e 100644 --- a/src/be/dependency_analyzer/analyzers/python.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/python.py @@ -6,7 +6,7 @@ import os -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/analyzers/typescript.py b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py similarity index 99% rename from src/be/dependency_analyzer/analyzers/typescript.py rename to codewiki/src/be/dependency_analyzer/analyzers/typescript.py index 0a7cc284..24a5b6e0 100644 --- a/src/be/dependency_analyzer/analyzers/typescript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py @@ -9,7 +9,7 @@ from tree_sitter import Parser, Language import tree_sitter_typescript -from ..models.core import Node, CallRelationship +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship logger = logging.getLogger(__name__) diff --git a/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py similarity index 67% rename from src/be/dependency_analyzer/ast_parser.py rename to codewiki/src/be/dependency_analyzer/ast_parser.py index 78b2c8a3..83dbe959 100644 --- a/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -7,14 +7,9 @@ from pathlib import Path import re -from .analysis.analysis_service import AnalysisService -from .utils.patterns import CODE_EXTENSIONS -from .models.core import Node +from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService +from codewiki.src.be.dependency_analyzer.models.core import Node -from pathlib import Path -from config import MAIN_MODEL -from ..llm_services import call_llm -from ..prompt_template import FILTER_FOLDERS_PROMPT logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -31,7 +26,7 @@ def __init__(self, repo_path: str): self.analysis_service = AnalysisService() def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node]: - logger.info(f"Parsing repository at {self.repo_path}") + logger.debug(f"Parsing repository at {self.repo_path}") structure_result = self.analysis_service._analyze_structure( self.repo_path, @@ -46,7 +41,7 @@ def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node self._build_components_from_analysis(call_graph_result) - logger.info(f"Found {len(self.components)} components across {len(self.modules)} modules") + logger.debug(f"Found {len(self.components)} components across {len(self.modules)} modules") return self.components def _build_components_from_analysis(self, call_graph_result: Dict): @@ -146,64 +141,5 @@ def save_dependency_graph(self, output_path: str): with open(output_path, 'w', encoding='utf-8') as f: json.dump(result, f, indent=2, ensure_ascii=False) - logger.info(f"Saved {len(self.components)} components to {output_path}") + logger.debug(f"Saved {len(self.components)} components to {output_path}") return result - - def filter_folders(self) -> List[str]: - - def get_items_at_depth_pathlib(project_path, k): - """ - Alternative implementation using pathlib with recursion. - - Args: - project_path (str): Path to the project directory - k (int): Depth level (0 = root level, 1 = one level deep, etc.) - - Returns: - list: List of relative paths (strings) to files and folders at depth k - """ - project_path = Path(project_path).resolve() - - if not project_path.exists(): - return [] - - if k == 0: - return ['.'] - - def get_items_recursive(current_path, current_depth, target_depth): - items = [] - - if current_depth == target_depth: - # We're at the target depth, return this item - rel_path = current_path.relative_to(project_path) - return [str(rel_path)] - - if current_depth < target_depth and current_path.is_dir(): - # Go deeper - try: - for item in current_path.iterdir(): - items.extend(get_items_recursive(item, current_depth + 1, target_depth)) - except PermissionError: - # Skip directories we can't access - pass - - return items - - result = [] - try: - for item in project_path.iterdir(): - result.extend(get_items_recursive(item, 1, k)) - except PermissionError: - pass - - return "\n".join(sorted(result)) - - prompt = FILTER_FOLDERS_PROMPT.format(files=get_items_at_depth_pathlib(self.repo_path, 1), project_name=Path(self.repo_path).name) - - response = call_llm(prompt, model=MAIN_MODEL) - - # regrex get content between [ and ] - match = re.search(r'\[.*?\]', response, re.DOTALL) - - return eval(match.group(0)) - diff --git a/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py similarity index 89% rename from src/be/dependency_analyzer/dependency_graphs_builder.py rename to codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index f0e87017..df669896 100644 --- a/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -1,9 +1,9 @@ from typing import Dict, List, Any import os -from config import Config -from .ast_parser import DependencyParser -from .topo_sort import build_graph_from_components, get_leaf_nodes -from utils import file_manager +from codewiki.src.config import Config +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes +from codewiki.src.utils import file_manager import logging logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: filtered_folders = None # if os.path.exists(filtered_folders_path): - # logger.info(f"Loading filtered folders from {filtered_folders_path}") + # logger.debug(f"Loading filtered folders from {filtered_folders_path}") # filtered_folders = file_manager.load_json(filtered_folders_path) # else: # # Parse repository diff --git a/src/be/dependency_analyzer/models/__init__.py b/codewiki/src/be/dependency_analyzer/models/__init__.py similarity index 100% rename from src/be/dependency_analyzer/models/__init__.py rename to codewiki/src/be/dependency_analyzer/models/__init__.py diff --git a/src/be/dependency_analyzer/models/analysis.py b/codewiki/src/be/dependency_analyzer/models/analysis.py similarity index 85% rename from src/be/dependency_analyzer/models/analysis.py rename to codewiki/src/be/dependency_analyzer/models/analysis.py index 28e42856..37c0d547 100644 --- a/src/be/dependency_analyzer/models/analysis.py +++ b/codewiki/src/be/dependency_analyzer/models/analysis.py @@ -1,6 +1,6 @@ from pydantic import BaseModel from typing import List, Dict, Any, Optional -from .core import Node, CallRelationship, Repository +from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship, Repository class AnalysisResult(BaseModel): diff --git a/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py similarity index 100% rename from src/be/dependency_analyzer/models/core.py rename to codewiki/src/be/dependency_analyzer/models/core.py diff --git a/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py similarity index 98% rename from src/be/dependency_analyzer/topo_sort.py rename to codewiki/src/be/dependency_analyzer/topo_sort.py index 1171464d..d088fba2 100644 --- a/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -10,7 +10,7 @@ from typing import Dict, List, Set, Any from collections import deque -from .models.core import Node +from codewiki.src.be.dependency_analyzer.models.core import Node logger = logging.getLogger(__name__) @@ -321,7 +321,7 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes = concise_node(leaf_nodes) if len(concise_leaf_nodes) >= 400: - logger.info(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") + logger.debug(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") # Remove nodes that are dependencies of other nodes for node, deps in acyclic_graph.items(): for dep in deps: diff --git a/src/be/dependency_analyzer/utils/__init__.py b/codewiki/src/be/dependency_analyzer/utils/__init__.py similarity index 100% rename from src/be/dependency_analyzer/utils/__init__.py rename to codewiki/src/be/dependency_analyzer/utils/__init__.py diff --git a/src/be/dependency_analyzer/utils/logging_config.py b/codewiki/src/be/dependency_analyzer/utils/logging_config.py similarity index 100% rename from src/be/dependency_analyzer/utils/logging_config.py rename to codewiki/src/be/dependency_analyzer/utils/logging_config.py diff --git a/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py similarity index 100% rename from src/be/dependency_analyzer/utils/patterns.py rename to codewiki/src/be/dependency_analyzer/utils/patterns.py diff --git a/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py similarity index 100% rename from src/be/dependency_analyzer/utils/security.py rename to codewiki/src/be/dependency_analyzer/utils/security.py diff --git a/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py similarity index 80% rename from src/be/documentation_generator.py rename to codewiki/src/be/documentation_generator.py index 35026e81..86018273 100644 --- a/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -10,22 +10,21 @@ logger = logging.getLogger(__name__) # Local imports -from .dependency_analyzer import DependencyGraphBuilder -from .llm_services import call_llm -from .prompt_template import ( +from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder +from codewiki.src.be.llm_services import call_llm +from codewiki.src.be.prompt_template import ( REPO_OVERVIEW_PROMPT, MODULE_OVERVIEW_PROMPT, ) -from .cluster_modules import cluster_modules -from config import ( +from codewiki.src.be.cluster_modules import cluster_modules +from codewiki.src.config import ( Config, FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME, - OVERVIEW_FILENAME, - MAIN_MODEL + OVERVIEW_FILENAME ) -from utils import file_manager -from .agent_orchestrator import AgentOrchestrator +from codewiki.src.utils import file_manager +from codewiki.src.be.agent_orchestrator import AgentOrchestrator class DocumentationGenerator: @@ -44,7 +43,7 @@ def create_documentation_metadata(self, working_dir: str, components: Dict[str, metadata = { "generation_info": { "timestamp": datetime.now().isoformat(), - "main_model": MAIN_MODEL, + "main_model": self.config.main_model, "generator_version": "1.0.0", "repo_path": self.config.repo_path, "commit_id": self.commit_id @@ -71,7 +70,7 @@ def create_documentation_metadata(self, working_dir: str, components: Dict[str, metadata_path = os.path.join(working_dir, "metadata.json") file_manager.save_json(metadata, metadata_path) - logger.info(f"Documentation metadata saved to: {metadata_path}") + logger.debug(f"Documentation metadata saved to: {metadata_path}") def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]: """Get the processing order using topological sort (leaf modules first).""" @@ -136,7 +135,7 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n # Get processing order (leaf modules first) processing_order = self.get_processing_order(first_module_tree) - logger.info(f"Processing {len(processing_order)} modules in dependency order:\n{processing_order}") + logger.debug(f"Processing {len(processing_order)} modules in dependency order:\n{processing_order}") # Process modules in dependency order final_module_tree = module_tree @@ -176,20 +175,24 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n continue # Generate repo overview - logger.info(f"Generating repo overview") + logger.debug(f"Generating repo overview") final_module_tree = await self.generate_parent_module_docs( [], working_dir ) else: - logger.info(f"Processing whole repo because repo can fit in the context window") + logger.debug(f"Processing whole repo because repo can fit in the context window") repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) final_module_tree = await self.agent_orchestrator.process_module( repo_name, components, leaf_nodes, [], working_dir ) + # save final_module_tree to module_tree.json + file_manager.save_json(final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)) + # rename repo_name.md to overview.md repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") - os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) + if os.path.exists(repo_overview_path): + os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) return working_dir @@ -198,12 +201,24 @@ async def generate_parent_module_docs(self, module_path: List[str], """Generate documentation for a parent module based on its children's documentation.""" module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path)) - logger.info(f"Generating parent documentation for: {module_name}") + logger.debug(f"Generating parent documentation for: {module_name}") # Load module tree module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) - + + # check if overview docs already exists + overview_docs_path = os.path.join(working_dir, OVERVIEW_FILENAME) + if os.path.exists(overview_docs_path): + logger.info(f"Overview docs already exists at {overview_docs_path}") + return module_tree + + # check if parent docs already exists + parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") + if os.path.exists(parent_docs_path): + logger.info(f"Parent docs already exists at {parent_docs_path}") + return module_tree + # Create repo structure with 1-depth children docs and target indicator repo_structure = self.build_overview_structure(module_tree, module_path, working_dir) @@ -216,15 +231,14 @@ async def generate_parent_module_docs(self, module_path: List[str], ) try: - parent_docs = call_llm(prompt) + parent_docs = call_llm(prompt, self.config) # Parse and save parent documentation parent_content = parent_docs.split("")[1].split("")[0].strip() # parent_content = prompt - parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") file_manager.save_text(parent_content, parent_docs_path) - logger.info(f"Successfully generated parent documentation for: {module_name}") + logger.debug(f"Successfully generated parent documentation for: {module_name}") return module_tree except Exception as e: @@ -237,8 +251,8 @@ async def run(self) -> None: # Build dependency graph components, leaf_nodes = self.graph_builder.build_dependency_graph() - logger.info(f"Found {len(leaf_nodes)} leaf nodes") - # logger.info(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") + logger.debug(f"Found {len(leaf_nodes)} leaf nodes") + # logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") # exit() # Cluster modules @@ -249,16 +263,16 @@ async def run(self) -> None: # Check if module tree exists if os.path.exists(first_module_tree_path): - logger.info(f"Module tree found at {first_module_tree_path}") + logger.debug(f"Module tree found at {first_module_tree_path}") module_tree = file_manager.load_json(first_module_tree_path) else: - logger.info(f"Module tree not found at {module_tree_path}, clustering modules") - module_tree = cluster_modules(leaf_nodes, components) + logger.debug(f"Module tree not found at {module_tree_path}, clustering modules") + module_tree = cluster_modules(leaf_nodes, components, self.config) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) - logger.info(f"Grouped components into {len(module_tree)} modules") + logger.debug(f"Grouped components into {len(module_tree)} modules") # Generate module documentation using dynamic programming approach # This processes leaf modules first, then parent modules @@ -267,9 +281,9 @@ async def run(self) -> None: # Create documentation metadata self.create_documentation_metadata(working_dir, components, len(leaf_nodes)) - logger.info(f"Documentation generation completed successfully using dynamic programming!") - logger.info(f"Processing order: leaf modules → parent modules → repository overview") - logger.info(f"Documentation saved to: {working_dir}") + logger.debug(f"Documentation generation completed successfully using dynamic programming!") + logger.debug(f"Processing order: leaf modules → parent modules → repository overview") + logger.debug(f"Documentation saved to: {working_dir}") except Exception as e: logger.error(f"Documentation generation failed: {str(e)}") diff --git a/codewiki/src/be/llm_services.py b/codewiki/src/be/llm_services.py new file mode 100644 index 00000000..4ff99dbb --- /dev/null +++ b/codewiki/src/be/llm_services.py @@ -0,0 +1,86 @@ +""" +LLM service factory for creating configured LLM clients. +""" +from pydantic_ai.models.openai import OpenAIModel +from pydantic_ai.providers.openai import OpenAIProvider +from pydantic_ai.models.openai import OpenAIModelSettings +from pydantic_ai.models.fallback import FallbackModel +from openai import OpenAI + +from codewiki.src.config import Config + + +def create_main_model(config: Config) -> OpenAIModel: + """Create the main LLM model from configuration.""" + return OpenAIModel( + model_name=config.main_model, + provider=OpenAIProvider( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ), + settings=OpenAIModelSettings( + temperature=0.0, + max_tokens=32768 + ) + ) + + +def create_fallback_model(config: Config) -> OpenAIModel: + """Create the fallback LLM model from configuration.""" + return OpenAIModel( + model_name=config.fallback_model, + provider=OpenAIProvider( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ), + settings=OpenAIModelSettings( + temperature=0.0, + max_tokens=32768 + ) + ) + + +def create_fallback_models(config: Config) -> FallbackModel: + """Create fallback models chain from configuration.""" + main = create_main_model(config) + fallback = create_fallback_model(config) + return FallbackModel(main, fallback) + + +def create_openai_client(config: Config) -> OpenAI: + """Create OpenAI client from configuration.""" + return OpenAI( + base_url=config.llm_base_url, + api_key=config.llm_api_key + ) + + +def call_llm( + prompt: str, + config: Config, + model: str = None, + temperature: float = 0.0 +) -> str: + """ + Call LLM with the given prompt. + + Args: + prompt: The prompt to send + config: Configuration containing LLM settings + model: Model name (defaults to config.main_model) + temperature: Temperature setting + + Returns: + LLM response text + """ + if model is None: + model = config.main_model + + client = create_openai_client(config) + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature, + max_tokens=32768 + ) + return response.choices[0].message.content \ No newline at end of file diff --git a/src/be/main.py b/codewiki/src/be/main.py similarity index 89% rename from src/be/main.py rename to codewiki/src/be/main.py index 842dd9ad..0f24e170 100644 --- a/src/be/main.py +++ b/codewiki/src/be/main.py @@ -17,8 +17,8 @@ logger = logging.getLogger(__name__) # Local imports -from .documentation_generator import DocumentationGenerator -from config import ( +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import ( Config, ) @@ -50,7 +50,7 @@ async def main() -> None: await doc_generator.run() except KeyboardInterrupt: - logger.info("Documentation generation interrupted by user") + logger.debug("Documentation generation interrupted by user") except Exception as e: logger.error(f"Unexpected error: {str(e)}") raise diff --git a/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py similarity index 99% rename from src/be/prompt_template.py rename to codewiki/src/be/prompt_template.py index 72ac3f59..a979c185 100644 --- a/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -209,7 +209,7 @@ """ from typing import Dict, Any -from utils import file_manager +from codewiki.src.utils import file_manager EXTENSION_TO_LANGUAGE = { ".py": "python", diff --git a/src/be/utils.py b/codewiki/src/be/utils.py similarity index 89% rename from src/be/utils.py rename to codewiki/src/be/utils.py index 4723c40e..953c8047 100644 --- a/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -33,7 +33,7 @@ def count_tokens(text: str) -> int: Count the number of tokens in a text. """ length = len(enc.encode(text)) - logger.info(f"Number of tokens: {length}") + logger.debug(f"Number of tokens: {length}") return length @@ -76,7 +76,7 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st errors.append(error_msg) if errors: - logger.info(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") + logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}") if errors: return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors) @@ -135,15 +135,27 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s Returns: Error message if invalid, empty string if valid """ + import sys + import os + from io import StringIO core_error = "" try: from mermaid_parser.parser import parse_mermaid_py - logger.info("Using mermaid-parser-py to validate mermaid diagrams") + logger.debug("Using mermaid-parser-py to validate mermaid diagrams") try: - json_output = await parse_mermaid_py(diagram_content) + # Redirect stderr to suppress mermaid parser JavaScript errors + old_stderr = sys.stderr + sys.stderr = open(os.devnull, 'w') + + try: + json_output = await parse_mermaid_py(diagram_content) + finally: + # Restore stderr + sys.stderr.close() + sys.stderr = old_stderr except Exception as e: error_str = str(e) @@ -160,7 +172,7 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s raise Exception(error_str) except Exception as e: - logger.info("Using mermaid-py to validate mermaid diagrams") + logger.debug("Using mermaid-py to validate mermaid diagrams") try: import mermaid as md # Create Mermaid object and check response diff --git a/codewiki/src/config.py b/codewiki/src/config.py new file mode 100644 index 00000000..4b8f9845 --- /dev/null +++ b/codewiki/src/config.py @@ -0,0 +1,114 @@ +from dataclasses import dataclass +import argparse +import os +import sys +from dotenv import load_dotenv +load_dotenv() + +# Constants +OUTPUT_BASE_DIR = 'output' +DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' +DOCS_DIR = 'docs' +FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' +MODULE_TREE_FILENAME = 'module_tree.json' +OVERVIEW_FILENAME = 'overview.md' +MAX_DEPTH = 2 +MAX_TOKEN_PER_MODULE = 36_369 +MAX_TOKEN_PER_LEAF_MODULE = 16_000 + +# CLI context detection +_CLI_CONTEXT = False + +def set_cli_context(enabled: bool = True): + """Set whether we're running in CLI context (vs web app).""" + global _CLI_CONTEXT + _CLI_CONTEXT = enabled + +def is_cli_context() -> bool: + """Check if running in CLI context.""" + return _CLI_CONTEXT + +# LLM services +# In CLI mode, these will be loaded from ~/.codewiki/config.json + keyring +# In web app mode, use environment variables +MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') +FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') +CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) +LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') +LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') + +@dataclass +class Config: + """Configuration class for CodeWiki.""" + repo_path: str + output_dir: str + dependency_graph_dir: str + docs_dir: str + max_depth: int + # LLM configuration + llm_base_url: str + llm_api_key: str + main_model: str + cluster_model: str + fallback_model: str = FALLBACK_MODEL_1 + + @classmethod + def from_args(cls, args: argparse.Namespace) -> 'Config': + """Create configuration from parsed arguments.""" + repo_name = os.path.basename(os.path.normpath(args.repo_path)) + sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) + + return cls( + repo_path=args.repo_path, + output_dir=OUTPUT_BASE_DIR, + dependency_graph_dir=os.path.join(OUTPUT_BASE_DIR, DEPENDENCY_GRAPHS_DIR), + docs_dir=os.path.join(OUTPUT_BASE_DIR, DOCS_DIR, f"{sanitized_repo_name}-docs"), + max_depth=MAX_DEPTH, + llm_base_url=LLM_BASE_URL, + llm_api_key=LLM_API_KEY, + main_model=MAIN_MODEL, + cluster_model=CLUSTER_MODEL, + fallback_model=FALLBACK_MODEL_1 + ) + + @classmethod + def from_cli( + cls, + repo_path: str, + output_dir: str, + llm_base_url: str, + llm_api_key: str, + main_model: str, + cluster_model: str, + fallback_model: str = FALLBACK_MODEL_1 + ) -> 'Config': + """ + Create configuration for CLI context. + + Args: + repo_path: Repository path + output_dir: Output directory for generated docs + llm_base_url: LLM API base URL + llm_api_key: LLM API key + main_model: Primary model + cluster_model: Clustering model + fallback_model: Fallback model + + Returns: + Config instance + """ + repo_name = os.path.basename(os.path.normpath(repo_path)) + base_output_dir = os.path.join(output_dir, "temp") + + return cls( + repo_path=repo_path, + output_dir=base_output_dir, + dependency_graph_dir=os.path.join(base_output_dir, DEPENDENCY_GRAPHS_DIR), + docs_dir=output_dir, + max_depth=MAX_DEPTH, + llm_base_url=llm_base_url, + llm_api_key=llm_api_key, + main_model=main_model, + cluster_model=cluster_model, + fallback_model=fallback_model + ) \ No newline at end of file diff --git a/src/fe/__init__.py b/codewiki/src/fe/__init__.py similarity index 100% rename from src/fe/__init__.py rename to codewiki/src/fe/__init__.py diff --git a/src/fe/background_worker.py b/codewiki/src/fe/background_worker.py similarity index 95% rename from src/fe/background_worker.py rename to codewiki/src/fe/background_worker.py index bce9a077..eec230f7 100644 --- a/src/fe/background_worker.py +++ b/codewiki/src/fe/background_worker.py @@ -15,13 +15,13 @@ from typing import Dict from dataclasses import asdict -from be.documentation_generator import DocumentationGenerator -from config import Config, MAIN_MODEL +from codewiki.src.be.documentation_generator import DocumentationGenerator +from codewiki.src.config import Config from .models import JobStatus from .cache_manager import CacheManager from .github_processor import GitHubRepoProcessor from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class BackgroundWorker: """Background worker for processing documentation generation jobs.""" @@ -203,14 +203,12 @@ def _process_job(self, job_id: str): # Generate documentation job.progress = "Analyzing repository structure..." - # Create config for documentation generation - config = Config( - repo_path=temp_repo_dir, - output_dir="output", - dependency_graph_dir=os.path.join("output", "dependency_graphs"), - docs_dir=os.path.join("output", "docs", f"{job_id}-docs"), - max_depth=2 - ) + # Create config for documentation generation (using env vars) + import argparse + args = argparse.Namespace(repo_path=temp_repo_dir) + config = Config.from_args(args) + # Override docs_dir with job-specific directory + config.docs_dir = os.path.join("output", "docs", f"{job_id}-docs") job.progress = "Generating documentation..." diff --git a/src/fe/cache_manager.py b/codewiki/src/fe/cache_manager.py similarity index 98% rename from src/fe/cache_manager.py rename to codewiki/src/fe/cache_manager.py index e9e41d8a..d1560519 100644 --- a/src/fe/cache_manager.py +++ b/codewiki/src/fe/cache_manager.py @@ -10,7 +10,7 @@ from .models import CacheEntry from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class CacheManager: diff --git a/src/fe/config.py b/codewiki/src/fe/config.py similarity index 100% rename from src/fe/config.py rename to codewiki/src/fe/config.py diff --git a/src/fe/github_processor.py b/codewiki/src/fe/github_processor.py similarity index 100% rename from src/fe/github_processor.py rename to codewiki/src/fe/github_processor.py diff --git a/src/fe/models.py b/codewiki/src/fe/models.py similarity index 100% rename from src/fe/models.py rename to codewiki/src/fe/models.py diff --git a/src/fe/routes.py b/codewiki/src/fe/routes.py similarity index 99% rename from src/fe/routes.py rename to codewiki/src/fe/routes.py index 299ec5c9..40da7955 100644 --- a/src/fe/routes.py +++ b/codewiki/src/fe/routes.py @@ -19,7 +19,7 @@ from .templates import WEB_INTERFACE_TEMPLATE from .template_utils import render_template from .config import WebAppConfig -from utils import file_manager +from codewiki.src.utils import file_manager class WebRoutes: diff --git a/src/fe/template_utils.py b/codewiki/src/fe/template_utils.py similarity index 100% rename from src/fe/template_utils.py rename to codewiki/src/fe/template_utils.py diff --git a/src/fe/templates.py b/codewiki/src/fe/templates.py similarity index 100% rename from src/fe/templates.py rename to codewiki/src/fe/templates.py diff --git a/src/fe/visualise_docs.py b/codewiki/src/fe/visualise_docs.py similarity index 99% rename from src/fe/visualise_docs.py rename to codewiki/src/fe/visualise_docs.py index bf5a68eb..2c8648dc 100644 --- a/src/fe/visualise_docs.py +++ b/codewiki/src/fe/visualise_docs.py @@ -23,7 +23,7 @@ from .template_utils import render_template from .templates import DOCS_VIEW_TEMPLATE -from utils import file_manager +from codewiki.src.utils import file_manager app = FastAPI(title="Documentation Server", description="Simple documentation server for hosting markdown documentation folders") diff --git a/src/fe/web_app.py b/codewiki/src/fe/web_app.py similarity index 100% rename from src/fe/web_app.py rename to codewiki/src/fe/web_app.py diff --git a/src/utils.py b/codewiki/src/utils.py similarity index 100% rename from src/utils.py rename to codewiki/src/utils.py diff --git a/codewiki/templates/github_pages/.gitkeep b/codewiki/templates/github_pages/.gitkeep new file mode 100644 index 00000000..d5f5da1f --- /dev/null +++ b/codewiki/templates/github_pages/.gitkeep @@ -0,0 +1,3 @@ +# Placeholder file to ensure directory is tracked +# This directory will contain HTML templates for GitHub Pages viewer + diff --git a/codewiki/templates/github_pages/viewer_template.html b/codewiki/templates/github_pages/viewer_template.html new file mode 100644 index 00000000..39629ade --- /dev/null +++ b/codewiki/templates/github_pages/viewer_template.html @@ -0,0 +1,643 @@ + + + + + + {{TITLE}} + + + + + +
+ + +
+
+ +

Loading documentation...

+
+ + +
+
+ + + + + diff --git a/.dockerignore b/docker/.dockerignore similarity index 100% rename from .dockerignore rename to docker/.dockerignore diff --git a/docker/DOCKER_README.md b/docker/DOCKER_README.md new file mode 100644 index 00000000..c45d7f06 --- /dev/null +++ b/docker/DOCKER_README.md @@ -0,0 +1,443 @@ +# CodeWiki Docker Setup + +This document explains how to run CodeWiki using Docker and Docker Compose. + +## Overview + +The Docker setup provides a containerized environment for running the CodeWiki web application, which allows you to generate documentation for GitHub repositories through a web interface. + +## File Structure + +All Docker-related files are located in the `docker/` directory: + +``` +docker/ +├── Dockerfile # Container image definition +├── docker-compose.yml # Service orchestration +├── env.example # Environment variables template +└── DOCKER_README.md # This file +``` + +The Dockerfile builds from the project root context to include all necessary application code. + +--- + +## Quick Start + +### 1. Clone the Repository + +```bash +git clone +cd CodeWiki +``` + +### 2. Set Up Environment Variables + +```bash +# Copy the example environment file +cp docker/env.example .env + +# Edit .env file with your configuration +nano .env # or use your preferred editor +``` + +Required configuration in `.env`: + +```bash +# LLM API Configuration +MAIN_MODEL=claude-sonnet-4 +FALLBACK_MODEL_1=glm-4p5 +CLUSTER_MODEL=claude-sonnet-4 +LLM_BASE_URL=https://api.anthropic.com # or your LiteLLM proxy +LLM_API_KEY=your-api-key-here + +# Application Port +APP_PORT=8000 + +# Optional: Logfire Configuration (for monitoring) +LOGFIRE_TOKEN= +LOGFIRE_PROJECT_NAME=codewiki +LOGFIRE_SERVICE_NAME=codewiki +``` + +### 3. Create Docker Network + +```bash +docker network create codewiki-network +``` + +### 4. Start the Services + +**Option A: From project root** +```bash +docker-compose -f docker/docker-compose.yml up -d +``` + +**Option B: From docker directory** +```bash +cd docker +docker-compose up -d +``` + +### 5. Access the Application + +- Web Application: http://localhost:8000 + +The application will be available at the port specified in your `.env` file (default: 8000). + +--- + +## Docker Compose Configuration + +The `docker-compose.yml` file defines the CodeWiki service with the following features: + +### Service Configuration + +- **Image**: `codewiki:0.0.1` +- **Build Context**: Parent directory (`.` relative to docker/) +- **Container Name**: `codewiki` +- **Port Mapping**: `${APP_PORT:-8000}:8000` +- **Network**: `codewiki-network` (external) + +### Environment Variables + +The container uses environment variables from the `.env` file: +- `PYTHONPATH=/app/src` - Set Python module path +- `PYTHONUNBUFFERED=1` - Enable real-time logging +- All variables from `.env` file + +### Volume Mounts + +The following directories are mounted as volumes: + +```yaml +volumes: + - ./output:/app/output # Persistent storage for generated docs + - ~/.ssh:/root/.ssh:ro # SSH keys for private repos (read-only) +``` + +**Note**: Git credentials can be mounted if needed for private repositories: +```yaml + # Uncomment in docker-compose.yml if needed + - ~/.gitconfig:/root/.gitconfig:ro +``` + +### Health Check + +The service includes a health check that: +- Runs every 30 seconds +- Times out after 10 seconds +- Retries 3 times on failure +- Starts checking after 20 seconds + +### Restart Policy + +The container is set to restart automatically unless explicitly stopped (`restart: unless-stopped`). + +--- + +## Dockerfile Details + +The Dockerfile (`docker/Dockerfile`) builds the CodeWiki image with: + +### Base Image +- Python 3.12 slim image for smaller size + +### System Dependencies +- `git` - For repository cloning +- `curl` - For health checks +- `nodejs` and `npm` - For mermaid diagram validation + +### Application Setup +1. Copies `requirements.txt` first (for better caching) +2. Installs Python dependencies +3. Copies entire application code +4. Creates output directories: + - `output/cache` + - `output/temp` + - `output/docs` + - `output/dependency_graphs` + +### Runtime Configuration +- **Working Directory**: `/app` +- **Exposed Port**: `8000` +- **Entry Point**: `python codewiki/run_web_app.py --host 0.0.0.0 --port 8000` + +--- + +## Common Operations + +### View Logs + +```bash +# From project root +docker-compose -f docker/docker-compose.yml logs -f + +# From docker directory +cd docker +docker-compose logs -f + +# View specific service +docker logs codewiki -f +``` + +### Stop Services + +```bash +# From project root +docker-compose -f docker/docker-compose.yml stop + +# From docker directory +cd docker +docker-compose stop +``` + +### Stop and Remove Containers + +```bash +# From project root +docker-compose -f docker/docker-compose.yml down + +# From docker directory +cd docker +docker-compose down + +# Remove volumes as well +docker-compose down -v +``` + +### Rebuild Image + +If you've made changes to the code or Dockerfile: + +```bash +# From project root +docker-compose -f docker/docker-compose.yml build --no-cache + +# From docker directory +cd docker +docker-compose build --no-cache + +# Rebuild and restart +docker-compose up -d --build +``` + +### Access Container Shell + +```bash +docker exec -it codewiki /bin/bash +``` + +--- + +## Persistent Storage + +### Output Directory + +The `output/` directory is mounted as a volume, ensuring generated documentation persists across container restarts: + +``` +output/ +├── cache/ # Cached dependency graphs and jobs +├── docs/ # Generated documentation +├── dependency_graphs/ # JSON dependency graphs +└── temp/ # Temporary files +``` + +### SSH Keys + +If you need to clone private repositories, ensure your SSH keys are available: + +```bash +# Verify SSH keys are accessible +ls -la ~/.ssh/ + +# The docker-compose.yml mounts ~/.ssh as read-only +``` + +--- + +## Troubleshooting + +### Port Already in Use + +If port 8000 is already in use: + +```bash +# Change APP_PORT in .env file +echo "APP_PORT=8001" >> .env + +# Restart services +docker-compose -f docker/docker-compose.yml down +docker-compose -f docker/docker-compose.yml up -d +``` + +### Container Won't Start + +Check logs for errors: + +```bash +docker logs codewiki +``` + +Common issues: +- **Invalid API key**: Verify `LLM_API_KEY` in `.env` +- **Network not found**: Create network with `docker network create codewiki-network` +- **Port conflict**: Change `APP_PORT` in `.env` + +### Health Check Failing + +```bash +# Check if the application is responding +curl http://localhost:8000/ + +# Check container health status +docker inspect codewiki --format='{{.State.Health.Status}}' + +# View health check logs +docker inspect codewiki --format='{{range .State.Health.Log}}{{.Output}}{{end}}' +``` + +### Permission Issues with Volumes + +If you encounter permission issues with mounted volumes: + +```bash +# On Linux, ensure proper ownership +sudo chown -R $(id -u):$(id -g) output/ + +# Or run container with user mapping +docker-compose -f docker/docker-compose.yml down +# Add to docker-compose.yml under 'codewiki' service: +# user: "${UID}:${GID}" +``` + +### Private Repository Access + +For private repositories: + +1. Ensure SSH keys are properly mounted: + ```yaml + volumes: + - ~/.ssh:/root/.ssh:ro + ``` + +2. Verify key permissions: + ```bash + chmod 600 ~/.ssh/id_rsa + chmod 644 ~/.ssh/id_rsa.pub + ``` + +3. Add GitHub to known_hosts: + ```bash + docker exec -it codewiki ssh-keyscan github.com >> /root/.ssh/known_hosts + ``` + +--- + +## Production Deployment + +### Security Considerations + +1. **Environment Variables**: Never commit `.env` file to version control +2. **API Keys**: Use secrets management in production +3. **Network**: Use isolated Docker networks +4. **Volumes**: Set appropriate permissions on mounted volumes +5. **Updates**: Regularly update base image and dependencies + +### Recommended Production Setup + +```yaml +# Use secrets instead of .env file +services: + codewiki: + secrets: + - llm_api_key + environment: + - LLM_API_KEY_FILE=/run/secrets/llm_api_key + +secrets: + llm_api_key: + external: true +``` + +### Resource Limits + +Add resource limits in production: + +```yaml +services: + codewiki: + deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '1' + memory: 2G +``` + +### Reverse Proxy + +Use nginx or traefik as a reverse proxy: + +```yaml +services: + nginx: + image: nginx:alpine + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./certs:/etc/nginx/certs:ro +``` + +--- + +## Integration with External Services + +### Using LiteLLM Proxy + +If using a LiteLLM proxy for LLM API management: + +```bash +# In .env file +LLM_BASE_URL=http://litellm:4000/ +LLM_API_KEY=sk-your-proxy-key + +# Add LiteLLM service to docker-compose.yml +services: + litellm: + image: ghcr.io/berriai/litellm:latest + ports: + - "4000:4000" + environment: + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - OPENAI_API_KEY=${OPENAI_API_KEY} + networks: + - codewiki-network +``` + +--- + +## More Information + +- **Main Documentation**: See [../README.md](../README.md) for complete feature list and usage +- **CLI Tool**: For command-line documentation generation +- **Web Interface**: For GitHub URL-based documentation generation + +--- + +## Support + +For issues related to Docker deployment: +1. Check logs: `docker logs codewiki` +2. Verify configuration: `docker exec codewiki env | grep -E '(LLM|APP)'` +3. Test connectivity: `docker exec codewiki curl -I http://localhost:8000` +4. Report issues: https://github.com/yourusername/codewiki/issues + +--- + +**Happy documenting with Docker! 🐳📚** diff --git a/Dockerfile b/docker/Dockerfile similarity index 77% rename from Dockerfile rename to docker/Dockerfile index 04ee4de2..d0a70d16 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -# Use Python 3.11 slim image as base +# Use Python 3.12 slim image as base FROM python:3.12-slim # Set working directory @@ -19,13 +19,16 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Copy application code -COPY . . +COPY codewiki ./codewiki +COPY img ./img +COPY pyproject.toml . +COPY README.md . # Create output directories RUN mkdir -p output/cache output/temp output/docs output/dependency_graphs # Set environment variables -ENV PYTHONPATH=/app/src +ENV PYTHONPATH=/app ENV PYTHONUNBUFFERED=1 # Expose port @@ -36,4 +39,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD curl -f http://localhost:8000/ || exit 1 # Default command -CMD ["python", "run_web_app.py", "--host", "0.0.0.0", "--port", "8000"] +CMD ["python", "codewiki/run_web_app.py", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 84% rename from docker-compose.yml rename to docker/docker-compose.yml index 33091789..28084e7f 100644 --- a/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,21 +2,21 @@ services: codewiki: image: codewiki:0.0.1 build: - context: . - dockerfile: Dockerfile + context: .. + dockerfile: docker/Dockerfile container_name: codewiki ports: - "${APP_PORT:-8000}:8000" environment: - - PYTHONPATH=/app/src + - PYTHONPATH=/app - PYTHONUNBUFFERED=1 env_file: - - .env + - ../.env networks: - net volumes: # Persistent storage for cache and output - - ./output:/app/output + - ../output:/app/output # Git credentials (if needed for private repos) # - ~/.gitconfig:/root/.gitconfig:ro - ~/.ssh:/root/.ssh:ro diff --git a/env.example b/docker/env.example similarity index 100% rename from env.example rename to docker/env.example diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..ce711dc7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,123 @@ +[build-system] +requires = ["setuptools>=68.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "codewiki" +version = "1.0.0" +description = "Transform codebases into comprehensive documentation using AI-powered analysis" +readme = "README.md" +requires-python = ">=3.12" +license = {text = "MIT"} +authors = [ + {name = "CodeWiki Contributors"} +] +keywords = ["documentation", "code-analysis", "ai", "llm", "developer-tools"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Documentation", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +dependencies = [ + "click>=8.1.0", + "keyring>=24.0.0", + "GitPython>=3.1.40", + "Jinja2>=3.1.6", + "tree-sitter>=0.23.2", + "tree-sitter-language-pack>=0.8.0", + "tree-sitter-python>=0.23.6", + "tree-sitter-java>=0.23.5", + "tree-sitter-javascript>=0.21.4", + "tree-sitter-typescript>=0.21.2", + "tree-sitter-c>=0.21.4", + "tree-sitter-cpp>=0.23.4", + "tree-sitter-c-sharp>=0.23.1", + "openai>=1.107.0", + "litellm>=1.77.0", + "pydantic>=2.11.7", + "pydantic-settings>=2.10.1", + "pydantic-ai>=1.0.6", + "requests>=2.32.4", + "python-dotenv>=1.1.1", + "rich>=14.1.0", + "networkx>=3.5", + "psutil>=7.0.0", + "PyYAML>=6.0.2", + "mermaid-parser-py>=0.0.2", + "mermaid-py>=0.8.0" +] + +[external] +# Node.js is required for mermaid-py which validates mermaid diagrams in generated documentation +build-requires = [ + { name = "nodejs", version = ">=14.0.0" } +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-cov>=4.1.0", + "pytest-asyncio>=0.21.0", + "black>=23.0.0", + "mypy>=1.5.0", + "ruff>=0.1.0", +] + +[project.scripts] +codewiki = "codewiki.cli.main:cli" + +[project.urls] +Homepage = "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/yourusername/codewiki" +Documentation = "/yourusername/codewiki/docs" +Repository = "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/yourusername/codewiki" +Issues = "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/yourusername/codewiki/issues" + +[tool.setuptools] +packages = [ + "codewiki", + "codewiki.cli", + "codewiki.cli.commands", + "codewiki.cli.models", + "codewiki.cli.utils", + "codewiki.cli.adapters", + "codewiki.src", + "codewiki.src.be", + "codewiki.src.be.agent_tools", + "codewiki.src.be.dependency_analyzer", + "codewiki.src.be.dependency_analyzer.analysis", + "codewiki.src.be.dependency_analyzer.analyzers", + "codewiki.src.be.dependency_analyzer.models", + "codewiki.src.be.dependency_analyzer.utils", + "codewiki.src.fe" +] + +[tool.setuptools.package-data] +codewiki = ["templates/**/*", "py.typed"] + +[tool.black] +line-length = 100 +target-version = ['py312'] + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "-v --cov=codewiki --cov-report=term-missing" + diff --git a/src/be/dependency_analyzer/__init__.py b/src/be/dependency_analyzer/__init__.py deleted file mode 100644 index 3183dac1..00000000 --- a/src/be/dependency_analyzer/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates -""" -Dependency analyzer module for building and processing import dependency graphs -between Python code components. -""" - -from .models.core import Node -from .ast_parser import DependencyParser -from .topo_sort import topological_sort, resolve_cycles, build_graph_from_components, dependency_first_dfs, get_leaf_nodes -from .dependency_graphs_builder import DependencyGraphBuilder - -__all__ = [ - 'Node', - 'DependencyParser', - 'topological_sort', - 'resolve_cycles', - 'build_graph_from_components', - 'dependency_first_dfs', - 'get_leaf_nodes', - 'DependencyGraphBuilder' -] \ No newline at end of file diff --git a/src/be/llm_services.py b/src/be/llm_services.py deleted file mode 100644 index 52b28021..00000000 --- a/src/be/llm_services.py +++ /dev/null @@ -1,50 +0,0 @@ -from pydantic_ai.models.openai import OpenAIModel -from pydantic_ai.providers.openai import OpenAIProvider -from pydantic_ai.models.openai import OpenAIModelSettings -from pydantic_ai.models.fallback import FallbackModel - -from config import MAIN_MODEL, FALLBACK_MODEL_1, LLM_BASE_URL, LLM_API_KEY - - -main_model = OpenAIModel( - model_name=MAIN_MODEL, - provider=OpenAIProvider( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY - ), - settings=OpenAIModelSettings( - temperature=0.0, - max_tokens=32768 - ) -) - -fallback_model_1 = OpenAIModel( - model_name=FALLBACK_MODEL_1, - provider=OpenAIProvider( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY - ), - settings=OpenAIModelSettings( - temperature=0.0, - max_tokens=32768 - ) -) - -fallback_models = FallbackModel(main_model, fallback_model_1) - -# ------------------------------------------------------------ -from openai import OpenAI - -client = OpenAI( - base_url=LLM_BASE_URL, - api_key=LLM_API_KEY -) - -def call_llm(prompt: str, model: str = MAIN_MODEL, temperature: float = 0.0): - response = client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=temperature, - max_tokens=32768 - ) - return response.choices[0].message.content \ No newline at end of file diff --git a/src/config.py b/src/config.py deleted file mode 100644 index 29157353..00000000 --- a/src/config.py +++ /dev/null @@ -1,46 +0,0 @@ -from dataclasses import dataclass -import argparse -import os -from dotenv import load_dotenv -load_dotenv() - -# Constants -OUTPUT_BASE_DIR = 'output' -DEPENDENCY_GRAPHS_DIR = 'dependency_graphs' -DOCS_DIR = 'docs' -FIRST_MODULE_TREE_FILENAME = 'first_module_tree.json' -MODULE_TREE_FILENAME = 'module_tree.json' -OVERVIEW_FILENAME = 'overview.md' -MAX_DEPTH = 2 -MAX_TOKEN_PER_MODULE = 36_369 -MAX_TOKEN_PER_LEAF_MODULE = 16_000 - -# LLM services -MAIN_MODEL = os.getenv('MAIN_MODEL', 'claude-sonnet-4') -FALLBACK_MODEL_1 = os.getenv('FALLBACK_MODEL_1', 'glm-4p5') -CLUSTER_MODEL = os.getenv('CLUSTER_MODEL', MAIN_MODEL) -LLM_BASE_URL = os.getenv('LLM_BASE_URL', 'http://0.0.0.0:4000/') -LLM_API_KEY = os.getenv('LLM_API_KEY', 'sk-1234') - -@dataclass -class Config: - """Configuration class for CodeWiki.""" - repo_path: str - output_dir: str - dependency_graph_dir: str - docs_dir: str - max_depth: int - - @classmethod - def from_args(cls, args: argparse.Namespace) -> 'Config': - """Create configuration from parsed arguments.""" - repo_name = os.path.basename(os.path.normpath(args.repo_path)) - sanitized_repo_name = ''.join(c if c.isalnum() else '_' for c in repo_name) - - return cls( - repo_path=args.repo_path, - output_dir=OUTPUT_BASE_DIR, - dependency_graph_dir=os.path.join(OUTPUT_BASE_DIR, DEPENDENCY_GRAPHS_DIR), - docs_dir=os.path.join(OUTPUT_BASE_DIR, DOCS_DIR, f"{sanitized_repo_name}-docs"), - max_depth=MAX_DEPTH - ) \ No newline at end of file