diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..10c743d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,48 @@ +name: Publish to PyPI + +# Runs when a GitHub release is published. Uploads with PyPI trusted publishing: +# no token is stored in the repository. One-time setup on pypi.org: add a trusted +# publisher for project "code2test", repository Cirkitly/code2test, workflow +# "publish.yml" and environment "pypi". + +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check that the release tag matches the package version + run: | + version=$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' code2test/__init__.py) + if [ "v$version" != "$GITHUB_REF_NAME" ]; then + echo "Release tag $GITHUB_REF_NAME does not match package version $version" + exit 1 + fi + - name: Build and check + run: | + python -m pip install --upgrade build twine + python -m build + twine check dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # trusted publishing + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5ce5bc5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +All notable changes to Code2Test are listed here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow [Semantic Versioning](https://semver.org/). + +## [0.1.0] - 2026-10-15 + +First release: an alpha for Python projects. + +### Added + +- `code2test test` generates a pytest suite for existing Python code. For each top-level function and class it infers an intent from the docstring, type hints, naming and call sites, and asks the model when that inference is uncertain. It then generates tests, runs them, repairs failures from their pytest output (up to two rounds), and diagnoses tests that still fail as a wrong test, a code bug or a wrong intent. Suspected code bugs are kept as `xfail` tests that name the bug; everything else that fails is dropped, and a whole-suite run removes tests that only fail alongside others. The written suite therefore always passes. + - Interactive by default: review, edit or skip each intent and set of tests. `--auto` runs without prompts, and `--exit-code` makes it usable in CI. +- `code2test verify` runs the generated suite; `--diagnose` explains failures and `--fix` repairs generated tests after the code changed, without touching tests you edited. +- `code2test benchmark` measures the suite: the share of injected bugs it catches (mutation score), and how often the diagnosis names the right cause of a failure whose cause is known. +- `code2test report coverage` and `code2test test --report html|json|all` report, per component, the behaviors its passing tests check, the injected bugs they catch and the diagnostic accuracy. +- `code2test intent show|edit|export` to review and correct intents. Edited intents are never overwritten by later runs. +- `code2test init` saves project defaults for `code2test test`: confidence threshold, output directory and exclude patterns. +- `code2test config set|show|validate` stores the API key and model; `validate` sends the model one test request. +- Works with any model pydantic-ai supports, including OpenAI-compatible endpoints through `OPENAI_BASE_URL`. + +### Measured + +On inflection 0.5.1 and TinyDB 4.8.2, with their own tests removed, using MiniMax `MiniMax-M3.1-Flash-Preview` and `MiniMax-M3`, two runs each: + +| Measure | Result | Target | +| --- | --- | --- | +| Components with passing tests | 96–100% | 70% | +| Generated tests passing before any repair | 84–99% | 80% | +| Injected bugs caught | 59–84% | | +| Diagnosis correct on failures with a known cause | 97–100% | 75% | + +One run kept a test as an expected failure that exposes a real defect in inflection 0.5.1: `singularize("campuses")` returns `"campuse"`. Details are in the README and in [`research/`](research/). + +### Known limitations + +- Python only; files in other languages are analyzed for the dependency graph but get no tests. +- Each run sends source code to the configured model provider and costs model calls. +- Evaluated on two small libraries with one provider's models; large codebases and other models are not yet measured. +- Not yet available: Markdown reports, and regenerating tests only for code that changed. + +[0.1.0]: https://github.com/Cirkitly/code2test/releases/tag/v0.1.0 diff --git a/README.md b/README.md index a4601e5..78cb6ab 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,12 @@ On two open-source libraries whose own tests were removed, Code2Test produced pa ## Installation ```bash -git clone https://github.com/Cirkitly/code2test.git -cd code2test -pip install . -code2test --help +pip install code2test +code2test --version ``` +To install a release straight from GitHub instead, use `pip install git+https://github.com/Cirkitly/code2test@v0.1.0`. To work on Code2Test itself, see [Development](#development). Changes in each release are listed in [CHANGELOG.md](CHANGELOG.md). + ## Choosing a model Code2Test picks the model in this order: @@ -100,7 +100,7 @@ The suite Code2Test writes therefore always passes: every test either passes or | `code2test benchmark [PATH]` | Measure mutation score and diagnosis accuracy of the generated tests | | `code2test intent show\|edit\|export [NAME]` | Review or correct inferred intents | | `code2test report summary\|coverage` | Summarize stored intents, or report behavioral coverage and diagnostic accuracy | -| `code2test init` | Write a starter `.code2test/config.json` (the other commands take their settings from options, not from this file) | +| `code2test init` | Save project defaults for `code2test test` (confidence threshold, output directory, exclude patterns) in `.code2test/config.json` | | `code2test config set\|show\|validate` | Store the API key and model (`--api-key`, `--main-model`); `validate` sends the model one test request | | `code2test version` | Show the installed version | @@ -110,12 +110,15 @@ The suite Code2Test writes therefore always passes: every test either passes or | --- | --- | --- | | `--auto` | off | Run without prompts | | `--confidence FLOAT` | 0.6 | Intent confidence needed to generate tests in `--auto` mode | -| `--output-dir PATH` | `tests` | Where generated tests are written | -| `--include`, `--exclude GLOB` | | Limit which files are analyzed | +| `--output-dir PATH` | `tests` | Where generated tests are written, relative to the project root | +| `--include GLOB` | | Analyze only matching files | +| `--exclude GLOB` | | Leave matching files out of the analysis; repeatable, and added to the built-in exclusions | | `--dry-run` | off | Generate tests (this calls the model) without writing or running test files | | `--exit-code` | off | Exit 1 if any written file failed verification (for CI) | | `--report html\|json\|all` | none | Write a run report to `/reports/` | +Defaults for `--confidence`, `--output-dir` and extra exclude patterns can be saved per project with `code2test init`, for example `code2test init --confidence 0.7 --exclude "*migrations*"`. They are stored in `.code2test/config.json`; options given on the command line take precedence. The output directory and the usual test, cache and virtual-environment folders are never analyzed. + `PATH` can be the repository, a subdirectory or a single file. Imports, the `tests/` directory and `.code2test/` always resolve to the project root, which is the nearest directory containing `.git`, `pyproject.toml`, `setup.py`, `setup.cfg` or `.code2test`. ### `code2test verify` @@ -172,7 +175,7 @@ The last two come from the most recent `code2test benchmark` run; the report say | `.code2test/metrics.json` | Metrics of the last `test --auto` run | | `.code2test/benchmark.json` | Results of the last `benchmark` run, including uncaught mutants | | `.code2test/dependency_graphs/` | The analyzed dependency graph | -| `.code2test/config.json` | Starter settings written by `code2test init` (not read by other commands) | +| `.code2test/config.json` | Project defaults written by `code2test init` and read by `code2test test` | | `tests/reports/report.html`, `report.json` | Run reports, when `--report` is used | | `tests/reports/coverage.html`, `coverage.json` | Coverage reports from `code2test report coverage --format html\|json` | diff --git a/code2test/__main__.py b/code2test/__main__.py index 00f0ee4..4fcc91c 100644 --- a/code2test/__main__.py +++ b/code2test/__main__.py @@ -1,5 +1,5 @@ """ -Entry point for running codewiki as a module: python -m codewiki +Entry point for running Code2Test as a module: python -m code2test """ from code2test.cli.main import cli diff --git a/code2test/cli/__init__.py b/code2test/cli/__init__.py index 128a92f..6cae345 100644 --- a/code2test/cli/__init__.py +++ b/code2test/cli/__init__.py @@ -1,3 +1,3 @@ -"""CLI module for CodeWiki.""" +"""Command-line interface for Code2Test.""" __all__ = [] diff --git a/code2test/cli/commands/init.py b/code2test/cli/commands/init.py index d39105e..887d4cc 100644 --- a/code2test/cli/commands/init.py +++ b/code2test/cli/commands/init.py @@ -2,69 +2,75 @@ Init command for Code2Test. """ -import json from pathlib import Path +from typing import Optional, Tuple + import click -from rich.prompt import Confirm, Prompt +from rich.prompt import Confirm from code2test.cli.display import DisplayManager +from code2test.cli.utils.project_config import ( + PROJECT_DEFAULTS, + project_config_path, + write_project_settings, +) @click.command("init") @click.argument("path", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=".") -def init_command(path: str) -> None: +@click.option( + "--confidence", + type=click.FloatRange(0.0, 1.0), + default=None, + help=f"Intent confidence needed to generate tests in --auto mode (default {PROJECT_DEFAULTS['confidence_threshold']})", +) +@click.option( + "--output-dir", + type=str, + default=None, + help=f"Where generated tests are written (default {PROJECT_DEFAULTS['output_dir']})", +) +@click.option( + "--exclude", + type=str, + multiple=True, + help="Glob pattern of files to leave out of the analysis (repeatable)", +) +def init_command( + path: str, confidence: Optional[float], output_dir: Optional[str], exclude: Tuple[str, ...] +) -> None: """ - Initialize Code2Test in a repository. + Save project settings for 'code2test test' in .code2test/config.json. + + 'code2test test' uses these settings as its defaults: the confidence + threshold, the output directory and extra exclude patterns. Options + given to 'code2test test' still take precedence. Edit the file to + change them later. - Creates a .code2test configuration directory and detects project settings. + \b + Examples: + code2test init + code2test init --confidence 0.7 --exclude "*migrations*" """ display = DisplayManager() project_root = Path(path).resolve() + config_file = project_config_path(project_root) - display.info(f"Initializing Code2Test in: {project_root}") - - config_dir = project_root / ".code2test" - config_file = config_dir / "config.json" - - if config_file.exists(): - if not Confirm.ask("Configuration already exists. Overwrite?"): - display.info("Aborted initialization.") - return + if config_file.exists() and not Confirm.ask(f"{config_file} already exists. Overwrite?"): + display.info("Aborted initialization.") + return - config_dir.mkdir(parents=True, exist_ok=True) + if not any(project_root.rglob("*.py")): + display.warning("No Python files found; Code2Test generates tests for Python code only.") - # Auto-detect language - language = "python" # Default - framework = "pytest" - - if (project_root / "package.json").exists(): - language = "javascript" - framework = "jest" - display.info("Detected JavaScript/TypeScript project") - elif (project_root / "pom.xml").exists() or (project_root / "build.gradle").exists(): - language = "java" - framework = "junit" - display.info("Detected Java project") - else: - display.info("Detected Python project (default)") - - # User prompts - language = Prompt.ask( - "Project language", choices=["python", "javascript", "java"], default=language - ) - framework = Prompt.ask("Test framework", default=framework) - - config = { - "language": language, - "framework": framework, - "confidence_threshold": 0.6, - "auto_accept": False, - "max_concurrency": 5, - "exclude_patterns": ["node_modules", "venv", ".git", "__pycache__", "dist", "build"], + settings = { + "confidence_threshold": PROJECT_DEFAULTS["confidence_threshold"] + if confidence is None + else confidence, + "output_dir": output_dir or PROJECT_DEFAULTS["output_dir"], + "exclude_patterns": list(exclude), } + written = write_project_settings(project_root, settings) - with open(config_file, "w") as f: - json.dump(config, f, indent=2) - - display.success(f"Initialized Code2Test configuration in {config_file}") + display.success(f"Saved project settings to {written}") display.info("Run 'code2test test .' to start generating tests.") diff --git a/code2test/cli/commands/test.py b/code2test/cli/commands/test.py index 7f4e086..d647c86 100644 --- a/code2test/cli/commands/test.py +++ b/code2test/cli/commands/test.py @@ -6,14 +6,20 @@ import logging import asyncio from pathlib import Path -from typing import Optional +from typing import List, Optional, Tuple import click +from click.core import ParameterSource from rich.console import Console from code2test.cli.display import DisplayManager from code2test.cli.interactive import run_interactive_generation from code2test.cli.utils.model import resolve_model +from code2test.cli.utils.project_config import ( + PROJECT_DEFAULTS, + load_project_settings, + project_config_path, +) from code2test.core import TestGenerator, GenerationConfig, TestFramework from code2test.src.be.dependency_analyzer import DependencyGraphBuilder from code2test.src.config import Config @@ -33,8 +39,10 @@ @click.option( "--confidence", type=float, - default=0.6, - help="Minimum confidence threshold for auto-acceptance (0.0-1.0)", + default=PROJECT_DEFAULTS["confidence_threshold"], + show_default=True, + help="Intent confidence needed to generate tests in --auto mode (0.0-1.0); " + "overrides .code2test/config.json", ) @click.option( "--dry-run", @@ -43,19 +51,22 @@ help="Show what would be generated without writing files", ) @click.option( - "--output-dir", type=click.Path(), default="tests", help="Output directory for generated tests" -) -@click.option( - "--framework", - type=click.Choice(["pytest", "unittest"]), - default="pytest", - help="Test framework (v1 generates pytest tests; unittest is not supported yet)", + "--output-dir", + type=click.Path(), + default=PROJECT_DEFAULTS["output_dir"], + show_default=True, + help="Directory for generated tests, relative to the project root; " + "overrides .code2test/config.json", ) @click.option( "--include", type=str, default=None, help="Glob pattern for files to include (e.g., '*.py')" ) @click.option( - "--exclude", type=str, default=None, help="Glob pattern for files to exclude (e.g., '*test*')" + "--exclude", + type=str, + multiple=True, + help="Glob pattern of files to leave out of the analysis (repeatable); " + "added to the built-in exclusions and those in .code2test/config.json", ) @click.option( "--exit-code/--no-exit-code", @@ -69,15 +80,16 @@ help="Generate reports after test generation", ) @click.option("--verbose", "-v", is_flag=True, default=False, help="Enable verbose logging") +@click.pass_context def test_command( + ctx: click.Context, path: str, auto: bool, confidence: float, dry_run: bool, output_dir: str, - framework: str, include: Optional[str], - exclude: Optional[str], + exclude: Tuple[str, ...], exit_code: bool, report: str, verbose: bool, @@ -103,14 +115,24 @@ def test_command( else: logging.basicConfig(level=logging.WARNING) + target_path = Path(path).resolve() + repo_path = find_repo_root(target_path) + + # Project settings from `code2test init` are defaults; options on the command line win + settings = load_project_settings(repo_path) + if ctx.get_parameter_source("confidence") == ParameterSource.DEFAULT: + confidence = settings.get("confidence_threshold", confidence) + if ctx.get_parameter_source("output_dir") == ParameterSource.DEFAULT: + output_dir = settings.get("output_dir", output_dir) + if settings: + display.info(f"Using project settings from {project_config_path(repo_path)}") + # Validate confidence if not 0.0 <= confidence <= 1.0: display.error("Confidence must be between 0.0 and 1.0") sys.exit(1) main_model = resolve_model(display) - target_path = Path(path).resolve() - repo_path = find_repo_root(target_path) display.info(f"Using model: {main_model}") display.info(f"Analyzing: {target_path}") @@ -121,7 +143,7 @@ def test_command( auto_accept=auto, dry_run=dry_run, output_dir=output_dir, - framework=TestFramework.PYTEST if framework == "pytest" else TestFramework.UNITTEST, + framework=TestFramework.PYTEST, model=main_model, ) @@ -141,7 +163,13 @@ def test_command( ".env", "venv", ] - exclude_patterns = [exclude] if exclude else default_excludes + # Never analyze the generated tests themselves, wherever they are written + exclude_patterns = ( + default_excludes + + _relative_dir_patterns(output_dir, repo_path) + + settings.get("exclude_patterns", []) + + list(exclude) + ) # Analyze codebase display.info("Parsing codebase...") @@ -244,7 +272,7 @@ async def run_batch(): from code2test.core.quality import quality_report from code2test.reporting import HTMLReportGenerator, JSONReportGenerator - report_dir = Path(output_dir) / "reports" + report_dir = repo_path / output_dir / "reports" report_dir.mkdir(parents=True, exist_ok=True) intents = generator.intent_db.get_all_intents_dict() quality = quality_report( @@ -320,6 +348,18 @@ def find_repo_root(target: Path) -> Path: return start +def _relative_dir_patterns(directory: str, repo_path: Path) -> List[str]: + """The output directory as an exclude pattern relative to the project root.""" + path = Path(directory) + if path.is_absolute(): + try: + path = path.relative_to(repo_path) + except ValueError: + return [] # outside the project: never analyzed anyway + text = path.as_posix().strip("/") + return [] if text in ("", ".") else [text] + + def _is_within(file_path: str, repo_path: Path, target: Path) -> bool: """True if a component's file is the target file or inside the target directory.""" path = Path(file_path) diff --git a/code2test/cli/display.py b/code2test/cli/display.py index 47324c0..c2dace7 100644 --- a/code2test/cli/display.py +++ b/code2test/cli/display.py @@ -197,6 +197,20 @@ def show_summary_table(self, stats: Dict[str, Any]) -> None: self.console.print() self.console.print(table) + def show_session_summary( + self, components: int, files_saved: int, tests_saved: int, skipped: int + ) -> None: + """Display what an interactive session saved.""" + table = Table(title="Session Summary") + table.add_column("Metric", style="cyan") + table.add_column("Value", style="green") + table.add_row("Components", str(components)) + table.add_row("Test files saved", str(files_saved)) + table.add_row("Tests saved", str(tests_saved)) + table.add_row("Skipped", str(skipped)) + self.console.print() + self.console.print(table) + def show_test_tree(self, test_files: List[TestFile]) -> None: """ Display hierarchical test structure. diff --git a/code2test/cli/interactive.py b/code2test/cli/interactive.py index d3a44b9..2eaec79 100644 --- a/code2test/cli/interactive.py +++ b/code2test/cli/interactive.py @@ -271,21 +271,28 @@ def run_interactive_generation( async def run(): skipped = 0 generated = 0 + tests_saved = 0 for comp_id, component in components.items(): - # Extract intent - intent = generator.intent_extractor.extract_intent(component, {}) + # An intent the user edited earlier is kept; otherwise infer it from the code + stored = generator.intent_db.get_intent(comp_id) + if stored is not None and stored.user_edited: + intent = stored + else: + intent = generator.intent_extractor.extract_intent(component, {}) # Present intent if not auto_accept or intent.needs_clarification(): action = session.present_intent(intent, component.get("name", "")) if action == UserAction.SKIP: + generator.intent_db.save_intent(intent) skipped += 1 continue elif action == UserAction.EDIT_INTENT: new_text = session.edit_intent(intent) intent.update_intent(new_text) + generator.intent_db.save_intent(intent) # Generate tests test_file = await generator.test_agent.generate_unit_tests(component, intent) @@ -303,6 +310,7 @@ async def run(): continue elif action == UserAction.EDIT_INTENT: intent.update_intent(session.edit_intent(intent)) + generator.intent_db.save_intent(intent) test_file = await generator.test_agent.generate_unit_tests(component, intent) if not test_file.test_cases: display.warning(f"No tests generated for {comp_id}") @@ -323,19 +331,13 @@ async def run(): if auto_accept or session.confirm_save(test_file.path): generated += 1 + tests_saved += test_file.total_tests display.success(f"Saved {test_file.path} ({test_file.total_tests} tests)") else: (generator.repo_path / test_file.path).unlink(missing_ok=True) generator.test_registry.delete_test(test_file.path) skipped += 1 - # Show summary - display.show_summary_table( - { - "total_intents": len(components), - "total_test_files": generated, - "skipped": skipped, - } - ) + display.show_session_summary(len(components), generated, tests_saved, skipped) asyncio.run(run()) diff --git a/code2test/cli/utils/errors.py b/code2test/cli/utils/errors.py index 6bcf530..2fecfce 100644 --- a/code2test/cli/utils/errors.py +++ b/code2test/cli/utils/errors.py @@ -23,8 +23,8 @@ EXIT_FILESYSTEM_ERROR = 5 -class CodeWikiError(Exception): - """Base exception for CodeWiki CLI errors.""" +class Code2TestError(Exception): + """Base exception for Code2Test CLI errors.""" def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): self.message = message @@ -32,28 +32,28 @@ def __init__(self, message: str, exit_code: int = EXIT_GENERAL_ERROR): super().__init__(self.message) -class ConfigurationError(CodeWikiError): +class ConfigurationError(Code2TestError): """Configuration-related errors.""" def __init__(self, message: str): super().__init__(message, EXIT_CONFIG_ERROR) -class RepositoryError(CodeWikiError): +class RepositoryError(Code2TestError): """Repository-related errors.""" def __init__(self, message: str): super().__init__(message, EXIT_REPOSITORY_ERROR) -class APIError(CodeWikiError): +class APIError(Code2TestError): """LLM API-related errors.""" def __init__(self, message: str): super().__init__(message, EXIT_API_ERROR) -class FileSystemError(CodeWikiError): +class FileSystemError(Code2TestError): """File system-related errors.""" def __init__(self, message: str): @@ -71,7 +71,7 @@ def handle_error(error: Exception, verbose: bool = False) -> int: Returns: Exit code for the error """ - if isinstance(error, CodeWikiError): + if isinstance(error, Code2TestError): click.secho(f"\n✗ Error: {error.message}", fg="red", err=True) return error.exit_code else: diff --git a/code2test/cli/utils/project_config.py b/code2test/cli/utils/project_config.py new file mode 100644 index 0000000..0ba8652 --- /dev/null +++ b/code2test/cli/utils/project_config.py @@ -0,0 +1,71 @@ +""" +Per-project settings in .code2test/config.json, written by `code2test init`. + +`code2test test` uses them as defaults; options given on the command line win. +""" + +import json +from pathlib import Path +from typing import Any, Dict + +import click + +PROJECT_DEFAULTS: Dict[str, Any] = { + "confidence_threshold": 0.6, + "output_dir": "tests", + "exclude_patterns": [], +} + + +def project_config_path(repo_root: Path) -> Path: + return Path(repo_root) / ".code2test" / "config.json" + + +def load_project_settings(repo_root: Path) -> Dict[str, Any]: + """ + The settings saved for this project, or an empty dict when there are none. + + Keys other than those in PROJECT_DEFAULTS are ignored, so files written by older + versions (which also stored a language and framework) still load. + + Raises: + click.ClickException: If the file cannot be read or a setting has the wrong type + """ + path = project_config_path(repo_root) + if not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as e: + raise click.ClickException(f"Cannot read {path}: {e}") + if not isinstance(data, dict): + raise click.ClickException(f"{path} must contain a JSON object") + + settings: Dict[str, Any] = {} + if "confidence_threshold" in data: + value = data["confidence_threshold"] + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not 0.0 <= value <= 1.0 + ): + raise click.ClickException(f"{path}: confidence_threshold must be a number from 0 to 1") + settings["confidence_threshold"] = float(value) + if "output_dir" in data: + if not isinstance(data["output_dir"], str) or not data["output_dir"]: + raise click.ClickException(f"{path}: output_dir must be a non-empty string") + settings["output_dir"] = data["output_dir"] + if "exclude_patterns" in data: + value = data["exclude_patterns"] + if not isinstance(value, list) or not all(isinstance(p, str) for p in value): + raise click.ClickException(f"{path}: exclude_patterns must be a list of strings") + settings["exclude_patterns"] = value + return settings + + +def write_project_settings(repo_root: Path, settings: Dict[str, Any]) -> Path: + """Write settings to .code2test/config.json and return its path.""" + path = project_config_path(repo_root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(settings, indent=2) + "\n") + return path diff --git a/code2test/core/intent.py b/code2test/core/intent.py index 67bd52c..3a687c0 100644 --- a/code2test/core/intent.py +++ b/code2test/core/intent.py @@ -19,6 +19,24 @@ ) +_ABBREVIATIONS = ("e.g.", "i.e.", "etc.", "vs.", "cf.") + + +def first_docstring_sentence(docstring: str) -> str: + """ + The first sentence of a docstring's summary paragraph, on one line, without its final period. + + A sentence ends at '.', '!' or '?' followed by a space or the end of the paragraph, so + decimals such as 0.2 and abbreviations such as "e.g." do not end it. + """ + paragraph = " ".join(docstring.strip().split("\n\n")[0].split()) + for match in re.finditer(r"[.!?](?=\s|$)", paragraph): + if paragraph[: match.end()].lower().endswith(_ABBREVIATIONS): + continue + return paragraph[: match.start()].strip() + return paragraph.strip() + + @dataclass class IntentSignals: """Raw signals extracted from code for intent inference.""" @@ -227,8 +245,7 @@ def _generate_intent_text(self, component: Dict[str, Any], signals: IntentSignal # Start with docstring if available if signals.docstring: - # Extract first sentence - first_sentence = signals.docstring.split(".")[0].strip() + first_sentence = first_docstring_sentence(signals.docstring) if first_sentence: parts.append(first_sentence) diff --git a/code2test/reporting/json_report.py b/code2test/reporting/json_report.py index 505e11b..8da0d9e 100644 --- a/code2test/reporting/json_report.py +++ b/code2test/reporting/json_report.py @@ -6,6 +6,7 @@ import datetime from typing import Any, Dict, Optional +from code2test import __version__ from code2test.core.models import TestSuite, Intent @@ -37,7 +38,7 @@ def generate_report( "metadata": { "timestamp": datetime.datetime.now().isoformat(), "tool": "code2test", - "version": "0.1.0", + "version": __version__, }, "summary": { "module_path": test_suite.module_path, @@ -80,7 +81,7 @@ def generate_quality_report(self, quality: Dict[str, Any], output_path: str) -> "metadata": { "timestamp": datetime.datetime.now().isoformat(), "tool": "code2test", - "version": "0.1.0", + "version": __version__, }, **quality, } diff --git a/code2test/src/__init__.py b/code2test/src/__init__.py index cc5a3b9..632315e 100644 --- a/code2test/src/__init__.py +++ b/code2test/src/__init__.py @@ -1 +1 @@ -"""CodeWiki backend and frontend modules.""" +"""Repository analysis (dependency graph), derived from CodeWiki.""" diff --git a/code2test/src/be/__init__.py b/code2test/src/be/__init__.py index f85bf77..6057383 100644 --- a/code2test/src/be/__init__.py +++ b/code2test/src/be/__init__.py @@ -1 +1 @@ -"""CodeWiki backend modules for documentation generation.""" +"""Dependency analysis of a repository, derived from CodeWiki.""" diff --git a/code2test/src/be/dependency_analyzer/utils/logging_config.py b/code2test/src/be/dependency_analyzer/utils/logging_config.py index 78b305d..a7f1cd5 100644 --- a/code2test/src/be/dependency_analyzer/utils/logging_config.py +++ b/code2test/src/be/dependency_analyzer/utils/logging_config.py @@ -1,5 +1,5 @@ """ -Colored logging configuration for CodeWiki. +Colored logging configuration for the dependency analyzer. This module provides a custom logging formatter with colored output for better readability. diff --git a/code2test/src/config.py b/code2test/src/config.py index 5f90382..a799de7 100644 --- a/code2test/src/config.py +++ b/code2test/src/config.py @@ -38,8 +38,7 @@ def is_cli_context() -> bool: # LLM services -# In CLI mode, these will be loaded from ~/.codewiki/config.json + keyring -# In web app mode, use environment variables +# Inherited from CodeWiki; the dependency analysis does not use them MAIN_MODEL = os.getenv("MAIN_MODEL", "claude-sonnet-4") FALLBACK_MODEL_1 = os.getenv("FALLBACK_MODEL_1", "glm-4p5") CLUSTER_MODEL = os.getenv("CLUSTER_MODEL", MAIN_MODEL) @@ -49,7 +48,7 @@ def is_cli_context() -> bool: @dataclass class Config: - """Configuration class for CodeWiki.""" + """Configuration for the dependency analysis.""" repo_path: str output_dir: str diff --git a/docs/quickstart.md b/docs/quickstart.md index 1b667be..a036045 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -7,12 +7,12 @@ This guide takes you from installation to a verified, generated test suite for y Code2Test needs Python 3.12 or later. ```bash -git clone https://github.com/Cirkitly/code2test.git -cd code2test -pip install . +pip install code2test code2test --version ``` +If the package index is not available to you, install the release from GitHub: `pip install git+https://github.com/Cirkitly/code2test@v0.1.0`. + ## 2. Choose a model Set an API key and, optionally, the model. Any [pydantic-ai model string](https://ai.pydantic.dev/models/) works: diff --git a/docs/task.md b/docs/task.md index 92c55d4..0509aca 100644 --- a/docs/task.md +++ b/docs/task.md @@ -28,7 +28,7 @@ Python only. Test generation, verification and repair target pytest. Files in ot - [x] `benchmark`: mutation score and diagnosis accuracy on failures of known cause - [x] `report summary`: intents by confidence - [x] `report coverage` and the `--report` output: behavioral coverage (behaviors tested, mutants caught) and diagnostic accuracy per component -- [x] `init`: writes a starter `.code2test/config.json` +- [x] `init`: saves project defaults for `test` (confidence threshold, output directory, exclude patterns) in `.code2test/config.json` - [x] `config set|show|validate`: API key and model; `validate` sends the model one test request **Measurement and evaluation** @@ -38,18 +38,17 @@ Python only. Test generation, verification and repair target pytest. Files in ot **Project** - [x] CI on every push and pull request: tests on Python 3.12 and 3.13, ruff lint and format check - [x] CI examples for GitHub Actions and GitLab CI ([examples/ci](../examples/ci)) +- [x] Release 0.1.0: [CHANGELOG](../CHANGELOG.md), PyPI publishing from GitHub releases (`.github/workflows/publish.yml`) ## In the code but not used by the v1 pipeline - `code2test/adapters/` (pytest, Jest and JUnit adapters) and `code2test/core/templates.py`: the pipeline generates tests with the test agent and runs them with `core/verifier.py` instead. - `code2test/core/intent_analyzers/` for JavaScript and Java: never reached, because `code2test test` passes only Python components to the pipeline. - `TestAgent.generate_integration_tests`: not called by any command. -- `--framework unittest`: accepted, but tests are generated in pytest style. ## Not done - [ ] Markdown run reports -- [ ] Settings from `.code2test/config.json` (written by `init`, not read by other commands) - [ ] Incremental generation for changed files only (re-runs reuse verified tests per component, whether or not the code changed) - [ ] An experiment on code with known bugs, to test whether intent-first generation avoids locking bugs into the suite (see the paper's Discussion) diff --git a/examples/ci/github_actions.yml b/examples/ci/github_actions.yml index e48ba3d..7de1f9e 100644 --- a/examples/ci/github_actions.yml +++ b/examples/ci/github_actions.yml @@ -26,7 +26,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e . - pip install git+https://github.com/Cirkitly/code2test.git + pip install code2test - name: Generate tests # --exit-code fails the job if any written test file failed verification diff --git a/examples/ci/gitlab_ci.yml b/examples/ci/gitlab_ci.yml index cf22dfc..4d5fa6d 100644 --- a/examples/ci/gitlab_ci.yml +++ b/examples/ci/gitlab_ci.yml @@ -19,7 +19,7 @@ default: before_script: - pip install --upgrade pip - pip install -e . - - pip install git+https://github.com/Cirkitly/code2test.git + - pip install code2test generate_tests: stage: generate diff --git a/pyproject.toml b/pyproject.toml index 883120d..19a70a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code2test" -version = "0.1.0" +dynamic = ["version"] description = "Intent-first, execution-verified pytest generation for legacy Python code" readme = "README.md" requires-python = ">=3.12" @@ -89,6 +89,9 @@ packages = [ "code2test.src.be.dependency_analyzer.utils", ] +[tool.setuptools.dynamic] +version = {attr = "code2test.__version__"} + [tool.setuptools.package-data] code2test = ["templates/**/*", "py.typed"] diff --git a/tests/test_cli.py b/tests/test_cli.py index bbdc447..6ee42ba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -126,6 +126,22 @@ def test_interactive_mode_saves_accepted_tests(repo, monkeypatch): assert (repo / GENERATED).exists() +def test_interactive_edit_is_stored_and_reused(repo, monkeypatch): + use_model(monkeypatch, scripted([tests_reply(PASSING)])) + + # confirm batch, edit intent, type it, accept tests, confirm save + result = run("test", ".", input="y\ne\nAdds two integers, also negative ones\ny\ny\n") + assert result.exit_code == 0, result.output + assert "Tests saved" in result.output and "│ 1 " in result.output.split("Tests saved")[1][:20] + + shown = run("intent", "show", "add") + assert "Adds two integers, also negative ones" in shown.output + + # The next session starts from the edited intent instead of re-inferring it + result = run("test", ".", input="y\nn\n") + assert "Adds two integers, also negative ones" in result.output + + def test_interactive_mode_declined_save_leaves_no_file(repo, monkeypatch): use_model(monkeypatch, scripted([tests_reply(PASSING)])) @@ -214,6 +230,85 @@ def test_report_coverage_in_text_json_and_html(repo, monkeypatch): assert "Test Quality" in (repo / "cov.html").read_text() +# Project settings from `code2test init` (.code2test/config.json) + + +def _metrics(repo): + return json.loads((repo / ".code2test" / "metrics.json").read_text())["summary"] + + +def test_init_writes_only_the_settings_test_uses(repo): + result = run("init", "--confidence", "0.7", "--exclude", "*migrations*", ".") + assert result.exit_code == 0, result.output + config = json.loads((repo / ".code2test" / "config.json").read_text()) + assert config == { + "confidence_threshold": 0.7, + "output_dir": "tests", + "exclude_patterns": ["*migrations*"], + } + + # An existing file is kept unless the user agrees to overwrite it + assert run("init", "--confidence", "0.2", ".", input="n\n").exit_code == 0 + assert ( + json.loads((repo / ".code2test" / "config.json").read_text())["confidence_threshold"] == 0.7 + ) + + +def test_test_takes_confidence_and_output_dir_from_project_settings(repo, monkeypatch): + use_model(monkeypatch, scripted([tests_reply(PASSING)])) + assert run("init", "--confidence", "0.95", "--output-dir", "generated", ".").exit_code == 0 + + # The scripted intent has confidence 0.9: below the saved threshold, so nothing is generated + assert run("test", "--auto", ".").exit_code == 0 + assert _metrics(repo)["skipped_low_confidence"] == 1 + assert not (repo / "generated").exists() + + # Options on the command line win over the saved settings + assert run("test", "--auto", "--confidence", "0", "--output-dir", "other", ".").exit_code == 0 + assert (repo / "other" / "calc" / "test_ops_add.py").exists() + assert not (repo / "generated").exists() + + +def test_exclude_patterns_add_to_the_built_in_exclusions(repo, monkeypatch): + (repo / "calc" / "legacy.py").write_text("def old(x):\n return x\n") + use_model(monkeypatch, scripted([tests_reply(PASSING)])) + assert run("init", "--exclude", "*legacy*", "--output-dir", "generated", ".").exit_code == 0 + + assert run("test", "--auto", "--confidence", "0", ".").exit_code == 0 + assert _metrics(repo)["components"] == 1 # only calc.ops.add: legacy.py is excluded + assert (repo / "generated" / "calc" / "test_ops_add.py").exists() # saved output_dir + + # --exclude adds to the built-in exclusions (it used to replace them, so the + # generated test files were analyzed as project code on the next run) + assert run("test", "--auto", "--confidence", "0", "--exclude", "*nothing*", ".").exit_code == 0 + assert _metrics(repo)["components"] == 1 + + +def test_output_dir_becomes_an_exclude_pattern(tmp_path): + patterns = test_cmd_module._relative_dir_patterns + assert patterns("generated/", tmp_path) == ["generated"] + assert patterns("./out/tests", tmp_path) == ["out/tests"] + assert patterns(str(tmp_path / "gen"), tmp_path) == ["gen"] + assert patterns(".", tmp_path) == [] and patterns("/elsewhere", tmp_path) == [] + + +def test_older_or_invalid_project_settings(repo, monkeypatch): + use_model(monkeypatch, scripted([tests_reply(PASSING)])) + config = repo / ".code2test" / "config.json" + config.parent.mkdir() + + # Files from older versions also stored a language and framework; those keys are ignored + config.write_text( + json.dumps({"language": "python", "framework": "pytest", "confidence_threshold": 0.0}) + ) + assert run("test", "--auto", ".").exit_code == 0 + assert (repo / GENERATED).exists() + + config.write_text(json.dumps({"confidence_threshold": "high"})) + result = CliRunner().invoke(cli, ["test", "--auto", "."]) + assert result.exit_code == 1 and "confidence_threshold must be a number" in result.output + + class _Display: def __init__(self): self.messages = [] diff --git a/tests/test_intent.py b/tests/test_intent.py index 611aa40..c05029d 100644 --- a/tests/test_intent.py +++ b/tests/test_intent.py @@ -2,7 +2,7 @@ import pytest -from code2test.core.intent import IntentExtractor +from code2test.core.intent import IntentExtractor, first_docstring_sentence from code2test.core.intent_analyzers import JavaAnalyzer, JavascriptAnalyzer, PythonAnalyzer TYPED = '''def add(a: int, b: int) -> int: @@ -159,3 +159,26 @@ def test_clarification_questions_cover_missing_evidence(extractor): assert any("primary purpose" in q for q in questions) assert any("types of inputs" in q for q in questions) assert any("complex logic" in q for q in questions) + + +@pytest.mark.parametrize( + "docstring, expected", + [ + ("Return the sum of a and b.", "Return the sum of a and b"), + # Decimals and abbreviations do not end the sentence + ( + "Return the total plus tax at rate (0.2 means 20%), rounded.", + "Return the total plus tax at rate (0.2 means 20%), rounded", + ), + ("Format an amount, e.g. '12.50 USD'. More.", "Format an amount, e.g. '12.50 USD'"), + # Wrapped lines are joined; later paragraphs are ignored + ( + "Add rules for plurals and singular\nfor irregular words.\n\n:param word: x", + "Add rules for plurals and singular for irregular words", + ), + ("Is it valid? More text.", "Is it valid"), + ("No period at all", "No period at all"), + ], +) +def test_first_docstring_sentence(docstring, expected): + assert first_docstring_sentence(docstring) == expected