From 09a08663d68e4d61ea66a4da319aa1d42490e427 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Sun, 6 Sep 2026 13:42:19 -0400 Subject: [PATCH] fix(scope): read the primordial scope from jrt:/; ship a PyPI wheel with a bundled JVM WALA's primordial scope was loaded by walking $JAVA_HOME/jmods, which tied the analyzer to a full JDK with jmods and a JAVA_HOME. Unset, level 2 silently fell back to declared-only edges and the L3/L4 WALA engine was unavailable. The scope now comes from the JVM running the analysis, through the jrt:/ filesystem (lib/modules), so any runtime works, including jlink'd ones. WALA's own JrtModule cannot be used: it closes its Files.list stream inside try-with-resources before returning it, and iterating fails with "source already consumed or closed" (still true in 1.8.0). A small eager JrtModule under com.ibm.cldk.utils replaces it. packaging/python builds a pure wheel, codeanalyzer-java, with the fat jar under _bin/ and jdk4py (Temurin 21) as the runtime, installing a canjv launcher. jdk4py is pinned below 22 because Shrike 1.6.7 rejects class files above version 67. release.yml verifies the tag against gradle.properties, builds the wheel after the jar, and publishes it to PyPI via Trusted Publishing, so jar and wheel stay in lockstep. Bump to 3.0.2. Verified on daytrader8 under jdk4py 21 with JAVA_HOME unset: -a 2 edge set equal to the jmods baseline (1876 edges), -a 3 --l3-engine wala byte-identical. Closes #222 --- .github/workflows/release.yml | 43 ++++++- README.md | 9 +- gradle.properties | 2 +- packaging/python/.gitignore | 2 + packaging/python/build_wheel.sh | 57 +++++++++ packaging/python/pyproject.toml | 57 +++++++++ .../python/src/codeanalyzer_java/__init__.py | 52 ++++++++ .../python/src/codeanalyzer_java/__main__.py | 27 ++++ .../src/codeanalyzer_java/_bin/.gitignore | 5 + .../java/com/ibm/cldk/utils/JrtModule.java | 118 ++++++++++++++++++ .../java/com/ibm/cldk/utils/ScopeUtils.java | 28 ++--- 11 files changed, 380 insertions(+), 20 deletions(-) create mode 100644 packaging/python/.gitignore create mode 100755 packaging/python/build_wheel.sh create mode 100644 packaging/python/pyproject.toml create mode 100644 packaging/python/src/codeanalyzer_java/__init__.py create mode 100644 packaging/python/src/codeanalyzer_java/__main__.py create mode 100644 packaging/python/src/codeanalyzer_java/_bin/.gitignore create mode 100644 src/main/java/com/ibm/cldk/utils/JrtModule.java diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49e4f3ff..969b8405 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,8 @@ on: - "v*.*.*" permissions: - contents: write + contents: write + id-token: write # PyPI Trusted Publishing (OIDC) -- no API token needed jobs: release: @@ -40,9 +41,18 @@ jobs: git push --delete origin ${GITHUB_REF#refs/tags/} exit 1 # Fail the workflow - - name: Resolve version + # The tag must match gradle.properties: it names the jar, the wheel, and the GitHub Release, + # and a mismatch would publish the wrong version to PyPI. + - name: Resolve and verify version id: ver - run: echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT" + run: | + version="${GITHUB_REF#refs/tags/v}" + declared="$(sed -n 's/^version=//p' gradle.properties)" + if [[ "$version" != "$declared" ]]; then + echo "::error::Tag version ($version) != gradle.properties version ($declared). Bump gradle.properties or fix the tag." + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" # Stage release assets: a stable-named codeanalyzer.jar (what the installer fetches), the # Neo4j schema contract (platform-independent, version-locked to this build), and the @@ -56,6 +66,27 @@ jobs: cp packaging/install/codeanalyzer-installer.sh release-assets/codeanalyzer-installer.sh ls -lh release-assets + # The PyPI package: the same jar plus a `canjv` launcher, on a jdk4py runtime (see + # packaging/python). One pure wheel; the JVM comes from jdk4py's own platform wheels. + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build the Python wheel + env: + PKG_VERSION: ${{ steps.ver.outputs.version }} + run: | + python -m pip install --upgrade build hatchling + ./packaging/python/build_wheel.sh + + - name: Publish wheel to PyPI (Trusted Publishing / OIDC) + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: packaging/python/dist + # Re-runs stay idempotent: a version already on PyPI is skipped, not a 400. + skip-existing: true + - name: Build Changelog id: gen_changelog uses: mikepenz/release-changelog-builder-action@v6 @@ -73,6 +104,12 @@ jobs: echo "## Install" echo echo '```sh' + echo "pip install codeanalyzer-java==${{ steps.ver.outputs.version }} # bundles a JVM; installs the canjv launcher" + echo '```' + echo + echo "Or the jar with a \`codeanalyzer\` launcher (requires Java 11+):" + echo + echo '```sh' echo "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/${GITHUB_REPOSITORY}/releases/download/v${{ steps.ver.outputs.version }}/codeanalyzer-installer.sh | sh" echo '```' echo diff --git a/README.md b/README.md index e1d7152d..933b7eb9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,14 @@ and emits them either as the canonical `analysis.json`, or as a **Neo4j property ## Quick install -Grab the latest release jar and a `codeanalyzer` launcher (requires a Java 11+ runtime): +From PyPI, with a bundled JVM (no system Java needed). Installs a `canjv` launcher: + +```sh +pip install codeanalyzer-java +canjv -i /path/to/project -a 2 -o ./out +``` + +Or grab the latest release jar and a `codeanalyzer` launcher (requires a Java 11+ runtime): ```sh curl --proto '=https' --tlsv1.2 -LsSf https://github.com/codellm-devkit/codeanalyzer-java/releases/latest/download/codeanalyzer-installer.sh | sh diff --git a/gradle.properties b/gradle.properties index bad42edb..c9517d1a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=3.0.1 +version=3.0.2 diff --git a/packaging/python/.gitignore b/packaging/python/.gitignore new file mode 100644 index 00000000..864489f3 --- /dev/null +++ b/packaging/python/.gitignore @@ -0,0 +1,2 @@ +README.md +dist/ diff --git a/packaging/python/build_wheel.sh b/packaging/python/build_wheel.sh new file mode 100755 index 00000000..824d5482 --- /dev/null +++ b/packaging/python/build_wheel.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Build the codeanalyzer-java wheel: the fat JAR under _bin/ plus a `canjv` launcher that runs it +# on the jdk4py runtime. The JAR is platform-independent and the JVM comes from jdk4py's own +# platform wheels, so exactly one py3-none-any wheel is produced. +# +# Requirements on the build host: +# - the fat JAR already built: ./gradlew fatJar (build/libs/codeanalyzer-.jar) +# - python -m pip install build hatchling +# +# Usage: +# PKG_VERSION=3.0.2 ./build_wheel.sh # writes ./dist/codeanalyzer_java-3.0.2-py3-none-any.whl +# twine upload dist/*.whl # publish +# +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$HERE/../.." && pwd)" +PKG_VERSION="${PKG_VERSION:-$(sed -n 's/^version=//p' "$REPO_ROOT/gradle.properties")}" +JAR="${JAR:-$REPO_ROOT/build/libs/codeanalyzer-${PKG_VERSION}.jar}" +BIN_DIR="$HERE/src/codeanalyzer_java/_bin" +INIT_PY="$HERE/src/codeanalyzer_java/__init__.py" + +[[ -f "$JAR" ]] || { echo "error: $JAR not found; run ./gradlew fatJar first" >&2; exit 1; } + +# Stamp $PKG_VERSION into __init__.py for the build and restore it on exit; likewise remove the +# copied JAR and README, so the working tree stays pristine. +ORIG_INIT="$(cat "$INIT_PY")" +cleanup() { + printf '%s\n' "$ORIG_INIT" > "$INIT_PY" + find "$BIN_DIR" -mindepth 1 ! -name '.gitignore' -delete + rm -f "$HERE/README.md" +} +trap cleanup EXIT +python - "$INIT_PY" "$PKG_VERSION" <<'PY' +import re, sys +path, version = sys.argv[1], sys.argv[2] +text = open(path).read() +new, n = re.subn(r'__version__ = "[^"]*"', f'__version__ = "{version}"', text) +if n != 1: + raise SystemExit(f"expected exactly one __version__ assignment in {path}, found {n}") +open(path, "w").write(new) +print(f">>> stamped __version__ = {version}") +PY + +rm -rf "$HERE/dist" +mkdir -p "$HERE/dist" "$BIN_DIR" +cp "$JAR" "$BIN_DIR/codeanalyzer.jar" +cp "$REPO_ROOT/README.md" "$HERE/README.md" + +python -m build --wheel --no-isolation -o "$HERE/dist" "$HERE" + +echo +echo ">>> Built wheel:" +ls -lh "$HERE/dist"/*.whl +echo +echo "Publish with: twine upload $HERE/dist/*.whl" diff --git a/packaging/python/pyproject.toml b/packaging/python/pyproject.toml new file mode 100644 index 00000000..dd561855 --- /dev/null +++ b/packaging/python/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "codeanalyzer-java" +dynamic = ["version"] +description = "codeanalyzer-java backend for CLDK (codellm-devkit): the analyzer JAR plus a bundled JVM." +# The PyPI long description is the repo root README, copied in by build_wheel.sh at build time +# (single source of truth; the copy is gitignored). +readme = "README.md" +requires-python = ">=3.8" +license = { text = "Apache-2.0" } +authors = [ + { name = "Rahul Krishna", email = "i.m.ralk@gmail.com" }, +] +keywords = ["cldk", "codellm-devkit", "java", "code analysis", "wala"] +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Java", +] +# jdk4py ships a jlink'd Temurin runtime per platform, so this wheel is pure (py3-none-any) and +# pip picks the right JVM. Pinned below 22: WALA 1.6.x's Shrike reads class files up to version 67 +# (Java 23), and jdk4py 25 ships version-69 class files that it rejects. Bump this with WALA. +dependencies = ["jdk4py>=21,<22"] + +[project.urls] +Homepage = "https://codellm-devkit.info" +Repository = "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/codellm-devkit/codeanalyzer-java" +Issues = "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/codellm-devkit/codeanalyzer-java/issues" + +# `canjv` follows the sibling analyzers (canpy, cants, caniac). There is deliberately no +# `codeanalyzer` script: codeanalyzer-python already installs one as a deprecated alias, and two +# packages claiming the same console script silently overwrite each other in one environment. +[project.scripts] +canjv = "codeanalyzer_java.__main__:main" + +# Single source of truth for the package version: __init__.py's __version__. +# build_wheel.sh overwrites it from $PKG_VERSION (set from gradle.properties in CI), so a +# release of vX.Y.Z always produces a codeanalyzer-java==X.Y.Z wheel. +[tool.hatch.version] +path = "src/codeanalyzer_java/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/codeanalyzer_java"] +artifacts = ["src/codeanalyzer_java/_bin/*"] + +[tool.hatch.build.targets.sdist] +# No meaningful sdist: the JAR needs Gradle + a JDK to build. Ship the wheel only. +include = [ + "src/codeanalyzer_java/__init__.py", + "README.md", + "build_wheel.sh", +] diff --git a/packaging/python/src/codeanalyzer_java/__init__.py b/packaging/python/src/codeanalyzer_java/__init__.py new file mode 100644 index 00000000..38018038 --- /dev/null +++ b/packaging/python/src/codeanalyzer_java/__init__.py @@ -0,0 +1,52 @@ +"""Prebuilt ``codeanalyzer-java`` backend for CLDK, with a bundled JVM. + +This package carries the self-contained analyzer JAR and depends on ``jdk4py`` for a +Temurin runtime, so ``pip install codeanalyzer-java`` needs no system Java. CLDK's Python +SDK depends on this package and calls :func:`command` to run the analyzer, exactly as it +imports ``codeanalyzer-python`` and ``codeanalyzer-typescript`` for the other backends. + +The analyzer reads its Java standard-library scope from the JVM it runs on (``jrt:/``), +so no ``JAVA_HOME`` and no ``jmods/`` are required. +""" + +from __future__ import annotations + +from importlib import resources +from pathlib import Path +from typing import List + +__version__ = "0.0.0" + +__all__ = ["jar_path", "java_path", "command", "__version__"] + +_JAR_NAME = "codeanalyzer.jar" + + +def jar_path() -> Path: + """Return the absolute path to the bundled ``codeanalyzer.jar``. + + Raises: + FileNotFoundError: if the wheel did not include the JAR (a source/dev install + with no build run). + """ + resource = resources.files("codeanalyzer_java") / "_bin" / _JAR_NAME + with resources.as_file(resource) as extracted: + path = Path(extracted) + if not path.exists(): + raise FileNotFoundError( + f"Bundled {_JAR_NAME} not found at {path}. Build it with `./gradlew fatJar` and " + "copy it there, or install the published wheel." + ) + return path + + +def java_path() -> Path: + """Return the ``java`` executable of the bundled runtime.""" + import jdk4py + + return Path(jdk4py.JAVA) + + +def command(*args: str) -> List[str]: + """Return ``[java, -jar, codeanalyzer.jar, *args]`` ready for ``subprocess``.""" + return [str(java_path()), "-jar", str(jar_path()), *args] diff --git a/packaging/python/src/codeanalyzer_java/__main__.py b/packaging/python/src/codeanalyzer_java/__main__.py new file mode 100644 index 00000000..2ff950d5 --- /dev/null +++ b/packaging/python/src/codeanalyzer_java/__main__.py @@ -0,0 +1,27 @@ +"""Console-script entry point: run the bundled analyzer on the bundled JVM. + +``pip install codeanalyzer-java`` installs a ``canjv`` launcher (see ``[project.scripts]`` +in ``pyproject.toml``) that calls :func:`main`, passing through every CLI argument and the +exit code unchanged. +""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from . import command + + +def main() -> "int | None": + argv = command(*sys.argv[1:]) + if os.name == "posix": + # Replace this process with the JVM: no extra Python process lingers, and + # signals/exit codes are handled by the analyzer directly. + os.execv(argv[0], argv) + return subprocess.call(argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packaging/python/src/codeanalyzer_java/_bin/.gitignore b/packaging/python/src/codeanalyzer_java/_bin/.gitignore new file mode 100644 index 00000000..f61e4d4d --- /dev/null +++ b/packaging/python/src/codeanalyzer_java/_bin/.gitignore @@ -0,0 +1,5 @@ +# The fat JAR is copied here at wheel-build time (see ../../build_wheel.sh) and +# bundled via `artifacts` in pyproject.toml. Never commit it. Keep the directory +# present so the package is importable in dev. +* +!.gitignore diff --git a/src/main/java/com/ibm/cldk/utils/JrtModule.java b/src/main/java/com/ibm/cldk/utils/JrtModule.java new file mode 100644 index 00000000..2e3a9014 --- /dev/null +++ b/src/main/java/com/ibm/cldk/utils/JrtModule.java @@ -0,0 +1,118 @@ +package com.ibm.cldk.utils; + +import com.ibm.wala.classLoader.Module; +import com.ibm.wala.classLoader.ModuleEntry; +import com.ibm.wala.core.util.shrike.ShrikeClassReaderHandle; +import com.ibm.wala.core.util.strings.ImmutableByteArray; +import com.ibm.wala.shrike.shrikeCT.InvalidClassFileException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A WALA {@link Module} over one named module of the running JVM, read through the {@code jrt:/} + * filesystem ({@code lib/modules}). + * + *

This exists because WALA's own {@code com.ibm.wala.core.java11.JrtModule} (still in 1.8.0) + * returns a lazy stream from inside a try-with-resources that closes it first, so iterating the + * entries fails with {@code IllegalStateException: source already consumed or closed}. Here the + * walk is done eagerly in the constructor; entries are held as paths and read on demand. + * + *

Reading the primordial scope from the JVM that is executing the analysis (rather than from + * {@code $JAVA_HOME/jmods}) means any runtime works, including jlink'd ones that ship no jmods. + */ +public final class JrtModule implements Module { + + private final Path root; + private final List files; + + public JrtModule(String module) throws IOException { + root = FileSystems.getFileSystem(URI.create("jrt:/")).getPath("modules", module); + try (Stream walk = Files.walk(root)) { + files = walk.filter(Files::isRegularFile).collect(Collectors.toList()); + } + } + + @Override + public String toString() { + return "[module " + root + "]"; + } + + @Override + public Iterator getEntries() { + return files.stream().map(this::entry).iterator(); + } + + private ModuleEntry entry(Path path) { + Module container = this; + String name = root.relativize(path).toString(); + return new ModuleEntry() { + private String className; + + @Override + public String getName() { + return name; + } + + @Override + public boolean isClassFile() { + return name.endsWith(".class"); + } + + @Override + public boolean isSourceFile() { + return name.endsWith(".java"); + } + + @Override + public InputStream getInputStream() { + try { + return new ByteArrayInputStream(Files.readAllBytes(path)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public boolean isModuleFile() { + return false; + } + + @Override + public Module asModule() { + throw new UnsupportedOperationException(name + " is not a module"); + } + + @Override + public String getClassName() { + if (className == null) { + try { + className = ImmutableByteArray.make(new ShrikeClassReaderHandle(this).get().getName()).toString(); + } catch (InvalidClassFileException e) { + throw new IllegalStateException("Invalid class file " + name, e); + } + } + return className; + } + + @Override + public Module getContainer() { + return container; + } + + @Override + public String toString() { + return "[" + (isClassFile() ? "class" : "file") + " " + name + "]"; + } + }; + } +} diff --git a/src/main/java/com/ibm/cldk/utils/ScopeUtils.java b/src/main/java/com/ibm/cldk/utils/ScopeUtils.java index ed0476db..d2dabb6a 100644 --- a/src/main/java/com/ibm/cldk/utils/ScopeUtils.java +++ b/src/main/java/com/ibm/cldk/utils/ScopeUtils.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -68,22 +69,19 @@ public static AnalysisScope createScope(String projectPath, String applicationDe AnalysisScope scope = new JavaSourceAnalysisScope(); addDefaultExclusions(scope); - Log.info("Loading Java SE standard libs."); - - if (System.getenv("JAVA_HOME") == null) { - Log.error("JAVA_HOME is not set."); - throw new RuntimeException("JAVA_HOME is not set."); - } - - String[] stdlibs = Files.walk(Paths.get(System.getenv("JAVA_HOME"), "jmods")) - .filter(path -> path.toString().endsWith(".jmod")) - .map(path -> path.toAbsolutePath().toString()) - .toArray(String[]::new); - - for (String stdlib : stdlibs) { - scope.addToScope(ClassLoaderReference.Primordial, new JarFile(stdlib)); + // The primordial scope comes from the JVM we are running on, read through the jrt:/ filesystem + // (lib/modules). That works on any runtime, including jlink'd ones that ship no jmods/, and it + // never disagrees with the JVM actually executing the analysis, unlike a $JAVA_HOME lookup. + Log.info("Loading Java SE standard libs from the running JVM (jrt:/)."); + List stdlibs = new ArrayList<>(); + for (Module module : ModuleLayer.boot().modules()) { + String name = module.getName(); + if (name.startsWith("java.") || name.startsWith("jdk.")) { + scope.addToScope(ClassLoaderReference.Primordial, new JrtModule(name)); + stdlibs.add(name); + } } - setStdLibs(stdlibs); + setStdLibs(stdlibs.toArray(String[]::new)); // Build the application classes first: their names are needed to keep a dependency jar from // shadowing them (see below), so this must precede adding the dependency jars.