diff --git a/docs/design/specs/schema-v2-l3-l4-design.md b/docs/design/specs/schema-v2-l3-l4-design.md index 5cde382f..33f0b0b5 100644 --- a/docs/design/specs/schema-v2-l3-l4-design.md +++ b/docs/design/specs/schema-v2-l3-l4-design.md @@ -152,6 +152,20 @@ Global/static state modeled as **extra** formal/actual vertices (rides the same **Cost controls:** flag-gated (nothing at L4 runs unless `-a 4`); k-limiting mandatory for termination; summaries content-hashed/cached with recorded dependency metadata (incremental re-analysis aspirational); parallel-by-construction wavefront over the SCC DAG, `-j N` byte-identical to `-j 1`. +### 9a. Implementation delta + +What the L4 branch actually shipped, against the sketch above. Output conforms; the construction does not, and two vertex families were deferred. + +**Deferred (not built):** +- **Global/static state as extra formal/actual vertices.** No such vertices are emitted. Static-field *flow* is not lost: `L4WalaOverlays` maps a `StaticFieldKey` heap location to the field name, so static-mediated dependence still rides the `prov:["points-to"]` ddg — it is just not addressable as a vertex a consumer can enumerate. +- **`formal_out` for by-ref parameters.** `SdgVertices` emits exactly one `@formal_out` per callable, for `$ret`, and only when the callable returns a value. Java has no by-ref parameters; mutation-through-a-reference-argument therefore has no dedicated vertex and shows up (if at all) as points-to ddg. + +**Built differently (engine deviations):** +- **`param_in`/`param_out` are derived, not sliced.** They come straight from the v2 tree — body `call` nodes with L2-backfilled `callee`, wired to the callee's parameter list — rather than from a WALA `SDG` pruned with `GraphSlicer.prune`. The edges are structurally determined by the call graph alone, so the derived result is identical and byte-deterministic, and it avoids re-opening the whole-program ModRef closure that OOMs at 4 GB over a JDK-inclusive call graph (`WalaAnalysis.emptyDefaultingMap`). Recorded in `docs/design/plans/2026-08-27-l4-sdg.md`. +- **Semantic ddg comes from per-method PDGs, not a whole-program SDG.** `L4WalaOverlays` primes mod/ref restricted to application-scope CG nodes (the bounded answer to that same OOM), re-runs `WalaPdgBuilder` per application method, and projects caller-side `HeapStatement`s onto their call's own `NormalStatement` so the interprocedural round trip lands on real body nodes. Consequence: library-mediated heap flow is conservatively absent. + +**No `SDG` object is ever constructed** — a reader looking for one will not find it. + --- ## 10. CLI contract diff --git a/schema.neo4j.json b/schema.neo4j.json index 338c2b23..236d7f25 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -1,5 +1,5 @@ { - "schema_version": "2.0.0", + "schema_version": "2.1.0", "generator": "codeanalyzer-java", "marker_labels": [ "JEntrypoint" @@ -156,6 +156,8 @@ "argument_types": "string[]", "argument_expr": "string[]", "_module": "string", + "var": "string", + "call_node": "string", "start_line": "integer", "end_line": "integer" } diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index 1ab609c2..080201cb 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -31,6 +31,8 @@ import com.ibm.cldk.syntactic_analysis.L1Cache; import com.ibm.cldk.syntactic_analysis.L1Extractor; import com.ibm.cldk.syntactic_analysis.L2CallGraph; +import com.ibm.cldk.syntactic_analysis.dataflow.SdgVertices; +import com.ibm.cldk.syntactic_analysis.dataflow.SummaryPass; import com.ibm.cldk.utils.BuildProject; import com.ibm.cldk.utils.Log; import com.ibm.cldk.wala.WalaAnalysis; @@ -97,7 +99,7 @@ public class CodeAnalyzer implements Runnable { public static String projectRootPom; @Option(names = { "-a", - "--analysis-level" }, description = "Level of analysis to perform. Options: 1 (for just symbol table); 2 (for call graph); 3 (for intraprocedural dataflow: cfg/cdg/ddg). Default: 1") + "--analysis-level" }, description = "Level of analysis to perform. Options: 1 (for just symbol table); 2 (for call graph); 3 (for intraprocedural dataflow: cfg/cdg/ddg); 4 (adds the interprocedural SDG). Default: 1") public static int analysisLevel = 1; @Option(names = { "--include-test-classes" }, hidden = true, description = "Print logs to console.") @@ -144,7 +146,8 @@ public class CodeAnalyzer implements Runnable { @Option(names = { "--no-rta" }, description = "Skip the WALA RTA overlay at --schema v2 --analysis-level 2, " - + "emitting declared-only call edges without building the application.") + + "emitting declared-only call edges without building the application. Does not " + + "suppress the L4 WALA build at --analysis-level 4: the semantic ddg still needs it.") private boolean noRta = false; @Option(names = { @@ -159,6 +162,11 @@ public class CodeAnalyzer implements Runnable { + "class files — uses WALA RTA + PDG for cfg/cdg/ddg).") private String l3Engine = "ast"; + @Option(names = { + "--precision" }, description = "L4 points-to precision: rta (default; reuses the L2 " + + "call-graph build) | 0-cfa | 0-1-cfa (rebuild the call graph for L4).") + public static String precision = "rta"; + @Option(names = { "--graph-field-depth" }, description = "DDG access-path bound k at --analysis-level 3 (default 3).") private int graphFieldDepth = 3; @@ -353,21 +361,19 @@ private boolean isV2Schema() { } /** - * Emit the canonical schema v2 payload. Levels 1 (containment tree) and 2 (the {@code call_graph} - * overlay) are supported, whole-project, JSON. Anything else is an explicit error rather than a - * silently different result. + * Emit the canonical schema v2 payload. Levels 1 (containment tree), 2 (the {@code call_graph} + * overlay), 3 (the intraprocedural {@code cfg}/{@code cdg}/{@code ddg} overlays) and 4 (the + * interprocedural SDG overlays) are supported, whole-project, JSON. Anything else is an explicit + * error rather than a silently different result. */ private void analyzeV2() throws Exception { - if (analysisLevel > 3) { + if (analysisLevel > 4) { throw new ParameterException(spec.commandLine(), - "error: --schema v2 currently supports --analysis-level 1, 2, and 3 only"); - } - if (analysisLevel >= 3 - && !"ast".equalsIgnoreCase(l3Engine) - && !"wala".equalsIgnoreCase(l3Engine)) { - throw new ParameterException(spec.commandLine(), - "error: unknown --l3-engine '" + l3Engine + "'; use ast or wala"); + "error: --schema v2 currently supports --analysis-level 1, 2, 3, and 4 only"); } + // Ahead of the flag-value checks below, because it raises the effective level to 4 and those + // checks are level-gated: validating first would let `--emit neo4j --precision garbage` run at + // level 4 with an unrecognised value and silently fall back, against the CLI contract. if ("neo4j".equalsIgnoreCase(emit)) { // The graph is always full-depth (keystone depth rule): depth/section selectors cannot // be combined with it — error loudly rather than silently project a partial graph. @@ -381,9 +387,22 @@ private void analyzeV2() throws Exception { "error: --graph-field-depth does not apply to --emit neo4j; " + "the graph is always projected at full depth"); } - analysisLevel = 3; + analysisLevel = 4; externalCalls = true; } + if (analysisLevel >= 4 + && !"rta".equalsIgnoreCase(precision) + && !"0-cfa".equalsIgnoreCase(precision) + && !"0-1-cfa".equalsIgnoreCase(precision)) { + throw new ParameterException(spec.commandLine(), + "error: unknown --precision '" + precision + "'; use rta, 0-cfa or 0-1-cfa"); + } + if (analysisLevel >= 3 + && !"ast".equalsIgnoreCase(l3Engine) + && !"wala".equalsIgnoreCase(l3Engine)) { + throw new ParameterException(spec.commandLine(), + "error: unknown --l3-engine '" + l3Engine + "'; use ast or wala"); + } if (sourceAnalysis != null || targetFiles != null) { throw new ParameterException(spec.commandLine(), "error: --schema v2 supports whole-project analysis only " @@ -458,6 +477,24 @@ private void analyzeV2() throws Exception { if (wala != null) { L3WalaOverlays.apply(wala, input, modules, graphFieldDepth); } + // L4: the semantic ddg needs a WALA build regardless of --l3-engine, so build it here (or + // reuse the instance --l3-engine wala already built above) while the jars are still live. + // Must run strictly after the L3 overlay above: primeL4ModRef() permanently mutates the + // WalaAnalysis instance's shared mod/ref maps, and every PDG built afterwards — including + // L3's — would see the primed closure instead of the empty-defaulting one L3 relies on. + if (analysisLevel >= 4) { + if (wala == null) { + String buildCommand = noBuild ? null : (build == null ? "auto" : build); + String deps = dependencyDir == null ? null : dependencyDir.toString(); + wala = WalaAnalysis.of(input, deps, buildCommand, precision).orElse(null); + } + if (wala != null) { + L4WalaOverlays.apply(wala, modules, graphFieldDepth); + } else { + Log.warn("L4 semantic ddg unavailable (WALA build failed); emitting the derived " + + "SDG vertices and param edges only"); + } + } } finally { BuildProject.cleanLibraryDependencies(); } @@ -468,11 +505,23 @@ private void analyzeV2() throws Exception { L1Cache.save(cache, application, version, modules); } + // maxLevel reports the requested level: the L1-L3 passes above always run to that level (or + // degrade a specific overlay with a warning), and the L4 vertices/param edges below are + // engine-free, so they run whenever analysisLevel >= 4 regardless of the WALA build's fate. Analysis analysis; if (analysisLevel >= 2) { L2CallGraph.Result l2 = L2CallGraph.build(application, modules, rtaEndpoints, externalCalls); - analysis = V2Emitter.emit( - application, analysisLevel, modules, version, l2.callGraph(), l2.externalSymbols()); + // SdgVertices needs the callee ids L2CallGraph.build just backfilled onto call body nodes, + // and no dependency jars, so it runs here rather than inside the try block above. + SdgVertices.Result sdg = null; + if (analysisLevel >= 4) { + sdg = SdgVertices.apply(modules); + // Summaries read the vertices SdgVertices just added, so this must follow it. + SummaryPass.apply(modules, l2.callGraph(), graphFieldDepth); + } + analysis = V2Emitter.emit(application, analysisLevel, modules, version, + l2.callGraph(), l2.externalSymbols(), + sdg == null ? null : sdg.paramIn, sdg == null ? null : sdg.paramOut); } else { analysis = V2Emitter.emit(application, analysisLevel, modules, version); } diff --git a/src/main/java/com/ibm/cldk/L3WalaOverlays.java b/src/main/java/com/ibm/cldk/L3WalaOverlays.java index fa79879a..6f6167ff 100644 --- a/src/main/java/com/ibm/cldk/L3WalaOverlays.java +++ b/src/main/java/com/ibm/cldk/L3WalaOverlays.java @@ -89,47 +89,22 @@ public static void apply( int totalOverApprox = 0; for (MethodIr m : wala.applicationMethods()) { - // Derive the join keys using the same converters as RtaCallGraph (same package). - String binaryType = - RtaCallGraph.binaryTypeName(m.method.getDeclaringClass().getName().toString()); - String sig = RtaCallGraph.signature( - m.method.getName().toString(), m.method.getDescriptor().toString()); - - // Look up the JType then the JCallable. - TypeEntry entry = typeIndex.get(binaryType); - if (entry == null) { - skippedNoMatch++; - continue; - } - JCallable callable = entry.type.getCallables().get(sig); - if (callable == null) { + Optional joinedOpt = join(m, typeIndex, parseCache, modules); + if (!joinedOpt.isPresent()) { skippedNoMatch++; continue; } - - // Re-parse the source (memoized; source text comes from the L1 JModule). - CompilationUnit cu = parseOrCached(parseCache, entry.moduleKey, modules); - if (cu == null) { - skippedNoMatch++; - continue; - } - - // Find the BlockStmt for this method in the re-parsed CU. - Optional blockOpt = - findBody(cu, binaryType, entry.packageName, callable); - if (!blockOpt.isPresent()) { - skippedNoMatch++; - continue; - } - BlockStmt blockStmt = blockOpt.get(); + Joined joined = joinedOpt.get(); + JCallable callable = joined.callable; + BlockStmt blockStmt = joined.blockStmt; // Source text for L1BuildContext (spans need the original text for byte offsets). - String source = modules.get(entry.moduleKey).getSource(); + String source = modules.get(joined.moduleKey).getSource(); // Build a minimal L1BuildContext: only spanOf() is called in BodyNodeBuilder.populate; // the solver-dependent helpers are not used on this path. L1BuildContext ctx = new L1BuildContext( - applicationId, entry.moduleKey, source, 3, fieldDepth, "wala"); + applicationId, joined.moduleKey, source, 3, fieldDepth, "wala"); // Populate the body-node graph seeded with the callable's existing L1 call nodes. ControlFlowGraph cfg = new ControlFlowGraph(); @@ -161,13 +136,57 @@ public static void apply( + totalOverApprox + " sentinel over-approximation(s)"); } + // ----- the WALA-method → JCallable join ----------------------------------------------------- + + /** + * Resolves the WALA method {@code m} to the {@link JCallable} it was compiled from and the + * {@link BlockStmt} of its re-parsed source, or empty when any leg of the join fails (type not + * in the L1 map, signature not among its callables, source absent or unparseable, body not + * locatable — including a callable with no body at all, whose {@code body_span} is absent). + * + *

Shared with {@link L4WalaOverlays} so both overlay passes reach the same callable from the + * same WALA node; {@code parseCache} carries the memoized compilation units across the loop. + */ + static Optional join( + MethodIr m, + Map typeIndex, + Map parseCache, + Map modules) { + + // Derive the join keys using the same converters as RtaCallGraph (same package). + String binaryType = + RtaCallGraph.binaryTypeName(m.method.getDeclaringClass().getName().toString()); + String sig = RtaCallGraph.signature( + m.method.getName().toString(), m.method.getDescriptor().toString()); + + // Look up the JType then the JCallable. + TypeEntry entry = typeIndex.get(binaryType); + if (entry == null) { + return Optional.empty(); + } + JCallable callable = entry.type.getCallables().get(sig); + if (callable == null) { + return Optional.empty(); + } + + // Re-parse the source (memoized; source text comes from the L1 JModule). + CompilationUnit cu = parseOrCached(parseCache, entry.moduleKey, modules); + if (cu == null) { + return Optional.empty(); + } + + // Find the BlockStmt for this method in the re-parsed CU. + return findBody(cu, binaryType, entry.packageName, callable) + .map(block -> new Joined(entry.moduleKey, callable, block)); + } + // ----- index building ----------------------------------------------------------------------- /** * Builds a map from WALA binary type name (e.g. {@code "com.example.Widget$Inner"}) to the * module key and JType that holds that type's callables. */ - private static Map buildTypeIndex(Map modules) { + static Map buildTypeIndex(Map modules) { Map index = new LinkedHashMap<>(); for (Map.Entry entry : modules.entrySet()) { String moduleKey = entry.getKey(); @@ -357,9 +376,22 @@ private static String deriveApplicationId(Map modules) { return last > 0 ? moduleId.substring(0, last) : moduleId; } - // ----- inner type --------------------------------------------------------------------------- + // ----- inner types -------------------------------------------------------------------------- + + /** One WALA method successfully joined to its L1 callable and re-parsed body block. */ + static final class Joined { + final String moduleKey; + final JCallable callable; + final BlockStmt blockStmt; + + Joined(String moduleKey, JCallable callable, BlockStmt blockStmt) { + this.moduleKey = moduleKey; + this.callable = callable; + this.blockStmt = blockStmt; + } + } - private static final class TypeEntry { + static final class TypeEntry { final String moduleKey; final JType type; final String packageName; diff --git a/src/main/java/com/ibm/cldk/L4WalaOverlays.java b/src/main/java/com/ibm/cldk/L4WalaOverlays.java new file mode 100644 index 00000000..773f4b08 --- /dev/null +++ b/src/main/java/com/ibm/cldk/L4WalaOverlays.java @@ -0,0 +1,290 @@ +/* +Copyright IBM Corporation 2023, 2024 + +Licensed under the Apache Public License 2.0, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package com.ibm.cldk; + +import com.github.javaparser.ast.CompilationUnit; +import com.ibm.cldk.L3WalaOverlays.Joined; +import com.ibm.cldk.L3WalaOverlays.TypeEntry; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JDdgEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.utils.Log; +import com.ibm.cldk.wala.InstructionToNode; +import com.ibm.cldk.wala.WalaAnalysis; +import com.ibm.cldk.wala.WalaAnalysis.MethodIr; +import com.ibm.cldk.wala.WalaPdgBuilder; +import com.ibm.cldk.wala.WalaPdgBuilder.PdgOverlays; +import com.ibm.wala.ipa.callgraph.propagation.ArrayContentsKey; +import com.ibm.wala.ipa.callgraph.propagation.InstanceFieldKey; +import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; +import com.ibm.wala.ipa.callgraph.propagation.PointerKey; +import com.ibm.wala.ipa.callgraph.propagation.StaticFieldKey; +import com.ibm.wala.ipa.slicer.Dependency; +import com.ibm.wala.ipa.slicer.HeapStatement; +import com.ibm.wala.ipa.slicer.NormalStatement; +import com.ibm.wala.ipa.slicer.PDG; +import com.ibm.wala.ipa.slicer.Statement; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Post-build L4 orchestrator: the semantic half of {@code ddg}. + * + *

L3 answers "which definitions reach this use through named variables" — syntax the AST or the + * SSA form can see on its own, tagged {@code prov:["ssa"]}. What neither can see is dependence + * carried by the heap across a call: {@code put(v)} then {@code get()} touch the same field through + * two different frames, and only a points-to analysis can attest that they alias. Those edges are + * tagged {@code prov:["points-to"]} and are what this pass adds. + * + *

It starts by priming WALA's mod/ref maps ({@link WalaAnalysis#primeL4ModRef}), which is what + * makes the PDG materialize heap statements at call sites at all. The dependences then arrive in + * two shapes, and both are collected per method: + *

    + *
  • Intraprocedural — a write and a read of the same location inside one body. Both + * endpoints are {@code NormalStatement}s, so {@link WalaPdgBuilder} already emits them with + * {@code prov:["points-to"]} and is simply re-run here. (Under {@code --l3-engine wala} L3 + * emitted these too and the merge dedupes them; under the AST engine they are new.) + *
  • Interprocedural — the round trip through a callee. WALA anchors these on + * {@code HeapStatement}s, which have no body node, so the builder's deliberate + * {@code NormalStatement}-only filter cannot carry them. They are projected onto their own + * call statements here instead; see {@link #interproceduralHeapEdges}. + *
+ * + *

The merge is strictly additive: L4 may only add reach, never remove it, so every edge L3 + * produced survives byte-identical and only unseen {@code (src,dst,var,prov)} tuples are appended. + */ +public final class L4WalaOverlays { + + private L4WalaOverlays() {} + + /** + * Merges WALA's points-to {@code ddg} edges into the L3 overlays already on {@code modules}. + * + * @param wala the pre-built WALA analysis (the same instance L3 used) + * @param modules the module map, already carrying L3 overlays (mutated in place) + * @param fieldDepth the DDG access-path bound k, for parity with + * {@link L3WalaOverlays#apply} — heap access paths come from the WALA + * instruction's declared field, so it is not consulted here + */ + public static void apply(WalaAnalysis wala, Map modules, int fieldDepth) { + if (modules.isEmpty()) { + return; + } + + wala.primeL4ModRef(); + + Map typeIndex = L3WalaOverlays.buildTypeIndex(modules); + Map parseCache = new LinkedHashMap<>(); + + int matched = 0; + int addedEdges = 0; + + for (MethodIr m : wala.applicationMethods()) { + Optional joined = L3WalaOverlays.join(m, typeIndex, parseCache, modules); + if (!joined.isPresent()) { + continue; + } + JCallable callable = joined.get().callable; + if (callable.getDdg() == null) { + // No L3 ddg to extend (analysis ran below level 3): nothing to merge into. + continue; + } + + InstructionToNode mapper = new InstructionToNode( + InstructionToNode.statementsByLine(joined.get().blockStmt)); + + // Intraprocedural half: the builder's own points-to edges, both endpoints already + // NormalStatements. Under `--l3-engine wala` L3 emitted these too and the merge dedupes + // them away; under the AST engine they are new. + PdgOverlays pdg = WalaPdgBuilder.build(wala, m, mapper); + List produced = new ArrayList<>(pdg.ddg); + + // Interprocedural half: the edges only the primed mod/ref can see. + produced.addAll(interproceduralHeapEdges(wala, m, mapper)); + + addedEdges += merge(callable, produced); + matched++; + } + + Log.info("L4 WALA semantic ddg applied: " + matched + " callable(s) visited, " + + addedEdges + " points-to edge(s) added"); + } + + /** + * The heap dependences that cross a call boundary, projected onto the call statements that + * carry them. + * + *

WALA anchors every interprocedural heap dependence on a {@link HeapStatement}, never on a + * {@link NormalStatement}: {@code put(v); … get()} surfaces as {@code HeapReturnCaller(put) → + * HeapParamCaller(get)}. Those endpoints have no body node of their own, which is why + * {@link WalaPdgBuilder} — whose {@code NormalStatement}-only filter is deliberate — cannot see + * them. Each caller-side heap statement does name the SSA index of its call, so it projects + * onto that call's own {@code NormalStatement}, and the dependence lands on the two body nodes + * a reader would point at. + * + *

Callee-side heap statements ({@code HEAP_PARAM_CALLEE}, {@code HEAP_RET_CALLEE}) are + * dropped: they are the method's own SDG interface, which {@code SdgVertices} derives as + * {@code param_in}/{@code param_out} vertices rather than {@code ddg} edges. + * + *

This builds a second PDG for {@code m}. That is deliberate — it keeps {@link WalaPdgBuilder} + * and the L3 paths it serves untouched — and the pass runs only at {@code -a 4}. + */ + private static List interproceduralHeapEdges( + WalaAnalysis wala, MethodIr m, InstructionToNode mapper) { + + PDG pdg = wala.pdgFor(m.node); + + // Same lazy-edge trap as WalaPdgBuilder: WALA materializes heap du-pairs only when the + // *unlabeled* successor/predecessor accessor is called. Without this loop the labeled + // accessors below return nothing. + for (Statement s : pdg) { + pdg.getSuccNodes(s).forEachRemaining(x -> {}); + pdg.getPredNodes(s).forEachRemaining(x -> {}); + } + + Map callStatements = new HashMap<>(); + for (Statement s : pdg) { + if (s instanceof NormalStatement) { + NormalStatement ns = (NormalStatement) s; + callStatements.put(ns.getInstructionIndex(), ns); + } + } + + List edges = new ArrayList<>(); + for (Dependency label : new Dependency[]{Dependency.DATA_DEP, Dependency.HEAP_DATA_DEP}) { + for (Statement s : pdg) { + for (Iterator it = pdg.getSuccNodes(s, label); it.hasNext();) { + Statement d = it.next(); + + // An endpoint that is a heap statement is what makes the edge interprocedural; + // a pair of NormalStatements is the intraprocedural case WalaPdgBuilder emits. + if (!(s instanceof HeapStatement) && !(d instanceof HeapStatement)) { + continue; + } + // PDG labels an edge into a heap statement DATA_DEP even though it is heap flow + // (PDG.createHeapDataDependenceEdges), so a DATA_DEP edge carries the heap + // exactly when its target is one; every HEAP_DATA_DEP edge does. + if (label == Dependency.DATA_DEP && !(d instanceof HeapStatement)) { + continue; + } + NormalStatement src = projectToCall(s, callStatements); + NormalStatement dst = projectToCall(d, callStatements); + if (src == null || dst == null || src.equals(dst)) { + continue; + } + JDdgEdge edge = new JDdgEdge(); + edge.setSrc(mapper.map( + src.getInstruction(), wala.sourceLine(m, src.getInstructionIndex()))); + edge.setDst(mapper.map( + dst.getInstruction(), wala.sourceLine(m, dst.getInstructionIndex()))); + edge.setVar(heapVar(s, d)); + edge.getProv().add("points-to"); + edges.add(edge); + } + } + } + return edges; + } + + /** + * The {@link NormalStatement} that owns {@code s}: itself when it already is one, the statement + * of the call it decorates when it is a caller-side heap statement, and {@code null} otherwise + * (callee-side heap statements and the method entry/exit interface have no owning instruction). + */ + private static NormalStatement projectToCall( + Statement s, Map callStatements) { + if (s instanceof NormalStatement) { + return (NormalStatement) s; + } + if (s instanceof HeapStatement.HeapParamCaller) { + return callStatements.get(((HeapStatement.HeapParamCaller) s).getCallIndex()); + } + if (s instanceof HeapStatement.HeapReturnCaller) { + return callStatements.get(((HeapStatement.HeapReturnCaller) s).getCallIndex()); + } + return null; + } + + /** + * The access path of the heap location the edge carries, taken from whichever endpoint is a + * {@link HeapStatement}. Mirrors the vocabulary {@link WalaPdgBuilder} uses for its own heap + * edges: the field's simple name, {@code "[*]"} for array contents, {@code "heap"} otherwise — + * non-empty by schema contract. + */ + private static String heapVar(Statement src, Statement dst) { + PointerKey location = src instanceof HeapStatement + ? ((HeapStatement) src).getLocation() + : ((HeapStatement) dst).getLocation(); + if (location instanceof InstanceFieldKey) { + return ((InstanceFieldKey) location).getField().getName().toString(); + } + if (location instanceof StaticFieldKey) { + return ((StaticFieldKey) location).getField().getName().toString(); + } + if (location instanceof ArrayContentsKey) { + return "[*]"; + } + return "heap"; + } + + /** + * Appends the {@code points-to} edges of {@code produced} that {@code callable} does not + * already carry, then re-sorts the whole list on {@code (src,dst,var,prov)} — the same key + * both producers sort on, so the merged list stays in canonical order. + * + * @return the number of edges actually added + */ + private static int merge(JCallable callable, List produced) { + List merged = new ArrayList<>(callable.getDdg()); + Set seen = new LinkedHashSet<>(); + for (JDdgEdge edge : merged) { + seen.add(key(edge)); + } + + int added = 0; + for (JDdgEdge edge : produced) { + if (!edge.getProv().contains("points-to")) { + continue; + } + if (seen.add(key(edge))) { + merged.add(edge); + added++; + } + } + if (added == 0) { + return 0; + } + + merged.sort(Comparator.comparing(JDdgEdge::getSrc) + .thenComparing(JDdgEdge::getDst) + .thenComparing(JDdgEdge::getVar) + .thenComparing(e -> e.getProv().toString())); + callable.setDdg(merged); + return added; + } + + /** The dedup identity of a ddg edge: {@code (src, dst, var, prov)}. */ + private static String key(JDdgEdge edge) { + return edge.getSrc() + "\0" + edge.getDst() + "\0" + edge.getVar() + + "\0" + edge.getProv(); + } +} diff --git a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java index d24009b5..f0696928 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java @@ -25,6 +25,7 @@ import com.ibm.cldk.schema.JEnumConstant; import com.ibm.cldk.schema.JExternalSymbol; import com.ibm.cldk.schema.JField; +import com.ibm.cldk.schema.JIdEdge; import com.ibm.cldk.schema.JImport; import com.ibm.cldk.schema.JModule; import com.ibm.cldk.schema.JRecordComponent; @@ -41,10 +42,11 @@ /** * The schema v2 → Neo4j projection: a pure {@code (Analysis, appName) → GraphRows} function, no - * I/O, no driver. The vocabulary is {@link V2SchemaCatalog} (graph contract 2.0.0), mirroring + * I/O, no driver. The vocabulary is {@link V2SchemaCatalog} (graph contract 2.1.0), mirroring * codeanalyzer-python's projection: call sites are {@code :JBodyNode} rows (no call-site nodes), * parameters flatten to {@code parameters_json}, javadoc collapses to {@code docstring}, and the - * L3 {@code cfg}/{@code cdg}/{@code ddg} overlays become typed relationships between body nodes. + * L3 {@code cfg}/{@code cdg}/{@code ddg} and L4 {@code param_in}/{@code param_out}/{@code summary} + * overlays become typed relationships between body nodes. * *

Body-node identity is the global ordinal: {@code @} for real * statements ({@code 12:5}), {@code } for synthetic bookends whose local @@ -114,6 +116,21 @@ public static GraphRows project(Analysis analysis, String appName) { } } + if (analysis.getApplication().getParamIn() != null) { + for (JIdEdge e : analysis.getApplication().getParamIn()) { + b.edgeIfBothResolved("J_PARAM_IN", + new NodeRef("JBodyNode", "id", e.getSrc()), + new NodeRef("JBodyNode", "id", e.getDst()), RowBuilder.props()); + } + } + if (analysis.getApplication().getParamOut() != null) { + for (JIdEdge e : analysis.getApplication().getParamOut()) { + b.edgeIfBothResolved("J_PARAM_OUT", + new NodeRef("JBodyNode", "id", e.getSrc()), + new NodeRef("JBodyNode", "id", e.getDst()), RowBuilder.props()); + } + } + if (analysis.getApplication().getCallGraph() != null) { for (JCallEdge e : analysis.getApplication().getCallGraph()) { Map p = RowBuilder.props(); @@ -314,6 +331,9 @@ private static void projectCallable(RowBuilder b, NodeRef owner, String signatur np.put("argument_expr", n.getArgumentExpr()); putLines(np, n.getSpan()); np.put("_module", fileKey); + // L4 SDG synthetic-vertex payload: absent on every non-synthetic node (prune drops nulls). + np.put("var", n.getOf()); + np.put("call_node", n.getParent() == null ? null : globalOrdinal(c.getId(), n.getParent())); NodeRef nr = b.node(Arrays.asList("JBodyNode"), "id", id, RowBuilder.prune(np)); b.edge("J_HAS_BODY_NODE", ref, nr); if (n.getCallee() != null) { @@ -345,6 +365,14 @@ private static void projectCallable(RowBuilder b, NodeRef owner, String signatur RowBuilder.prune(ep), k); } } + if (c.getSummary() != null) { + // Endpoints are call-site-local ids on c's own body (SummaryPass.emit), same globalOrdinal + // rule as J_CFG_NEXT/J_CDG/J_DDG above — no resolution gating needed, unlike the + // application-scope param_in/param_out edges above, which cross into another callable's body. + for (JIdEdge e : c.getSummary()) { + b.edge("J_SUMMARY", bodyRef(c, e.getSrc()), bodyRef(c, e.getDst())); + } + } for (Map.Entry local : c.getTypes().entrySet()) { projectType(b, ref, "J_DECLARES", local.getKey(), local.getValue(), module, fileKey, diff --git a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java index 2f420504..89920c2d 100644 --- a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java +++ b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java @@ -21,7 +21,7 @@ import java.util.Map; /** - * The schema v2 Neo4j graph catalog (graph contract {@code 2.0.0}) — the in-repo source of truth + * The schema v2 Neo4j graph catalog (graph contract {@code 2.1.0}) — the in-repo source of truth * for what {@link V2GraphProjector} may emit, serialized by {@code --emit schema} and enforced by * the v2 conformance test. Mirrors codeanalyzer-python's {@code neo4j/schema.py} vocabulary with * {@code J}/{@code J_} namespacing; java-only constructs (enum constants, record components, @@ -40,7 +40,9 @@ public final class V2SchemaCatalog { private V2SchemaCatalog() {} - public static final String SCHEMA_VERSION = "2.0.0"; + // 2.1.0: additive MINOR — L4 SDG overlay (JBodyNode.var/call_node; J_PARAM_IN/J_PARAM_OUT/ + // J_SUMMARY, reserved at 2.0.0, now actually emitted). + public static final String SCHEMA_VERSION = "2.1.0"; /** Labels layered onto a node in addition to its merge + specific labels. */ public static final List MARKER_LABELS = Arrays.asList("JEntrypoint"); @@ -125,7 +127,9 @@ private static List buildNodeLabels() { .put("return_type", "string").put("accessibility", "string") .put("is_constructor_call", "boolean").put("is_static_call", "boolean") .put("argument_types", "string[]").put("argument_expr", "string[]") - .put("_module", "string")))); + .put("_module", "string") + // L4 SDG synthetic-vertex payload (python-parity names). + .put("var", "string").put("call_node", "string")))); n.add(node("JPackage", "JPackage", "name", new P().put("name", "string").done())); @@ -168,7 +172,8 @@ private static List buildRelTypes() { r.add(rel("J_CDG", body, body, none)); r.add(rel("J_DDG", body, body, new P().put("var", "string").put("prov", "string[]").put("_k", "string").done())); - // L4 SDG — declared so the contract is stable; not emitted until L4 lands. + // L4 SDG — reserved at 2.0.0, emitted from 2.1.0: J_PARAM_IN/J_PARAM_OUT from the + // application-scope param_in/param_out edges, J_SUMMARY per callable. r.add(rel("J_PARAM_IN", body, body, new P().put("var", "string").done())); r.add(rel("J_PARAM_OUT", body, body, new P().put("var", "string").done())); r.add(rel("J_SUMMARY", body, body, none)); diff --git a/src/main/java/com/ibm/cldk/schema/JApplication.java b/src/main/java/com/ibm/cldk/schema/JApplication.java index 73962639..3f558c74 100644 --- a/src/main/java/com/ibm/cldk/schema/JApplication.java +++ b/src/main/java/com/ibm/cldk/schema/JApplication.java @@ -23,4 +23,10 @@ public class JApplication { * edge dangles. Left {@code null} at L1 so the key is omitted. */ private Map externalSymbols; + + /** L4 {@code actual_in → formal_in} edges (global ordinals); null (absent) below level 4. */ + private List paramIn; + + /** L4 {@code formal_out → actual_out} edges (global ordinals); null (absent) below level 4. */ + private List paramOut; } diff --git a/src/main/java/com/ibm/cldk/schema/JBodyNode.java b/src/main/java/com/ibm/cldk/schema/JBodyNode.java index f1839bfd..571082c9 100644 --- a/src/main/java/com/ibm/cldk/schema/JBodyNode.java +++ b/src/main/java/com/ibm/cldk/schema/JBodyNode.java @@ -68,6 +68,16 @@ public class JBodyNode { /** Syntactically evident (a {@code new} expression or {@code this(...)}/{@code super(...)}). */ private boolean isConstructorCall; + /** + * L4 synthetic-vertex payload: the value this vertex stands for — a parameter name on + * {@code formal_in}, {@code $ret} on {@code formal_out}/{@code actual_out}, {@code argN} on + * {@code actual_in}. Absent on every non-synthetic node. + */ + private String of; + + /** The call-site local id an {@code actual_in}/{@code actual_out} vertex belongs to. */ + private String parent; + /** * L2 backfill plumbing (§4): the binary name of the resolved callee's declaring type * ({@code java.util.Map$Entry}) — the fact {@code dst} needs but {@code receiver_type} cannot diff --git a/src/main/java/com/ibm/cldk/schema/JCallable.java b/src/main/java/com/ibm/cldk/schema/JCallable.java index fb8926e1..e1c02bc3 100644 --- a/src/main/java/com/ibm/cldk/schema/JCallable.java +++ b/src/main/java/com/ibm/cldk/schema/JCallable.java @@ -77,6 +77,9 @@ public class JCallable { /** L3 data-dependence edges (prov {@code ssa}) over this callable's body nodes; null below level 3. */ private List ddg; + /** L4 {@code actual_in → actual_out} summary edges (local ids); null (absent) below level 4. */ + private List summary; + /** Local (method-body) classes, keyed by simple name — nesting encoded by containment (D4). */ private Map types = new LinkedHashMap<>(); } diff --git a/src/main/java/com/ibm/cldk/schema/JIdEdge.java b/src/main/java/com/ibm/cldk/schema/JIdEdge.java new file mode 100644 index 00000000..a0fdaf0f --- /dev/null +++ b/src/main/java/com/ibm/cldk/schema/JIdEdge.java @@ -0,0 +1,13 @@ +package com.ibm.cldk.schema; + +import lombok.Data; + +/** + * An id-to-id edge with no payload — the shape of {@code param_in}/{@code param_out} (application + * scope, global-ordinal endpoints) and {@code summary} (callable scope, local endpoints). + */ +@Data +public class JIdEdge { + private String src; + private String dst; +} diff --git a/src/main/java/com/ibm/cldk/schema/V2Emitter.java b/src/main/java/com/ibm/cldk/schema/V2Emitter.java index e6b7d402..dbda9f53 100644 --- a/src/main/java/com/ibm/cldk/schema/V2Emitter.java +++ b/src/main/java/com/ibm/cldk/schema/V2Emitter.java @@ -38,6 +38,18 @@ public static Analysis emit( String analyzerVersion, List callGraph, Map externalSymbols) { + return emit(appName, maxLevel, modules, analyzerVersion, callGraph, externalSymbols, null, null); + } + + public static Analysis emit( + String appName, + int maxLevel, + Map modules, + String analyzerVersion, + List callGraph, + Map externalSymbols, + List paramIn, + List paramOut) { JApplication application = new JApplication(); application.setId(CanId.applicationId(appName)); @@ -55,6 +67,12 @@ public static Analysis emit( if (externalSymbols != null && !externalSymbols.isEmpty()) { application.setExternalSymbols(externalSymbols); } + if (paramIn != null && !paramIn.isEmpty()) { + application.setParamIn(paramIn); + } + if (paramOut != null && !paramOut.isEmpty()) { + application.setParamOut(paramOut); + } Analysis analysis = new Analysis(); analysis.setSchemaVersion("2.0.0"); diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/Scc.java b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/Scc.java new file mode 100644 index 00000000..d23e0a56 --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/Scc.java @@ -0,0 +1,139 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import com.ibm.cldk.schema.JCallEdge; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Tarjan SCC condensation of the call graph: the bottom-up (callees-first) processing order the L4 + * summary pass (Task 9) needs, since a function's summary can only be computed once its callees' + * are known, and mutual recursion must be solved together as one group. Tarjan is iterative — an + * explicit-stack DFS, not the textbook recursive one — because a real call graph is deep enough to + * overflow the JVM stack. Node visitation and component membership are both sorted so the schedule + * is independent of the caller's iteration order over {@code nodes}. + */ +public final class Scc { + + private Scc() {} + + /** Components in reverse-topological (bottom-up, callees-first) order; ids sorted within each. */ + public static List> condense(Collection nodes, List edges) { + Map> adjacency = buildAdjacency(nodes, edges); + + Map index = new HashMap<>(); + Map lowlink = new HashMap<>(); + Set onStack = new HashSet<>(); + Deque stack = new ArrayDeque<>(); + List> components = new ArrayList<>(); + int[] counter = {0}; + + for (String root : adjacency.keySet()) { + if (!index.containsKey(root)) { + strongConnect(root, adjacency, index, lowlink, onStack, stack, components, counter); + } + } + return components; + } + + /** {@code nodes} sorted, each mapped to its edges into other {@code nodes} (external targets dropped). */ + private static Map> buildAdjacency(Collection nodes, List edges) { + // TreeSet per node: dedups (src, dst) pairs and sorts successors in one step, so a cycle + // reached via more than one edge can't leak the caller's edge-list order into which member + // Tarjan happens to root the component on. + Map> bySource = new TreeMap<>(); + for (String n : nodes) { + bySource.put(n, new TreeSet<>()); + } + for (JCallEdge e : edges) { + TreeSet successors = bySource.get(e.getSrc()); + if (successors != null && bySource.containsKey(e.getDst())) { + successors.add(e.getDst()); + } + } + Map> adjacency = new TreeMap<>(); + for (Map.Entry> entry : bySource.entrySet()) { + adjacency.put(entry.getKey(), new ArrayList<>(entry.getValue())); + } + return adjacency; + } + + /** + * One frame of the explicit-stack DFS: a node plus how far its successor list has been + * consumed, standing in for the recursive call's local variables and program counter. + */ + private static final class Frame { + final String node; + int nextSuccessor; + + Frame(String node) { + this.node = node; + } + } + + private static void strongConnect( + String root, + Map> adjacency, + Map index, + Map lowlink, + Set onStack, + Deque stack, + List> components, + int[] counter) { + Deque work = new ArrayDeque<>(); + work.push(new Frame(root)); + index.put(root, counter[0]); + lowlink.put(root, counter[0]); + counter[0]++; + stack.push(root); + onStack.add(root); + + while (!work.isEmpty()) { + Frame frame = work.peek(); + List successors = adjacency.get(frame.node); + if (frame.nextSuccessor < successors.size()) { + String succ = successors.get(frame.nextSuccessor++); + if (!index.containsKey(succ)) { + index.put(succ, counter[0]); + lowlink.put(succ, counter[0]); + counter[0]++; + stack.push(succ); + onStack.add(succ); + work.push(new Frame(succ)); + } else if (onStack.contains(succ)) { + lowlink.put(frame.node, Math.min(lowlink.get(frame.node), index.get(succ))); + } + continue; + } + + // All of frame.node's successors are explored; if it's a component root, pop the + // component off the Tarjan stack, then fold its lowlink into its caller's (the next + // frame down) exactly as the recursive version does on return. + work.pop(); + if (lowlink.get(frame.node).equals(index.get(frame.node))) { + List component = new ArrayList<>(); + String member; + do { + member = stack.pop(); + onStack.remove(member); + component.add(member); + } while (!member.equals(frame.node)); + Collections.sort(component); + components.add(component); + } + if (!work.isEmpty()) { + Frame caller = work.peek(); + lowlink.put(caller.node, Math.min(lowlink.get(caller.node), lowlink.get(frame.node))); + } + } + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVertices.java b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVertices.java new file mode 100644 index 00000000..1227400a --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVertices.java @@ -0,0 +1,173 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JIdEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JParameter; +import com.ibm.cldk.schema.JType; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +/** + * L4 stage 7 (HRB assembly), derived: the synthetic parameter vertices and the cross-function + * param_in/param_out edges are structurally determined by the L2-backfilled call sites and the + * callee's parameter list, so they are built directly from the v2 tree — no engine, pure and + * deterministic. Summary edges are the SummaryPass's job (PR 2), and the semantic ddg is + * L4WalaOverlays' (points-to genuinely needs WALA). + */ +public final class SdgVertices { + + private SdgVertices() {} + + public static final class Result { + public final List paramIn; + public final List paramOut; + + private Result(List paramIn, List paramOut) { + this.paramIn = paramIn; + this.paramOut = paramOut; + } + } + + /** Mutates modules in place (adds synthetic body nodes); returns the application-scope edges. */ + public static Result apply(Map modules) { + Map byId = new LinkedHashMap<>(); + forEachCallable(modules, c -> { + if (c.getId() != null) { + byId.put(c.getId(), c); + } + }); + + List paramIn = new ArrayList<>(); + List paramOut = new ArrayList<>(); + forEachCallable(modules, c -> { + addFormals(c); + addActualsAndEdges(c, byId, paramIn, paramOut); + }); + + paramIn.sort(Comparator.comparing(JIdEdge::getSrc).thenComparing(JIdEdge::getDst)); + paramOut.sort(Comparator.comparing(JIdEdge::getSrc).thenComparing(JIdEdge::getDst)); + return new Result(paramIn, paramOut); + } + + /** + * Adds {@code c}'s own formal vertices: one {@code @formal_in} per declared parameter + * (declaration order), plus a {@code @formal_out} when {@code c} actually hands a value back — + * the thing a caller's {@code actual_out} eventually connects to. + */ + private static void addFormals(JCallable c) { + List params = c.getParameters(); + for (int i = 0; i < params.size(); i++) { + c.getBody().put("@formal_in:" + i, vertex("formal_in", params.get(i).getName(), null)); + } + if (returnsValue(c)) { + c.getBody().put("@formal_out", vertex("formal_out", "$ret", null)); + } + } + + /** {@code void} and a constructor's declared "return type" are not a value a caller can receive. */ + private static boolean returnsValue(JCallable c) { + return c.getReturnType() != null + && !"void".equals(c.getReturnType()) + && !"constructor".equals(c.getKind()); + } + + /** + * For each in-project call site already in {@code c}'s own body, adds its actual vertices and + * wires them to the resolved callee's formals. Iterates a snapshot of {@code c.getBody()} since + * the loop body inserts the very actual vertices it discovers into that same map. An in-project + * callee id that does not resolve through {@code byId} (a stale id) still gets its vertices — + * only the edges are skipped. + */ + private static void addActualsAndEdges( + JCallable c, Map byId, List paramIn, List paramOut) { + List> callSites = new ArrayList<>(c.getBody().entrySet()); + for (Map.Entry entry : callSites) { + String local = entry.getKey(); + JBodyNode node = entry.getValue(); + String calleeId = node.getCallee(); + if (!"call".equals(node.getKind()) || calleeId == null || calleeId.contains("/@external/")) { + continue; + } + + int nArgs = node.getArgumentExpr().size(); + for (int i = 0; i < nArgs; i++) { + c.getBody().put(local + "/actual_in:" + i, vertex("actual_in", "arg" + i, local)); + } + boolean hasActualOut = node.getReturnType() != null && !"void".equals(node.getReturnType()); + if (hasActualOut) { + c.getBody().put(local + "/actual_out", vertex("actual_out", "$ret", local)); + } + + JCallable callee = byId.get(calleeId); + if (callee == null) { + continue; // vertices stand on their own; edge-only-when-resolved + } + + int calleeParams = callee.getParameters().size(); + if (calleeParams > 0) { + // Loop over every argument, not just the first calleeParams of them: L4 is + // over-approximate/weak-update (may add reach, never drop it), so a call with more + // actuals than declared formals (varargs) must not silently lose the tail args — + // they all collapse onto the last formal instead of being left unconnected. + for (int i = 0; i < nArgs; i++) { + int formalIndex = Math.min(i, calleeParams - 1); + paramIn.add(edge( + global(c, local + "/actual_in:" + i), global(callee, "@formal_in:" + formalIndex))); + } + } + // Recomputed rather than read off callee.getBody(): traversal order is not call order, so + // the callee's own addFormals may not have run yet when the caller's site is processed. + if (hasActualOut && returnsValue(callee)) { + paramOut.add(edge(global(callee, "@formal_out"), global(c, local + "/actual_out"))); + } + } + } + + /** The global body-node id for a local key under {@code c} — real keys get {@code @}, synthetics concatenate. */ + private static String global(JCallable c, String local) { + return local.startsWith("@") ? c.getId() + local : c.getId() + "@" + local; + } + + /** A synthetic body node; synthetics carry no {@code span} — they are not a source position. */ + private static JBodyNode vertex(String kind, String of, String parent) { + JBodyNode n = new JBodyNode(); + n.setKind(kind); + n.setOf(of); + n.setParent(parent); + return n; + } + + private static JIdEdge edge(String src, String dst) { + JIdEdge e = new JIdEdge(); + e.setSrc(src); + e.setDst(dst); + return e; + } + + /** + * Visits every callable reachable from {@code modules} — mirrors the traversal shape of + * {@code V2GraphProjector.indexTypes} (nested types, then each callable's own local types) without + * depending on that class. + */ + private static void forEachCallable(Map modules, Consumer visitor) { + for (JModule m : modules.values()) { + walkTypes(m.getTypes(), visitor); + } + } + + private static void walkTypes(Map types, Consumer visitor) { + for (JType t : types.values()) { + walkTypes(t.getTypes(), visitor); + for (JCallable c : t.getCallables().values()) { + visitor.accept(c); + walkTypes(c.getTypes(), visitor); + } + } + } +} diff --git a/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPass.java b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPass.java new file mode 100644 index 00000000..310427ba --- /dev/null +++ b/src/main/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPass.java @@ -0,0 +1,450 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.schema.JCallEdge; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JDdgEdge; +import com.ibm.cldk.schema.JIdEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JParameter; +import com.ibm.cldk.schema.JType; +import com.ibm.cldk.schema.Span; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; + +/** + * L4 stage 8: the interprocedural {@code summary} edges. A summary edge is a shortcut inside a + * caller — {@code /actual_in:j → /actual_out} at one call site — recording that data + * entering the callee through argument {@code j} may come back out through its return value. It + * exists so a consumer (slicing, taint) can cross a call in one hop instead of descending into the + * callee. + * + *

Emitting it needs the callee's transfer relation {@code flows(callee) ⊆ params × {$ret}}, which + * needs its callees' first — hence the bottom-up walk over {@link Scc#condense} (callees + * first), with mutual recursion resolved by iterating one component to a fixpoint. + * + *

The relation is derived syntactically, in the weak-update (may-flow, never-drop) + * posture the L4 design takes: a parameter reaches the return if the callable's {@code ddg} carries + * it there, if a call site passes it on and that callee's own summary returns it, or if the + * parameter is simply named in a body node's source text. That last rule is what makes the + * pass useful at all on ordinary code, because {@link DdgBuilder} gives a parameter no defining node + * — nothing in the {@code ddg} is ever rooted at one. Without it {@code int c(int z) { return z - 3; + * }} has no def-use to follow at all, and {@code int m(int q) { int t = q; return t; }} has only + * {@code (decl → ret, var "t")}, which no seed reaches. The cost is counting a parameter merely + * mentioned in a statement as flowing out of it. Over-approximating there is the accepted + * trade. + */ +public final class SummaryPass { + + private SummaryPass() {} + + /** + * Sets {@code summary} on every callable holding at least one pass-through call site; leaves it + * null (absent — "no fact") elsewhere. Runs after {@link SdgVertices#apply}, whose synthetic + * vertices are both this pass's seeds and its edge endpoints. + * + * @param fieldDepth the k of the access-path k-limit the {@code ddg} was built with. Flow labels + * here are parameter indices, and {@link AccessPath#of} truncates only the tail of a + * path, never its base segment — so base matching already agrees with any k, and the label + * set is bounded by arity rather than by k. Carried because it is where the bound enters once + * the relation grows past {@code param → $ret} to field-sensitive labels. + */ + public static void apply(Map modules, List callGraph, int fieldDepth) { + // ponytail: single-threaded. Wavefront parallelism over the SCC DAG (a component may run as + // soon as its successors have) drops straight in if this ever dominates a real run — the + // condensation already supplies the dependency order. + // ponytail: recomputed every run. Caching summaries in `cache_dir` waits on a cache that + // survives analysisLevel >= 3, which CodeAnalyzer currently bypasses outright. + Map fns = index(modules); + Map> flows = solve(fns, callGraph); + emit(fns, flows); + } + + // ----- stage 1: the per-callable transfer relation ------------------------------------------- + + /** One resolved, in-project call site — where bridge edges and summary edges both come from. */ + private static final class Site { + private final String local; + private final String callee; + private final List args; + private final boolean hasActualOut; + + Site(String local, String callee, List args, boolean hasActualOut) { + this.local = local; + this.callee = callee; + this.args = args; + this.hasActualOut = hasActualOut; + } + + String actualIn(int j) { + return local + "/actual_in:" + j; + } + + String actualOut() { + return local + "/actual_out"; + } + } + + /** + * The round-invariant facts one callable contributes. Everything here is pure syntax, so it is + * computed once; only the callee bridges change between fixpoint rounds. + */ + private static final class Fn { + private final JCallable callable; + /** The intraprocedural reachability graph (src → dsts) — the fixed half; bridges are added per round. */ + private final Map> graph = new LinkedHashMap<>(); + /** Parameter index → the body nodes at which its value is first visible. */ + private final Map> seeds = new LinkedHashMap<>(); + /** Reaching one of these means "the value left through the return". */ + private final Set sinks = new LinkedHashSet<>(); + + private final List sites = new ArrayList<>(); + + Fn(JCallable callable) { + this.callable = callable; + } + } + + /** Indexes every callable's static facts, keyed by callable id. */ + private static Map index(Map modules) { + Map fns = new LinkedHashMap<>(); + for (JModule module : modules.values()) { + // Decoded once per module, not per node: `span.bytes` are offsets into this array. + byte[] source = + module.getSource() == null ? null : module.getSource().getBytes(StandardCharsets.UTF_8); + walkTypes(module.getTypes(), source, fns); + } + return fns; + } + + /** Mirrors {@link SdgVertices}'s traversal (nested types, then each callable's local types). */ + private static void walkTypes(Map types, byte[] source, Map fns) { + for (JType t : types.values()) { + walkTypes(t.getTypes(), source, fns); + for (JCallable c : t.getCallables().values()) { + if (c.getId() != null) { + fns.put(c.getId(), facts(c, source)); + } + walkTypes(c.getTypes(), source, fns); + } + } + } + + /** Stage 1's syntactic half: the reachability graph, the sinks, the call sites and the seeds. */ + private static Fn facts(JCallable c, byte[] source) { + Fn fn = new Fn(c); + if (c.getDdg() != null) { + for (JDdgEdge e : c.getDdg()) { + fn.graph.computeIfAbsent(e.getSrc(), k -> new ArrayList<>()).add(e.getDst()); + } + } + + Map body = c.getBody(); + Map spanText = new LinkedHashMap<>(); + for (Map.Entry entry : body.entrySet()) { + JBodyNode node = entry.getValue(); + String text = slice(source, node.getSpan()); + if (text != null) { + spanText.put(entry.getKey(), text); + } + if ("return".equals(node.getKind())) { + fn.sinks.add(entry.getKey()); + } else if ("call".equals(node.getKind())) { + // Where a call's value goes next. Deliberately outside the resolved/in-project filter + // below: an unresolved or external call is still a node the chain has to pass + // *through*. `int t = Math.abs(id(a));` encloses the `id(a)` site in the `Math.abs` + // node, which is an orphan (not a CFG node, so no ddg edge leaves it) and has no + // summary of its own — filtering it out here would strand the hop and silently drop + // the caller's summary edge, an under-approximation the L4 posture does not allow. + String outer = enclosing(entry.getKey(), body); + if (outer != null) { + fn.graph.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(outer); + } + if (node.getCallee() != null && !node.getCallee().contains("/@external/")) { + fn.sites.add(new Site( + entry.getKey(), + node.getCallee(), + node.getArgumentExpr(), + body.containsKey(entry.getKey() + "/actual_out"))); + } + } + } + if (body.containsKey("@formal_out")) { + fn.sinks.add("@formal_out"); + } + + // The other half of that route: a resolved call's returned value reaches the call node. Only + // resolved sites have an actual_out vertex to leave from, so unlike the hop above this one + // does belong inside the filter. Together they give actual_out → call node → enclosing node, + // which is the CFG node the ddg edges actually leave from — the `ddg` does not hang off a + // call that sits inside a larger expression, since `DdgBuilder` analyses CFG nodes and such a + // call is not one (`int t = f(x);` puts the def-use edge on the *statement*). + for (Site site : fn.sites) { + fn.graph.computeIfAbsent(site.actualOut(), k -> new ArrayList<>()).add(site.local); + } + + List params = c.getParameters(); + for (int i = 0; i < params.size(); i++) { + Set seeds = seedsFor(fn, params.get(i).getName(), spanText); + if (!seeds.isEmpty()) { + fn.seeds.put(i, seeds); + } + } + return fn; + } + + /** + * Where a parameter's value is visible, syntactically: the def end of any {@code ddg} edge whose + * access path is rooted at it, any call-site {@code actual_in} vertex whose argument text names + * it, and every body node whose source text names it (see the class javadoc — a parameter + * has no defining node in the {@code ddg}, so text is the only thing that can put it on the map at + * all). + * + *

Every kind with a span is seedable, containers included. A {@code branch}/{@code loop}/ + * {@code switch} span swallows the statements nested inside it, so seeding one on a name mentioned + * anywhere within hands the parameter that whole node's def-use reach — bulk over-approximation, + * which this posture explicitly accepts. Excluding them is the direction that is not + * allowed: those nodes carry real defs and uses of their own ({@code DdgBuilder.collect} attributes + * a {@code for}-each's loop variable and iterated expression, and an {@code if}'s condition, to the + * container node), while the header they come from gets no node of its own — so + * {@code int first(int[] q) { for (int x : q) { return x; } return 0; }} names {@code q} nowhere a + * non-container node can see, and excluding the loop drops a real flow. {@code entry}/{@code exit} + * and the synthetic L4 vertices need no rule: they carry no span, so {@link #slice} skips them. + */ + private static Set seedsFor(Fn fn, String name, Map spanText) { + Set seeds = new LinkedHashSet<>(); + if (name == null || name.isEmpty()) { + return seeds; + } + // Spelling, not binding — this is the syntactic seeding the L4 design settles for, and every + // way it can over-credit is a may-flow direction the weak-update posture accepts. It credits + // `name` when it is merely mentioned rather than used (`return other(x) + 1;`, `log(x);`); + // when the mention is a *shadowing* local or catch parameter of the same spelling, since this + // reads text and not scopes (`{ int x = 0; sink(x); }` in a method taking `x`); when it is a + // *field* of something else, since `.` is not a word character (`return a.z;` credits + // parameter `z`); when it appears inside a string literal (`return "x marks";` credits `x`); + // and when it sits next to `$`, which Java allows in identifiers but regex `\b` does not + // treat as a word character (`a$b` credits `a`). None of these can lose a real flow. + Pattern word = Pattern.compile("\\b" + Pattern.quote(name) + "\\b"); + + List ddg = fn.callable.getDdg(); + if (ddg != null) { + for (JDdgEdge e : ddg) { + if (name.equals(base(e.getVar()))) { + seeds.add(e.getSrc()); + } + } + } + for (Site site : fn.sites) { + for (int j = 0; j < site.args.size(); j++) { + String vertex = site.actualIn(j); + if (site.args.get(j) != null + && word.matcher(site.args.get(j)).find() + && fn.callable.getBody().containsKey(vertex)) { + seeds.add(vertex); + } + } + } + for (Map.Entry node : spanText.entrySet()) { + if (word.matcher(node.getValue()).find()) { + seeds.add(node.getKey()); + } + } + return seeds; + } + + /** + * The smallest body node whose span strictly encloses {@code local}'s — the statement, outer + * call, or enclosing {@code branch}/{@code loop}/{@code switch} a nested call sits inside. Null + * only when nothing encloses it: a call in statement position at the top level of the + * body, where the call node is itself the CFG node and needs no hop. A call in statement position + * inside an {@code if}/{@code for}/{@code while} body does get a hop, to that branch or + * loop node, whose span covers the whole statement — reach then continues along its ddg + * out-edges, which over-approximates (the condition's dependences are not the call's) in the + * may-flow direction the L4 posture accepts. Ties break on the node id so the choice cannot + * depend on map order. + */ + private static String enclosing(String local, Map body) { + int[] inner = bytes(body.get(local)); + if (inner == null) { + return null; + } + String best = null; + long bestWidth = Long.MAX_VALUE; + for (Map.Entry entry : body.entrySet()) { + int[] outer = bytes(entry.getValue()); + if (entry.getKey().equals(local) + || outer == null + || outer[0] > inner[0] + || outer[1] < inner[1] + || (outer[0] == inner[0] && outer[1] == inner[1])) { + continue; + } + long width = (long) outer[1] - outer[0]; + if (best == null || width < bestWidth || (width == bestWidth && entry.getKey().compareTo(best) < 0)) { + best = entry.getKey(); + bestWidth = width; + } + } + return best; + } + + /** A node's {@code span.bytes}, or null when it has none (every synthetic vertex). */ + private static int[] bytes(JBodyNode node) { + if (node == null || node.getSpan() == null || node.getSpan().getBytes() == null) { + return null; + } + return node.getSpan().getBytes().length < 2 ? null : node.getSpan().getBytes(); + } + + /** An access path's base segment — the text before the first {@code .} or {@code [}. */ + private static String base(String var) { + if (var == null) { + return null; + } + int dot = var.indexOf('.'); + int bracket = var.indexOf('['); + int cut = dot < 0 ? bracket : (bracket < 0 ? dot : Math.min(dot, bracket)); + return cut < 0 ? var : var.substring(0, cut); + } + + /** The UTF-8 byte slice a node's {@code span} names, or null when the span is unusable. */ + private static String slice(byte[] source, Span span) { + if (source == null || span == null || span.getBytes() == null || span.getBytes().length < 2) { + return null; + } + int from = span.getBytes()[0]; + int to = span.getBytes()[1]; + if (from < 0 || to > source.length || from >= to) { + return null; + } + return new String(source, from, to - from, StandardCharsets.UTF_8); + } + + // ----- stage 2: the bottom-up SCC fixpoint --------------------------------------------------- + + /** + * Solves every callable's {@code flows} relation, callees first. Within one component each member + * is recomputed until none grows. Termination is structural: a member's seeds, sinks and + * reachability graph are fixed, the only other edges — the callee bridges — appear as callee + * relations grow, and the result is unioned in rather than replaced, so relations only grow; + * {@code flows(c) ⊆ {0 … arity(c)-1}} bounds it, so a component settles in at most (its members' + * total arity + 1) rounds. + */ + private static Map> solve(Map fns, List callGraph) { + Map> flows = new LinkedHashMap<>(); + for (List component : Scc.condense(fns.keySet(), callGraph)) { + boolean changed = true; + while (changed) { + changed = false; + for (String id : component) { + Fn fn = fns.get(id); + if (fn == null || fn.seeds.isEmpty()) { + continue; // no seed, no flow — and seeds never appear later + } + Set reaching = computeFlows(fn, fns, flows); + changed |= flows.computeIfAbsent(id, k -> new TreeSet<>()).addAll(reaching); + } + } + } + return flows; + } + + /** One iteration of a callable's transfer relation: which parameters reach a return sink. */ + private static Set computeFlows(Fn fn, Map fns, Map> flows) { + // Built once per callable per round, then walked once per parameter. Successor lists are + // copied, not shared: the bridges below would otherwise accumulate into fn.graph each round. + Map> graph = new LinkedHashMap<>(); + fn.graph.forEach((node, next) -> graph.put(node, new ArrayList<>(next))); + for (Site site : fn.sites) { + for (int j : passThrough(site, fns, flows)) { + graph.computeIfAbsent(site.actualIn(j), k -> new ArrayList<>()).add(site.actualOut()); + } + } + + Set reaching = new TreeSet<>(); + for (Map.Entry> seed : fn.seeds.entrySet()) { + if (reaches(graph, seed.getValue(), fn.sinks)) { + reaching.add(seed.getKey()); + } + } + return reaching; + } + + /** BFS from {@code seeds}; true as soon as a sink is touched. */ + private static boolean reaches(Map> graph, Set seeds, Set sinks) { + Set seen = new LinkedHashSet<>(seeds); + Deque queue = new ArrayDeque<>(seeds); + while (!queue.isEmpty()) { + String node = queue.poll(); + if (sinks.contains(node)) { + return true; + } + for (String next : graph.getOrDefault(node, List.of())) { + if (seen.add(next)) { + queue.add(next); + } + } + } + return false; + } + + /** + * The argument positions of {@code site} the callee hands back through its return, per its + * relation as currently known — empty for a callee still being solved in this same SCC, + * which the next round then picks up. An argument past the callee's last formal collapses onto + * it, matching how {@link SdgVertices} wires varargs {@code param_in} edges. + */ + private static List passThrough(Site site, Map fns, Map> flows) { + Set calleeFlows = flows.get(site.callee); + Fn callee = fns.get(site.callee); + if (!site.hasActualOut || calleeFlows == null || callee == null) { + return List.of(); + } + int arity = callee.callable.getParameters().size(); + List out = new ArrayList<>(); + for (int j = 0; arity > 0 && j < site.args.size(); j++) { + if (calleeFlows.contains(Math.min(j, arity - 1))) { + out.add(j); + } + } + return out; + } + + // ----- stage 3: emission --------------------------------------------------------------------- + + /** Writes each caller's shortcut edges; a caller with none keeps {@code summary} absent. */ + private static void emit(Map fns, Map> flows) { + for (Fn fn : fns.values()) { + List summary = new ArrayList<>(); + for (Site site : fn.sites) { + for (int j : passThrough(site, fns, flows)) { + if (fn.callable.getBody().containsKey(site.actualIn(j))) { + summary.add(edge(site.actualIn(j), site.actualOut())); + } + } + } + if (!summary.isEmpty()) { + summary.sort(Comparator.comparing(JIdEdge::getSrc).thenComparing(JIdEdge::getDst)); + fn.callable.setSummary(summary); + } + } + } + + private static JIdEdge edge(String src, String dst) { + JIdEdge e = new JIdEdge(); + e.setSrc(src); + e.setDst(dst); + return e; + } +} diff --git a/src/main/java/com/ibm/cldk/wala/WalaAnalysis.java b/src/main/java/com/ibm/cldk/wala/WalaAnalysis.java index 90da386a..358b8ca5 100644 --- a/src/main/java/com/ibm/cldk/wala/WalaAnalysis.java +++ b/src/main/java/com/ibm/cldk/wala/WalaAnalysis.java @@ -22,6 +22,7 @@ import com.ibm.wala.cast.java.translator.jdt.ecj.ECJClassLoaderFactory; import com.ibm.wala.classLoader.IBytecodeMethod; import com.ibm.wala.classLoader.IMethod; +import com.ibm.wala.classLoader.Language; import com.ibm.wala.ipa.callgraph.AnalysisCacheImpl; import com.ibm.wala.ipa.callgraph.AnalysisOptions; import com.ibm.wala.ipa.callgraph.AnalysisOptions.ReflectionOptions; @@ -30,6 +31,7 @@ import com.ibm.wala.ipa.callgraph.CallGraph; import com.ibm.wala.ipa.callgraph.CallGraphBuilder; import com.ibm.wala.ipa.callgraph.IAnalysisCacheView; +import com.ibm.wala.ipa.callgraph.impl.PartialCallGraph; import com.ibm.wala.ipa.callgraph.impl.Util; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; @@ -43,12 +45,15 @@ import com.ibm.wala.util.intset.MutableMapping; import com.ibm.wala.util.intset.MutableSparseIntSet; import com.ibm.wala.util.intset.OrdinalSet; +import com.ibm.wala.util.intset.OrdinalSetMapping; import java.io.PrintStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.apache.commons.io.output.NullOutputStream; /** @@ -129,6 +134,20 @@ private WalaAnalysis( * skip building and stream pre-compiled {@code .class} files directly) */ public static Optional of(String input, String dependencies, String build) { + return of(input, dependencies, build, "rta"); + } + + /** + * As {@link #of(String, String, String)}, but with an explicit pointer-analysis precision: + * {@code "0-cfa"} and {@code "0-1-cfa"} select WALA's context-insensitive propagation builders, + * anything else (including {@code null}) keeps the default RTA builder. + * + *

Precision is chosen here rather than later because it is a property of the call-graph + * builder: the pointer analysis retained for L4's mod/ref closure is the one this builder + * produced. + */ + public static Optional of( + String input, String dependencies, String build, String precision) { // Mirror RtaCallGraph.endpoints: default the global so BuildProject's static initializer // does not NPE when it resolves the build-tool wrappers. if (CodeAnalyzer.projectRootPom == null) { @@ -152,7 +171,17 @@ public static Optional of(String input, String dependencies, Strin IAnalysisCacheView cache = new AnalysisCacheImpl(AstIRFactory.makeDefaultFactory(), options.getSSAOptions()); - CallGraphBuilder builder = Util.makeRTABuilder(options, cache, cha); + CallGraphBuilder builder; + switch (precision == null ? "rta" : precision.toLowerCase()) { + case "0-cfa": + builder = Util.makeZeroCFABuilder(Language.JAVA, options, cache, cha); + break; + case "0-1-cfa": + builder = Util.makeZeroOneCFABuilder(Language.JAVA, options, cache, cha); + break; + default: + builder = Util.makeRTABuilder(options, cache, cha); + } CallGraph cg = builder.makeCallGraph(options, null); PointerAnalysis pa = builder.getPointerAnalysis(); @@ -206,6 +235,36 @@ public PDG pdgFor(CGNode node) { modRef); } + /** + * Replaces the empty-defaulting mod/ref entries with a real interprocedural closure, computed + * over the application-scope subgraph only. This is what turns L3's intraprocedural + * heap dependences into L4's semantic ones: with real mod/ref, {@link PDG} materializes the + * heap parameter statements at call sites that {@link #emptyDefaultingMap} suppresses. + * + *

Scope, not laziness, is the point. The global closure over a JDK-inclusive call graph is + * the recorded 4 GB OOM (see {@link #emptyDefaultingMap}); {@link PartialCallGraph} restricts + * both the per-node scan and the bottom-up fixpoint — and with them the {@code PointerKey} + * domain the bit vectors are sized by — to application nodes. Library nodes keep the empty + * defaulting entry, so heap flow mediated by a library frame is conservatively absent. + * + *

Nodes whose closure came back empty are stored as an empty set in the computed + * domain rather than left to the default: {@code PDG.unionHeapLocations} both reads the + * backing set (which is null on an empty {@code BitVectorVariable}) and takes its result + * mapping from the PDG's own node, so every primed node must carry a non-null backing set and + * the same {@link OrdinalSetMapping} as its callees. + */ + public void primeL4ModRef() { + Set appNodes = new LinkedHashSet<>(); + for (CGNode node : callGraph) { + if (AnalysisUtils.isApplicationClass(node.getMethod().getDeclaringClass())) { + appNodes.add(node); + } + } + CallGraph appOnly = PartialCallGraph.make(callGraph, appNodes, appNodes); + prime(emptyMod, modRef.computeMod(appOnly, pa)); + prime(emptyRef, modRef.computeRef(appOnly, pa)); + } + /** * The RTA call graph's edges as endpoint pairs. Delegates to * {@link RtaCallGraph#toEndpoints(CallGraph)} so the L2 {@code rta} overlay is byte-identical @@ -217,6 +276,33 @@ public List rtaEndpoints() { // ----- helpers ------------------------------------------------------------------------------ + /** + * Copies {@code computed} into {@code target}, substituting a non-null-backed empty set for + * every entry whose backing set is null (see {@link #primeL4ModRef}). When no entry carries a + * domain at all — no application node touches the heap — nothing is written, leaving the + * empty-defaulting behaviour exactly as it was. + */ + private static void prime( + Map> target, + Map> computed) { + OrdinalSetMapping domain = null; + for (OrdinalSet set : computed.values()) { + if (set.getBackingSet() != null) { + domain = set.getMapping(); + break; + } + } + if (domain == null) { + return; + } + OrdinalSet emptyInDomain = + new OrdinalSet<>(MutableSparseIntSet.makeEmpty(), domain); + for (Map.Entry> entry : computed.entrySet()) { + OrdinalSet set = entry.getValue(); + target.put(entry.getKey(), set.getBackingSet() != null ? set : emptyInDomain); + } + } + private static List buildApplicationMethods(CallGraph callGraph) { List result = new ArrayList<>(); for (CGNode node : callGraph) { diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index f3065151..7c960caf 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -48,12 +48,16 @@ private static int run(String... args) { /** * The pre-existing CLI options on {@link CodeAnalyzer} are static, so a value set by one test - * would leak into the next. Reset the ones these tests touch so each case starts from defaults. + * would leak into the next. Reset the ones these tests touch so each case starts from defaults — + * and again afterwards, so whichever test happens to run last cannot leak {@code emit=neo4j} or a + * deliberately bad {@code --precision} into another test class that drives the same CLI. */ @BeforeEach + @AfterEach void resetStaticOptions() throws Exception { set("emit", "json"); set("analysisLevel", 1); + set("precision", "rta"); set("output", null); set("input", null); set("targetFiles", null); @@ -167,10 +171,10 @@ void emitSchemaAlwaysEmitsTheV2Catalog(@TempDir Path tmp) throws IOException { assertEquals(0, run("--emit", "schema", "-o", out.toString())); JsonObject doc = JsonParser.parseString(Files.readString(out.resolve("schema.neo4j.json"))) .getAsJsonObject(); - assertEquals("2.0.0", doc.get("schema_version").getAsString()); + assertEquals("2.1.0", doc.get("schema_version").getAsString()); assertEquals(0, run("--emit", "schema", "-o", out.toString(), "--schema", "v1"), "--emit schema ignores --schema"); - assertEquals("2.0.0", JsonParser.parseString(Files.readString(out.resolve("schema.neo4j.json"))) + assertEquals("2.1.0", JsonParser.parseString(Files.readString(out.resolve("schema.neo4j.json"))) .getAsJsonObject().get("schema_version").getAsString()); } @@ -277,13 +281,6 @@ void v2AtAnalysisLevelThreeEmitsDataflowOverlays(@TempDir Path tmp) throws IOExc assertTrue(json.contains("\"cfg\""), "level 3 lays the cfg overlay on callables"); } - @Test - void v2WithAnalysisLevelAboveThreeFailsLoudly(@TempDir Path tmp) throws IOException { - Path in = project(tmp.resolve("app")); - assertNotEquals(0, run("-i", in.toString(), "--schema", "v2", "-a", "4"), - "L4 is not implemented; asking for a level beyond 3 must be an error, not an L3 payload"); - } - @Test void v2WalaL3EngineDegradesClearlyWhenBuildAbsent(@TempDir Path tmp) throws IOException { // No class files present — WALA cannot build the call graph; must exit 0 with declared @@ -445,4 +442,38 @@ void v2ExternalCallsFlagHomesOutOfProjectTargets(@TempDir Path tmp) throws IOExc "--external-calls homes out-of-project targets (e.g. java.lang.Math)"); } + @Test + void analysisLevelFourIsAccepted(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "-a", "4", "--no-build"), + "level 4 must run; WALA-unavailable degrades, never crashes"); + JsonObject root = JsonParser.parseString(Files.readString(out.resolve("analysis.json"))).getAsJsonObject(); + assertEquals(4, root.get("max_level").getAsInt(), + "max_level is the requested level: the L4 vertices/param edges are engine-free and " + + "still ran, even though the semantic ddg (WALA-dependent) degraded"); + } + + @Test + void analysisLevelFiveStillFailsLoudly(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "-a", "5")); + } + + @Test + void unknownPrecisionFailsLoudly(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "-a", "4", "--precision", "2-cfa"), + "unrecognized flag values exit non-zero (CLI contract)"); + } + + @Test + void unknownPrecisionFailsLoudlyUnderEmitNeo4jToo(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + // --emit neo4j raises the level to 4 itself, and the precision check is level-gated: if it ran + // before the raise, this would exit 0 and silently fall back to RTA. + assertNotEquals(0, run("-i", in.toString(), "--emit", "neo4j", "--precision", "2-cfa"), + "unrecognized flag values exit non-zero (CLI contract), --emit neo4j included"); + } + } diff --git a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java index f214706a..92120978 100644 --- a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java @@ -26,6 +26,8 @@ import com.ibm.cldk.schema.V2Emitter; import com.ibm.cldk.syntactic_analysis.L1Extractor; import com.ibm.cldk.syntactic_analysis.L2CallGraph; +import com.ibm.cldk.syntactic_analysis.dataflow.SdgVertices; +import com.ibm.cldk.syntactic_analysis.dataflow.SummaryPass; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; @@ -38,15 +40,19 @@ import org.junit.jupiter.api.Test; /** - * Schema v2 graph conformance (no container needed): run the real L1–L3 pipeline over a fixture, - * project with {@link V2GraphProjector}, and assert the projector only ever produces what - * {@link V2SchemaCatalog} declares — the anti-drift guard for the 2.0.0 graph contract. Also pins + * Schema v2 graph conformance (no container needed): run the real L1–L3 pipeline plus the L4 SDG + * passes ({@link SdgVertices}, {@link SummaryPass}) over a fixture, project with + * {@link V2GraphProjector}, and assert the projector only ever produces what + * {@link V2SchemaCatalog} declares — the anti-drift guard for the 2.1.0 graph contract. Also pins * the convergence decisions: body nodes instead of call-site nodes, and the {@code _k}-keyed * CFG/DDG relationships. */ public class V2Neo4jSchemaConformanceTest { - private static final Path FIXTURE = Paths.get("src/test/resources/test-applications/call-graph-test"); + // l4-sdg-test (not call-graph-test): its calls are all 1-arg with a transitive a→b→c chain, so + // J_PARAM_IN/J_SUMMARY are guaranteed non-empty here. The v1/v2 conformance tests still exercise + // call-graph-test. + private static final Path FIXTURE = Paths.get("src/test/resources/test-applications/l4-sdg-test"); private static GraphRows rows; @@ -65,11 +71,14 @@ static void project() throws Exception { REL_BY_TYPE.put(rt.type, rt); } Map modules = L1Extractor.extractAll( - FIXTURE, "call-graph-test", null, new LinkedHashMap<>(), 3, 3, "ast"); - L2CallGraph.Result l2 = L2CallGraph.build("call-graph-test", modules, null, true); + FIXTURE, "l4-sdg-test", null, new LinkedHashMap<>(), 3, 3, "ast"); + L2CallGraph.Result l2 = L2CallGraph.build("l4-sdg-test", modules, null, true); + SdgVertices.Result sdg = SdgVertices.apply(modules); + SummaryPass.apply(modules, l2.callGraph(), 3); Analysis analysis = V2Emitter.emit( - "call-graph-test", 3, modules, "test", l2.callGraph(), l2.externalSymbols()); - rows = V2GraphProjector.project(analysis, "call-graph-test"); + "l4-sdg-test", 3, modules, "test", l2.callGraph(), l2.externalSymbols(), + sdg.paramIn, sdg.paramOut); + rows = V2GraphProjector.project(analysis, "l4-sdg-test"); } private static String specificLabel(List labels) { @@ -163,7 +172,7 @@ public void l3OverlayEdgesAreKeyedAndPresent() { @Test public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { - String cypher = CypherWriter.renderCypher(rows, "call-graph-test"); + String cypher = CypherWriter.renderCypher(rows, "l4-sdg-test"); assertTrue(cypher.contains("J_HAS_UNIT|J_HAS_MODULE"), "the wipe must traverse both generations' unit relationship"); assertTrue(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), @@ -174,4 +183,19 @@ public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { "wipe/prune descendant traversal must include " + rel); } } + + @Test + void l4OverlayProjectsParamAndSummaryEdges() { + boolean paramIn = false; + boolean paramOut = false; + boolean summary = false; + for (EdgeRow edge : rows.edges) { + paramIn |= edge.type.equals("J_PARAM_IN"); + paramOut |= edge.type.equals("J_PARAM_OUT"); + summary |= edge.type.equals("J_SUMMARY"); + } + assertTrue(paramIn, "J_PARAM_IN projected from application param_in"); + assertTrue(paramOut, "J_PARAM_OUT projected from application param_out"); + assertTrue(summary, "J_SUMMARY projected from callable summaries"); + } } diff --git a/src/test/java/com/ibm/cldk/schema/L4GateTest.java b/src/test/java/com/ibm/cldk/schema/L4GateTest.java new file mode 100644 index 00000000..9d28876a --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/L4GateTest.java @@ -0,0 +1,266 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +/** Spec §14 L4 gate over the l4-sdg-test fixture. */ +class L4GateTest { + + private static JsonObject root; + + @BeforeAll + static void analyze(@TempDir Path tmp) throws Exception { + int exit = new CommandLine(new com.ibm.cldk.CodeAnalyzer()).execute( + "-i", "src/test/resources/test-applications/l4-sdg-test", + "-a", "4", "-o", tmp.toString()); + assertEquals(0, exit); + root = JsonParser.parseString(Files.readString(tmp.resolve("analysis.json"))).getAsJsonObject(); + } + + @Test + void maxLevelIsFour() { + assertEquals(4, root.get("max_level").getAsInt()); + } + + @Test + void paramEdgeAritiesMatchAndNothingDangles() { + JsonObject app = root.getAsJsonObject("application"); + assertTrue(app.has("param_in") && app.has("param_out")); + + // Collect every global body-node ordinal actually emitted. + Set ordinals = new HashSet<>(); + collectOrdinals(app.getAsJsonObject("symbol_table"), ordinals); + + for (String key : new String[] {"param_in", "param_out"}) { + for (var e : app.getAsJsonArray(key)) { + JsonObject o = e.getAsJsonObject(); + assertTrue(ordinals.contains(o.get("src").getAsString()), key + " src dangles: " + o); + assertTrue(ordinals.contains(o.get("dst").getAsString()), key + " dst dangles: " + o); + } + } + + // Chain.a calls b(1 arg): exactly one actual_in:0 → b@formal_in:0 edge exists. + int found = 0; + for (var e : app.getAsJsonArray("param_in")) { + String dst = e.getAsJsonObject().get("dst").getAsString(); + if (dst.endsWith("/Chain/b(int)@formal_in:0")) { + found++; + } + } + assertEquals(1, found, "exactly one a→b param_in edge"); + } + + /** + * §14's "arity matches" as an actual count. Hand-derived from the four fixture files and + * confirmed against this run: + * + *

    + *
  • {@code param_in} = 6 — one per argument at each of the six in-project call sites with + * arguments: {@code a→b}, {@code b→c}, {@code even→odd}, {@code odd→even}, + * {@code roundTrip→put}, {@code callFirst→first}. {@code roundTrip→get()} passes none, so + * it contributes none. + *
  • {@code param_out} = 6 — one per site whose callee returns a value: the same four + * {@code Chain}/{@code Mutual} sites, plus {@code callFirst→first} and + * {@code roundTrip→get()}; {@code put} is {@code void}, so that site has no + * {@code actual_out} to reach. + *
  • {@code summary} = 5 — {@code Chain.a}, {@code Chain.b}, {@code Mutual.even}, + * {@code Mutual.odd} and {@code Loops.callFirst}, one shortcut each. + * {@code Heap.roundTrip}'s two sites are a void callee and a no-arg callee, so neither can + * carry one. {@code Loops.callFirst}'s is the one that needs container nodes to be + * seedable: {@code first}'s parameter is named only in its {@code for}-each header. + *
+ */ + @Test + void overlayCountsAreExactlyWhatTheFixtureImplies() { + JsonObject app = root.getAsJsonObject("application"); + assertEquals(6, app.getAsJsonArray("param_in").size(), "param_in: one per argument at a resolved site"); + assertEquals(6, app.getAsJsonArray("param_out").size(), "param_out: one per value-returning site"); + + int summaries = 0; + for (JsonObject c : callablesById(root).values()) { + JsonArray summary = c.getAsJsonArray("summary"); + summaries += summary == null ? 0 : summary.size(); + } + assertEquals(5, summaries, "summary: one shortcut per pass-through call site"); + } + + @Test + void semanticDdgAddsToNotReplacesSsa() { + JsonObject heap = callable(root, "Heap", "roundTrip(int)"); + JsonArray ddg = heap.getAsJsonArray("ddg"); + assertNotNull(ddg); + boolean ssa = false; + boolean pts = false; + for (var e : ddg) { + String prov = e.getAsJsonObject().getAsJsonArray("prov").toString(); + ssa |= prov.contains("ssa"); + pts |= prov.contains("points-to"); + } + assertTrue(ssa, "L3 ssa edges survive"); + // points-to presence requires a successful WALA build of the fixture; this environment has + // no gradle on PATH, so the CLI's `auto` build fails and the run degrades (Ruling R4). Guard + // rather than assert, so the gate still runs for real wherever a build is available. + Assumptions.assumeTrue(pts, "points-to edges require a working WALA build; none produced here"); + } + + @Test + void summaryEdgeExistsForTheKnownTransitiveFlow() { + JsonObject a = callable(root, "Chain", "a(int)"); + assertTrue(a.has("summary"), "caller carries summary edges"); + assertEquals(1, a.getAsJsonArray("summary").size()); + } + + @Test + void monotonicOverL3(@TempDir Path tmp) throws Exception { + int exit = new CommandLine(new com.ibm.cldk.CodeAnalyzer()).execute( + "-i", "src/test/resources/test-applications/l4-sdg-test", + "-a", "3", "-o", tmp.toString()); + assertEquals(0, exit); + JsonObject l3 = JsonParser.parseString(Files.readString(tmp.resolve("analysis.json"))).getAsJsonObject(); + + // Every callable, not a sample of one: L4 only ever adds, so the whole L3 payload has to + // survive it — body nodes, cfg and ddg alike. + Map l3Callables = callablesById(l3); + Map l4Callables = callablesById(root); + assertFalse(l3Callables.isEmpty(), "the -a 3 run produced callables to compare against"); + for (Map.Entry entry : l3Callables.entrySet()) { + String id = entry.getKey(); + JsonObject before = entry.getValue(); + JsonObject after = l4Callables.get(id); + assertNotNull(after, "L3 callable survives into L4: " + id); + for (String overlay : new String[] {"cfg", "cdg", "ddg"}) { + JsonArray l3Edges = before.getAsJsonArray(overlay); + if (l3Edges == null) { + continue; // no fact at L3 is nothing to preserve + } + JsonArray l4Edges = after.getAsJsonArray(overlay); + assertNotNull(l4Edges, id + " loses its " + overlay + " at L4"); + for (var e : l3Edges) { + assertTrue(l4Edges.contains(e), "L3 " + overlay + " ⊆ L4 " + overlay + " in " + id + ": " + e); + } + } + JsonObject l3Body = before.getAsJsonObject("body"); + if (l3Body == null) { + continue; + } + for (String key : l3Body.keySet()) { + assertTrue(after.getAsJsonObject("body").has(key), "L3 body node survives: " + id + " " + key); + } + assertFalse(l3Body.keySet().stream().anyMatch(k -> k.contains("formal") || k.contains("actual")), + "no L4 vertices leak into -a 3: " + id); + } + } + + // ----- helpers ------------------------------------------------------------------------------ + + /** Walks {@code symbol_table → types → callables}, matching a type whose {@code id} ends with + * {@code "/" + typeSuffix}, then returns the callable keyed by {@code sigKey} on that type. */ + private static JsonObject callable(JsonObject root, String typeSuffix, String sigKey) { + JsonObject symbolTable = root.getAsJsonObject("application").getAsJsonObject("symbol_table"); + for (Map.Entry fileEntry : symbolTable.entrySet()) { + JsonObject type = findType(fileEntry.getValue().getAsJsonObject().getAsJsonObject("types"), typeSuffix); + if (type != null) { + return type.getAsJsonObject("callables").getAsJsonObject(sigKey); + } + } + throw new AssertionError("no type found with suffix /" + typeSuffix); + } + + /** Every callable in a document, keyed by its {@code id} (nested and callable-local types included). */ + private static Map callablesById(JsonObject doc) { + Map byId = new LinkedHashMap<>(); + JsonObject symbolTable = doc.getAsJsonObject("application").getAsJsonObject("symbol_table"); + for (Map.Entry fileEntry : symbolTable.entrySet()) { + collectCallables(fileEntry.getValue().getAsJsonObject().getAsJsonObject("types"), byId); + } + return byId; + } + + private static void collectCallables(JsonObject types, Map byId) { + if (types == null) { + return; + } + for (Map.Entry typeEntry : types.entrySet()) { + JsonObject type = typeEntry.getValue().getAsJsonObject(); + collectCallables(type.getAsJsonObject("types"), byId); + JsonObject callables = type.getAsJsonObject("callables"); + if (callables == null) { + continue; + } + for (Map.Entry callableEntry : callables.entrySet()) { + JsonObject callable = callableEntry.getValue().getAsJsonObject(); + byId.put(callable.get("id").getAsString(), callable); + collectCallables(callable.getAsJsonObject("types"), byId); + } + } + } + + /** Depth-first search of a {@code types} map (and its nested {@code types}) for an id match. */ + private static JsonObject findType(JsonObject types, String typeSuffix) { + if (types == null) { + return null; + } + for (Map.Entry typeEntry : types.entrySet()) { + JsonObject type = typeEntry.getValue().getAsJsonObject(); + if (type.get("id").getAsString().endsWith("/" + typeSuffix)) { + return type; + } + JsonObject nested = findType(type.getAsJsonObject("types"), typeSuffix); + if (nested != null) { + return nested; + } + } + return null; + } + + /** Every callable's global-ordinal body-node ids, per the rule {@code local.startsWith("@") ? + * id + local : id + "@" + local} (matches {@code SdgVertices.global} / {@code V2GraphProjector}). */ + private static void collectOrdinals(JsonObject symbolTable, Set ordinals) { + for (Map.Entry fileEntry : symbolTable.entrySet()) { + collectOrdinalsFromTypes(fileEntry.getValue().getAsJsonObject().getAsJsonObject("types"), ordinals); + } + } + + private static void collectOrdinalsFromTypes(JsonObject types, Set ordinals) { + if (types == null) { + return; + } + for (Map.Entry typeEntry : types.entrySet()) { + JsonObject type = typeEntry.getValue().getAsJsonObject(); + collectOrdinalsFromTypes(type.getAsJsonObject("types"), ordinals); + JsonObject callables = type.getAsJsonObject("callables"); + if (callables == null) { + continue; + } + for (Map.Entry callableEntry : callables.entrySet()) { + JsonObject callable = callableEntry.getValue().getAsJsonObject(); + String id = callable.get("id").getAsString(); + JsonObject body = callable.getAsJsonObject("body"); + if (body == null) { + continue; + } + for (String local : body.keySet()) { + ordinals.add(local.startsWith("@") ? id + local : id + "@" + local); + } + } + } + } +} diff --git a/src/test/java/com/ibm/cldk/schema/L4ModelSerializationTest.java b/src/test/java/com/ibm/cldk/schema/L4ModelSerializationTest.java new file mode 100644 index 00000000..6a3348b3 --- /dev/null +++ b/src/test/java/com/ibm/cldk/schema/L4ModelSerializationTest.java @@ -0,0 +1,52 @@ +package com.ibm.cldk.schema; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** L4 model fields serialize under their canonical names and are absent (not empty) below L4. */ +class L4ModelSerializationTest { + + @Test + void syntheticVertexFieldsSerializeAsOfAndParent() { + JBodyNode n = new JBodyNode(); + n.setKind("actual_in"); + n.setOf("arg0"); + n.setParent("5:16"); + String json = V2Json.compact().toJson(n); + assertTrue(json.contains("\"of\":\"arg0\""), json); + assertTrue(json.contains("\"parent\":\"5:16\""), json); + } + + @Test + void applicationCarriesParamEdgesOnlyWhenSet() { + JApplication app = new JApplication(); + app.setId("can://java/x"); + assertFalse(V2Json.compact().toJson(app).contains("param_in"), + "absent means no fact — no empty lists below L4"); + + JIdEdge e = new JIdEdge(); + e.setSrc("can://java/x/f.java/A/a(int)@3:16/actual_in:0"); + e.setDst("can://java/x/f.java/A/b(int)@formal_in:0"); + app.setParamIn(List.of(e)); + String json = V2Json.compact().toJson(app); + assertTrue(json.contains("\"param_in\""), json); + assertTrue(json.contains("actual_in:0"), json); + } + + @Test + void emitterOverloadAttachesParamEdges() { + JIdEdge in = new JIdEdge(); + in.setSrc("s"); + in.setDst("d"); + Analysis a = V2Emitter.emit("app", 4, Map.of(), "test", null, null, List.of(in), List.of()); + assertEquals(4, a.getMaxLevel()); + assertEquals(1, a.getApplication().getParamIn().size()); + assertTrue(a.getApplication().getParamOut() == null, + "empty overlay is omitted, matching call_graph's absence rule"); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SccTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SccTest.java new file mode 100644 index 00000000..dcfdd09a --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SccTest.java @@ -0,0 +1,46 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.ibm.cldk.schema.JCallEdge; +import java.util.List; +import org.junit.jupiter.api.Test; + +class SccTest { + + private static JCallEdge e(String s, String d) { + JCallEdge x = new JCallEdge(); + x.setSrc(s); + x.setDst(d); + return x; + } + + @Test + void chainCondensesToSingletonsCalleesFirst() { + List> order = Scc.condense(List.of("a", "b", "c"), List.of(e("a", "b"), e("b", "c"))); + assertEquals(List.of(List.of("c"), List.of("b"), List.of("a")), order); + } + + @Test + void mutualRecursionFormsOneComponent() { + List> order = Scc.condense( + List.of("even", "odd", "main"), + List.of(e("even", "odd"), e("odd", "even"), e("main", "even"))); + assertEquals(2, order.size()); + assertEquals(List.of("even", "odd"), order.get(0), "SCC first (bottom-up), sorted within"); + assertEquals(List.of("main"), order.get(1)); + } + + @Test + void deterministicRegardlessOfInputOrder() { + List> a = Scc.condense(List.of("x", "y"), List.of(e("x", "y"))); + List> b = Scc.condense(List.of("y", "x"), List.of(e("x", "y"))); + assertEquals(a, b); + } + + @Test + void edgesToUnknownNodesAreIgnored() { + List> order = Scc.condense(List.of("a"), List.of(e("a", "can://…/@external/x"))); + assertEquals(List.of(List.of("a")), order); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVerticesTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVerticesTest.java new file mode 100644 index 00000000..6c09e919 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SdgVerticesTest.java @@ -0,0 +1,128 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JIdEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JParameter; +import com.ibm.cldk.schema.JType; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class SdgVerticesTest { + + /** a(int x) at 3:16 calls b(int) — the minimal HRB shape. */ + private static Map twoCallableModule() { + JModule m = new JModule(); + m.setId("can://java/app/A.java"); + + JCallable b = new JCallable(); + b.setId("can://java/app/A.java/A/b(int)"); + b.setReturnType("int"); + JParameter p = new JParameter(); + p.setName("y"); + p.setType("int"); + b.getParameters().add(p); + + JCallable a = new JCallable(); + a.setId("can://java/app/A.java/A/a(int)"); + a.setReturnType("int"); + JParameter px = new JParameter(); + px.setName("x"); + px.setType("int"); + a.getParameters().add(px); + JBodyNode call = new JBodyNode(); + call.setKind("call"); + call.setCallee("can://java/app/A.java/A/b(int)"); + call.getArgumentExpr().add("x + 1"); + call.setReturnType("int"); + a.getBody().put("3:16", call); + + JType t = new JType(); + t.setId("can://java/app/A.java/A"); + t.getCallables().put("a(int)", a); + t.getCallables().put("b(int)", b); + m.getTypes().put("A", t); + + Map modules = new LinkedHashMap<>(); + modules.put("A.java", m); + return modules; + } + + @Test + void buildsFormalActualVerticesAndParamEdges() { + Map modules = twoCallableModule(); + SdgVertices.Result r = SdgVertices.apply(modules); + + JCallable a = modules.get("A.java").getTypes().get("A").getCallables().get("a(int)"); + JCallable b = modules.get("A.java").getTypes().get("A").getCallables().get("b(int)"); + + assertEquals("formal_in", b.getBody().get("@formal_in:0").getKind()); + assertEquals("y", b.getBody().get("@formal_in:0").getOf()); + assertEquals("$ret", b.getBody().get("@formal_out").getOf()); + JBodyNode actualIn = a.getBody().get("3:16/actual_in:0"); + assertNotNull(actualIn, "actual_in child of the call node"); + assertEquals("3:16", actualIn.getParent()); + assertEquals("arg0", actualIn.getOf()); + assertNotNull(a.getBody().get("3:16/actual_out")); + + assertEquals(1, r.paramIn.size()); + JIdEdge in = r.paramIn.get(0); + assertEquals("can://java/app/A.java/A/a(int)@3:16/actual_in:0", in.getSrc()); + assertEquals("can://java/app/A.java/A/b(int)@formal_in:0", in.getDst()); + assertEquals(1, r.paramOut.size()); + assertEquals("can://java/app/A.java/A/b(int)@formal_out", r.paramOut.get(0).getSrc()); + assertEquals("can://java/app/A.java/A/a(int)@3:16/actual_out", r.paramOut.get(0).getDst()); + } + + @Test + void externalCalleeGetsNoVerticesAndNoEdges() { + Map modules = twoCallableModule(); + JCallable a = modules.get("A.java").getTypes().get("A").getCallables().get("a(int)"); + JBodyNode ext = new JBodyNode(); + ext.setKind("call"); + ext.setCallee("can://java/app/@external/java.lang.Math/max(int, int)"); + ext.getArgumentExpr().add("x"); + a.getBody().put("4:9", ext); + + SdgVertices.Result r = SdgVertices.apply(modules); + assertTrue(a.getBody().keySet().stream().noneMatch(k -> k.startsWith("4:9/")), + "external callee: no actual vertices"); + assertEquals(1, r.paramIn.size(), "only the in-project edge"); + } + + @Test + void extraVarargsArgumentsAllMapToTheLastFormal() { + // b(int y) has a single parameter; make the call site to it carry 3 arguments (as if the + // call were being resolved to a varargs-shaped callee with only one declared formal). + Map modules = twoCallableModule(); + JCallable a = modules.get("A.java").getTypes().get("A").getCallables().get("a(int)"); + JBodyNode call = a.getBody().get("3:16"); + call.getArgumentExpr().add("2"); + call.getArgumentExpr().add("3"); + + SdgVertices.Result r = SdgVertices.apply(modules); + + assertNotNull(a.getBody().get("3:16/actual_in:0")); + assertNotNull(a.getBody().get("3:16/actual_in:1")); + assertNotNull(a.getBody().get("3:16/actual_in:2")); + assertEquals(3, r.paramIn.size()); + assertTrue(r.paramIn.stream().allMatch(e -> e.getDst().endsWith("@formal_in:0")), + "every extra argument beyond the last formal collapses onto it"); + } + + @Test + void deterministicAcrossRuns() { + SdgVertices.Result r1 = SdgVertices.apply(twoCallableModule()); + SdgVertices.Result r2 = SdgVertices.apply(twoCallableModule()); + assertEquals( + com.ibm.cldk.schema.V2Json.compact().toJson(r1.paramIn), + com.ibm.cldk.schema.V2Json.compact().toJson(r2.paramIn)); + } +} diff --git a/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPassTest.java b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPassTest.java new file mode 100644 index 00000000..cb63dd51 --- /dev/null +++ b/src/test/java/com/ibm/cldk/syntactic_analysis/dataflow/SummaryPassTest.java @@ -0,0 +1,392 @@ +package com.ibm.cldk.syntactic_analysis.dataflow; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.schema.JCallEdge; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JDdgEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JParameter; +import com.ibm.cldk.schema.JType; +import com.ibm.cldk.schema.Span; +import com.ibm.cldk.schema.V2Json; +import com.ibm.cldk.syntactic_analysis.L1Extractor; +import com.ibm.cldk.syntactic_analysis.L2CallGraph; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class SummaryPassTest { + + private static final String FIXTURE = "src/test/resources/test-applications/l4-sdg-test"; + + private static Map analyzed() throws Exception { + Map modules = L1Extractor.extractAll( + Paths.get(FIXTURE), "l4-sdg-test", null, new LinkedHashMap<>(), 3, 3, "ast"); + L2CallGraph.Result l2 = L2CallGraph.build("l4-sdg-test", modules, null, true); + SdgVertices.apply(modules); + SummaryPass.apply(modules, l2.callGraph(), 3); + return modules; + } + + private static JCallable callable(Map modules, String idSuffix) { + return modules.values().stream() + .flatMap(m -> m.getTypes().values().stream()) + .flatMap(t -> t.getCallables().values().stream()) + .filter(c -> c.getId().endsWith(idSuffix)) + .findFirst() + .orElseThrow(); + } + + @Test + void transitiveFlowYieldsSummaryEdgeInTheCaller() throws Exception { + Map modules = analyzed(); + // Chain.a calls b; b's param flows to its return via c — so a's call site gets a summary edge. + JCallable a = callable(modules, "/Chain/a(int)"); + assertNotNull(a.getSummary(), "a carries a summary edge for the a→b→c flow"); + assertEquals(1, a.getSummary().size()); + String src = a.getSummary().get(0).getSrc(); + String dst = a.getSummary().get(0).getDst(); + assertTrue(src.endsWith("/actual_in:0") && dst.endsWith("/actual_out"), src + " → " + dst); + assertEquals( + src.substring(0, src.indexOf("/actual_in")), + dst.substring(0, dst.indexOf("/actual_out")), + "same call site"); + } + + @Test + void mutualRecursionReachesFixpointAndIsDeterministic() throws Exception { + Map m1 = analyzed(); // terminating at all is half the gate + Map m2 = analyzed(); + assertEquals( + V2Json.compact().toJson(m1), + V2Json.compact().toJson(m2), + "two runs byte-identical (fixpoint order must not leak)"); + } + + @Test + void mutualRecursionSummarisesBothDirections() throws Exception { + Map modules = analyzed(); + // even → odd → even is one SCC, so both are solved by the same fixpoint loop: reaching this + // assertion at all means it terminated, and each member ends up shortcutting its own site. + for (String id : new String[] {"/Mutual/even(int)", "/Mutual/odd(int)"}) { + JCallable c = callable(modules, id); + assertNotNull(c.getSummary(), id + " carries its call site's summary edge"); + assertEquals(1, c.getSummary().size(), id); + assertTrue(c.getSummary().get(0).getSrc().endsWith("/actual_in:0"), id); + assertTrue(c.getSummary().get(0).getDst().endsWith("/actual_out"), id); + } + } + + @Test + void aCallableWithNoPassThroughCallSiteKeepsSummaryAbsent() throws Exception { + Map modules = analyzed(); + // Chain.c calls nothing, so it has no call site to shortcut... + assertNull(callable(modules, "/Chain/c(int)").getSummary(), "no call sites, no summary"); + // ...and Heap.roundTrip's two sites are a void call (no actual_out) and a no-arg call (no + // actual_in), so neither can carry a shortcut. Absent, not an empty list. + assertNull(callable(modules, "/Heap/roundTrip(int)").getSummary(), "no viable site, no summary"); + } + + /** + * A call whose value passes through a local before being returned — {@code int t = f(x); return + * t;} — which is where summary composition has to work: nothing in the return statement + * names the parameter, so the syntactic return rule cannot rescue it and the only route is the + * callee's own summary. The tree is built by hand, node for node, from the shape the AST engine + * really emits for this source (verified by running {@code -a 4} over it): the call sits at its + * own body node, the def-use edge hangs off the enclosing statement, and the call node + * itself carries no ddg edge at all. + * + *

The assertion is on {@code top}, not {@code mid}: {@code mid}'s own summary edge follows + * from {@code flows(callee)} alone and holds either way, so only the grandparent's edge actually + * depends on {@code flows(mid)} having composed through the call. + */ + @Test + void aValuePassingThroughALocalStillComposes() { + Map modules = passThroughChain(); + SdgVertices.apply(modules); + SummaryPass.apply(modules, passThroughCallGraph(), 3); + + JCallable top = callable(modules, "/Pass/top(int)"); + assertNotNull(top.getSummary(), "top composes through mid, which composes through callee"); + assertEquals(1, top.getSummary().size()); + assertEquals("topCall/actual_in:0", top.getSummary().get(0).getSrc()); + assertEquals("topCall/actual_out", top.getSummary().get(0).getDst()); + + JCallable mid = callable(modules, "/Pass/mid(int)"); + assertNotNull(mid.getSummary()); + assertEquals(1, mid.getSummary().size()); + } + + /** + * The same composition, but with the in-project call wrapped in an unresolved one — + * {@code int t = Math.abs(id(a)); return t;}. The wrapper is a body node too, and its span is + * narrower than the statement's, so it is what encloses the inner call; it is external, has no + * summary, and is an orphan carrying no ddg edge. If the route out of a call node were built only + * for resolved in-project sites, the chain would stop dead on the wrapper and {@code top} would + * lose a summary edge it should have — an under-approximation. Node ids, spans and the + * {@code callee}-less wrapper mirror what {@code -a 4} really emits for this source. + */ + @Test + void anUnresolvedWrapperDoesNotBreakComposition() { + Map modules = wrappedChain(); + SdgVertices.apply(modules); + SummaryPass.apply(modules, List.of(callEdge("top(int)", "caller(int)"), callEdge("caller(int)", "id(int)")), 3); + + JCallable top = callable(modules, "/Pass/top(int)"); + assertNotNull(top.getSummary(), "top still composes through caller, wrapper notwithstanding"); + assertEquals(1, top.getSummary().size()); + assertEquals("topCall/actual_in:0", top.getSummary().get(0).getSrc()); + assertEquals("topCall/actual_out", top.getSummary().get(0).getDst()); + } + + /** + * A parameter named only in a {@code for}-each header — {@code Loops.first}. Real source through + * the real pipeline, because that is the point: {@link com.ibm.cldk.syntactic_analysis.controlflow.BodyNodeBuilder} + * gives a for-each exactly one {@code loop} node spanning the whole statement (the header never + * gets a node of its own), and {@link DdgBuilder} attributes the iterated expression and the loop + * variable to that node. So the loop node has a real outgoing ddg edge to {@code return x;}, but + * the only thing that can ever seed it for {@code q} is its own span text — a parameter has no ddg + * def site. With container kinds excluded from seeding, {@code flows(first)} was empty and the + * caller lost a summary edge for a flow that genuinely exists. + */ + @Test + void aParameterUsedOnlyInALoopHeaderStillReachesTheReturn() throws Exception { + Map modules = analyzed(); + JCallable callFirst = callable(modules, "/Loops/callFirst(int[])"); + assertNotNull(callFirst.getSummary(), "q flows out of first through the for-each, so callFirst shortcuts it"); + assertEquals(1, callFirst.getSummary().size()); + assertTrue(callFirst.getSummary().get(0).getSrc().endsWith("/actual_in:0")); + assertTrue(callFirst.getSummary().get(0).getDst().endsWith("/actual_out")); + } + + /** + * A parameter that reaches the return through a local — {@code int m(int q) { int t = q; + * return t; }} — the commonest Java shape after {@code return q;} itself. Nothing in the ddg is + * rooted at {@code q}: {@code DdgBuilder} gives a parameter no defining node, so the only edge is + * {@code (decl → ret, var "t")}; there is no call site; and the return text names {@code t}, not + * {@code q}. The seed therefore has to come from the declaration statement's own text, + * which is what the widened word-boundary rule supplies — the existing {@code decl → ret} edge + * then carries it to the sink. + * + *

Asserted through a caller, because that is where the miss actually shows up: with + * {@code flows(m)} empty, every caller of {@code m} silently loses its summary edge. + */ + @Test + void aParameterCopiedIntoALocalStillReachesTheReturn() { + Map modules = localCopyChain(); + SdgVertices.apply(modules); + SummaryPass.apply(modules, List.of(callEdge("top(int)", "m(int)")), 3); + + JCallable top = callable(modules, "/Pass/top(int)"); + assertNotNull(top.getSummary(), "q reaches m's return through the local t, so top shortcuts the call"); + assertEquals(1, top.getSummary().size()); + assertEquals("topCall/actual_in:0", top.getSummary().get(0).getSrc()); + assertEquals("topCall/actual_out", top.getSummary().get(0).getDst()); + } + + private static final String LOCAL_SOURCE = String.join( + "\n", + "class Pass {", + " int m(int q) { int t = q; return t; }", + " int top(int b) { int u = m(b); return u; }", + "}"); + + /** {@code m} copies its parameter into a local and returns the local; {@code top} calls it. */ + private static Map localCopyChain() { + JType type = new JType(); + type.setId("can://java/pass/Pass.java/Pass"); + + JCallable m = method("m(int)", "q"); + m.getBody().put("@entry", node("entry", null, LOCAL_SOURCE)); + m.getBody().put("mDecl", node("statement", "int t = q;", LOCAL_SOURCE)); + m.getBody().put("mRet", node("return", "return t;", LOCAL_SOURCE)); + // The only edge the ddg has: `q` itself is never a def site, so no edge is rooted at it. + m.setDdg(new ArrayList<>(List.of(ddg("mDecl", "mRet", "t")))); + type.getCallables().put("m(int)", m); + + JCallable top = method("top(int)", "b"); + top.getBody().put("@entry", node("entry", null, LOCAL_SOURCE)); + top.getBody().put("topCall", callNode("m(b)", "m(int)", LOCAL_SOURCE)); + top.getBody().put("topDecl", node("statement", "int u = m(b);", LOCAL_SOURCE)); + top.getBody().put("topRet", node("return", "return u;", LOCAL_SOURCE)); + top.setDdg(new ArrayList<>(List.of(ddg("topDecl", "topRet", "u")))); + type.getCallables().put("top(int)", top); + + JModule module = new JModule(); + module.setId("can://java/pass/Pass.java"); + module.setSource(LOCAL_SOURCE); + module.getTypes().put("Pass", type); + return new LinkedHashMap<>(Map.of("Pass.java", module)); + } + + private static final String WRAP_SOURCE = String.join( + "\n", + "class Pass {", + " int id(int p) { return p; }", + " int caller(int a) { int t = Math.abs(id(a)); return t; }", + " int top(int b) { int u = caller(b); return u; }", + "}"); + + private static Map wrappedChain() { + JType type = new JType(); + type.setId("can://java/pass/Pass.java/Pass"); + + JCallable id = method("id(int)", "p"); + id.getBody().put("@entry", node("entry", null, WRAP_SOURCE)); + id.getBody().put("idRet", node("return", "return p;", WRAP_SOURCE)); + id.setDdg(new ArrayList<>()); + type.getCallables().put("id(int)", id); + + JCallable caller = method("caller(int)", "a"); + caller.getBody().put("@entry", node("entry", null, WRAP_SOURCE)); + caller.getBody().put("wrapCall", node("call", "Math.abs(id(a))", WRAP_SOURCE)); // no callee: unresolved + caller.getBody().put("idCall", callNode("id(a)", "id(int)", WRAP_SOURCE)); + caller.getBody().put("callerDecl", node("statement", "int t = Math.abs(id(a));", WRAP_SOURCE)); + caller.getBody().put("callerRet", node("return", "return t;", WRAP_SOURCE)); + caller.setDdg(new ArrayList<>(List.of(ddg("callerDecl", "callerRet", "t")))); + type.getCallables().put("caller(int)", caller); + + JCallable top = method("top(int)", "b"); + top.getBody().put("@entry", node("entry", null, WRAP_SOURCE)); + top.getBody().put("topCall", callNode("caller(b)", "caller(int)", WRAP_SOURCE)); + top.getBody().put("topDecl", node("statement", "int u = caller(b);", WRAP_SOURCE)); + top.getBody().put("topRet", node("return", "return u;", WRAP_SOURCE)); + top.setDdg(new ArrayList<>(List.of(ddg("topDecl", "topRet", "u")))); + type.getCallables().put("top(int)", top); + + JModule module = new JModule(); + module.setId("can://java/pass/Pass.java"); + module.setSource(WRAP_SOURCE); + module.getTypes().put("Pass", type); + return new LinkedHashMap<>(Map.of("Pass.java", module)); + } + + // Line 2 holds callee, 3 mid, 4 top; every snippet below is unique, and the text is ASCII so a + // char offset is a byte offset. + private static final String PASS_SOURCE = String.join( + "\n", + "class Pass {", + " int callee(int q) { return q; }", + " int mid(int x) { int t = callee(x); return t; }", + " int top(int w) { int s = mid(w); return s; }", + "}"); + + /** {@code callee → mid → top}, each level passing its argument out through the return. */ + private static Map passThroughChain() { + JType type = new JType(); + type.setId("can://java/pass/Pass.java/Pass"); + + JCallable callee = method("callee(int)", "q"); + callee.getBody().put("@entry", node("entry", null, PASS_SOURCE)); + callee.getBody().put("calleeRet", node("return", "return q;", PASS_SOURCE)); + callee.setDdg(new ArrayList<>()); + type.getCallables().put("callee(int)", callee); + + JCallable mid = method("mid(int)", "x"); + mid.getBody().put("@entry", node("entry", null, PASS_SOURCE)); + mid.getBody().put("midCall", callNode("callee(x)", "callee(int)", PASS_SOURCE)); + mid.getBody().put("midDecl", node("statement", "int t = callee(x);", PASS_SOURCE)); + mid.getBody().put("midRet", node("return", "return t;", PASS_SOURCE)); + // The def-use edge leaves the *statement*, never the nested call node — that asymmetry is + // the whole point of the case. + mid.setDdg(new ArrayList<>(List.of(ddg("midDecl", "midRet", "t")))); + type.getCallables().put("mid(int)", mid); + + JCallable top = method("top(int)", "w"); + top.getBody().put("@entry", node("entry", null, PASS_SOURCE)); + top.getBody().put("topCall", callNode("mid(w)", "mid(int)", PASS_SOURCE)); + top.getBody().put("topDecl", node("statement", "int s = mid(w);", PASS_SOURCE)); + top.getBody().put("topRet", node("return", "return s;", PASS_SOURCE)); + top.setDdg(new ArrayList<>(List.of(ddg("topDecl", "topRet", "s")))); + type.getCallables().put("top(int)", top); + + JModule module = new JModule(); + module.setId("can://java/pass/Pass.java"); + module.setSource(PASS_SOURCE); + module.getTypes().put("Pass", type); + return new LinkedHashMap<>(Map.of("Pass.java", module)); + } + + private static JDdgEdge ddg(String src, String dst, String var) { + JDdgEdge e = new JDdgEdge(); + e.setSrc(src); + e.setDst(dst); + e.setVar(var); + e.setProv(List.of("ssa")); + return e; + } + + private static JCallable method(String signature, String param) { + JCallable c = new JCallable(); + c.setId("can://java/pass/Pass.java/Pass/" + signature); + c.setKind("method"); + c.setSignature(signature); + c.setReturnType("int"); + JParameter p = new JParameter(); + p.setName(param); + p.setType("int"); + c.setParameters(new ArrayList<>(List.of(p))); + return c; + } + + private static JBodyNode node(String kind, String snippet, String source) { + JBodyNode n = new JBodyNode(); + n.setKind(kind); + if (snippet != null) { + Span span = new Span(); + int at = source.indexOf(snippet); + span.setBytes(new int[] {at, at + snippet.length()}); + n.setSpan(span); + } + return n; + } + + private static JBodyNode callNode(String snippet, String calleeSignature, String source) { + JBodyNode n = node("call", snippet, source); + n.setCallee("can://java/pass/Pass.java/Pass/" + calleeSignature); + n.setReturnType("int"); + n.setArgumentExpr(new ArrayList<>(List.of(snippet.substring(snippet.indexOf('(') + 1, snippet.length() - 1)))); + return n; + } + + private static List passThroughCallGraph() { + return List.of( + callEdge("top(int)", "mid(int)"), + callEdge("mid(int)", "callee(int)")); + } + + private static JCallEdge callEdge(String from, String to) { + JCallEdge e = new JCallEdge(); + e.setSrc("can://java/pass/Pass.java/Pass/" + from); + e.setDst("can://java/pass/Pass.java/Pass/" + to); + return e; + } + + @Test + void summaryEndpointsAreExistingLocalBodyNodes() throws Exception { + Map modules = analyzed(); + int[] edges = {0}; + modules.values().stream() + .flatMap(m -> m.getTypes().values().stream()) + .flatMap(t -> t.getCallables().values().stream()) + .filter(c -> c.getSummary() != null) + .forEach(c -> c.getSummary().forEach(e -> { + assertTrue(c.getBody().containsKey(e.getSrc()), c.getId() + " src " + e.getSrc()); + assertTrue(c.getBody().containsKey(e.getDst()), c.getId() + " dst " + e.getDst()); + edges[0]++; + })); + // Without this the endpoint check passes vacuously on a regression that empties every + // summary. Five is the fixture's whole set: Chain.a→b, Chain.b→c, Mutual.even→odd, + // Mutual.odd→even, Loops.callFirst→first. Heap's two sites cannot carry one (void callee, + // no-arg callee). + assertEquals(5, edges[0], "every fixture summary edge is checked, and there are five of them"); + } +} diff --git a/src/test/java/com/ibm/cldk/wala/WalaL4ModRefTest.java b/src/test/java/com/ibm/cldk/wala/WalaL4ModRefTest.java new file mode 100644 index 00000000..ceec39bb --- /dev/null +++ b/src/test/java/com/ibm/cldk/wala/WalaL4ModRefTest.java @@ -0,0 +1,128 @@ +package com.ibm.cldk.wala; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.L4WalaOverlays; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JDdgEdge; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.syntactic_analysis.L1Extractor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Real WALA over the l4-sdg-test fixture: heap round-trip yields points-to ddg additions. */ +@Tag("realworld") +class WalaL4ModRefTest { + + private static final String FIXTURE = "src/test/resources/test-applications/l4-sdg-test"; + + /** + * The heap round trip, from {@code Heap.java}: + * 

+     * 14:     public int roundTrip(int v) {
+     * 15:         put(v);        // writes this.box  -> body node 15:9
+     * 16:         int r = get(); // reads  this.box  -> body node 16:9
+     * 17:         return r;
+     * 18:     }
+     *
+ */ + private static final String PUT_CALL = "15:9"; + private static final String GET_CALL = "16:9"; + + @Test + void heapRoundTripGainsPointsToEdgesWithoutLosingSsaOnes(@TempDir Path tmp) throws Exception { + Map modules = L1Extractor.extractAll( + Paths.get(FIXTURE), "l4-sdg-test", null, new LinkedHashMap<>(), 3, 3, "ast"); + Optional wala = WalaAnalysis.of(compileFixture(tmp), null, null, "rta"); + // Not an assumption: WalaAnalysis.of swallows every Throwable, so a regression inside it + // would silently downgrade this gate to a skip. + assertTrue(wala.isPresent(), "WalaAnalysis.of must succeed over the compiled fixture"); + + JCallable roundTrip = modules.values().stream() + .flatMap(m -> m.getTypes().values().stream()) + .filter(t -> t.getId().endsWith("/Heap")) + .flatMap(t -> t.getCallables().values().stream()) + .filter(c -> c.getId().endsWith("roundTrip(int)")) + .findFirst().orElseThrow(); + + // Snapshot every callable's L3 ddg, not just the one under test: additivity is the gate. + // JCallable's equals is structural and its ddg is about to be mutated, so index by + // position rather than hashing the callables themselves. + List callables = modules.values().stream() + .flatMap(m -> m.getTypes().values().stream()) + .flatMap(t -> t.getCallables().values().stream()) + .filter(c -> c.getDdg() != null) + .collect(Collectors.toList()); + // The fixture's methods declare no locals, so the AST L3 engine emits no ssa edges of its + // own — seed one, or the additivity gate below has nothing to be additive over. It shares + // (src,dst,var) with the points-to edge WALA is about to derive and differs only in prov, + // so it also pins prov as part of the dedup identity: collapse the two and one is lost. + roundTrip.getDdg().add(edge(PUT_CALL, GET_CALL, "box", List.of("ssa"))); + + // Deep-copy the snapshot: sharing the JDdgEdge references would compare a rewritten edge + // with itself, so an in-place mutation of an L3 edge would slip through the gate below. + List> before = new ArrayList<>(); + for (JCallable callable : callables) { + before.add(callable.getDdg().stream() + .map(e -> edge(e.getSrc(), e.getDst(), e.getVar(), e.getProv())) + .collect(Collectors.toList())); + } + + L4WalaOverlays.apply(wala.get(), modules, 3); + + for (int i = 0; i < callables.size(); i++) { + assertTrue(callables.get(i).getDdg().containsAll(before.get(i)), + "L3 ssa edges survive untouched and unrewritten (L3 ⊆ L4): " + + callables.get(i).getId()); + } + + // Pin the whole tuple, not just the provenance: the endpoints are the projection of WALA's + // caller-side heap statements onto the calls they decorate, so a regression that lost the + // attribution (both ids collapsing to the ":0" sentinel, or landing on the wrong call + // site) would still satisfy a bare "some edge is points-to" assertion. + assertTrue(roundTrip.getDdg().contains(edge(PUT_CALL, GET_CALL, "box", List.of("points-to"))), + "the this.box write→read round-trip is a points-to dependence from the put call to " + + "the get call, keyed on the field: " + roundTrip.getDdg()); + + // ...and the endpoints must be real body nodes of this callable, not merely well-formed ids. + assertTrue(roundTrip.getBody().keySet().containsAll(List.of(PUT_CALL, GET_CALL)), + "both endpoints are body nodes of roundTrip: " + roundTrip.getBody().keySet()); + } + + private static JDdgEdge edge(String src, String dst, String var, List prov) { + JDdgEdge edge = new JDdgEdge(); + edge.setSrc(src); + edge.setDst(dst); + edge.setVar(var); + edge.getProv().addAll(prov); + return edge; + } + + /** Compiles the fixture's sources into {@code tmp} and returns it as the WALA input root. */ + private static String compileFixture(Path tmp) throws Exception { + List sources; + try (Stream walk = Files.walk(Paths.get(FIXTURE, "src", "main", "java"))) { + sources = walk.filter(p -> p.toString().endsWith(".java")) + .map(Path::toString) + .collect(Collectors.toList()); + } + String[] args = Stream.concat( + Stream.of("-g", "-d", tmp.toString()), sources.stream()).toArray(String[]::new); + assertEquals(0, ToolProvider.getSystemJavaCompiler().run(null, null, null, args), + "fixture compilation must succeed"); + return tmp.toString(); + } +} diff --git a/src/test/resources/test-applications/l4-sdg-test/build.gradle b/src/test/resources/test-applications/l4-sdg-test/build.gradle new file mode 100644 index 00000000..c18e0fae --- /dev/null +++ b/src/test/resources/test-applications/l4-sdg-test/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'java' +} + +repositories { + mavenCentral() +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +sourceSets { + main { + java { + srcDirs = ["src/main/java"] + } + } +} diff --git a/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Chain.java b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Chain.java new file mode 100644 index 00000000..2d6d014b --- /dev/null +++ b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Chain.java @@ -0,0 +1,15 @@ +package com.l4; + +public class Chain { + public int a(int x) { + return b(x + 1); + } + + public int b(int y) { + return c(y * 2); + } + + public int c(int z) { + return z - 3; + } +} diff --git a/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Heap.java b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Heap.java new file mode 100644 index 00000000..665e7e7e --- /dev/null +++ b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Heap.java @@ -0,0 +1,19 @@ +package com.l4; + +public class Heap { + private int box; + + public void put(int v) { + this.box = v; + } + + public int get() { + return this.box; + } + + public int roundTrip(int v) { + put(v); + int r = get(); + return r; + } +} diff --git a/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Loops.java b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Loops.java new file mode 100644 index 00000000..e46d43e4 --- /dev/null +++ b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Loops.java @@ -0,0 +1,17 @@ +package com.l4; + +public class Loops { + // The parameter is named nowhere but the for-each header, which BodyNodeBuilder folds into the + // single `loop` node covering the whole statement. A parameter has no ddg def site, so nothing + // can seed `q` except that node's own source text. + public int first(int[] q) { + for (int x : q) { + return x; + } + return 0; + } + + public int callFirst(int[] a) { + return first(a); + } +} diff --git a/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Mutual.java b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Mutual.java new file mode 100644 index 00000000..80f957d6 --- /dev/null +++ b/src/test/resources/test-applications/l4-sdg-test/src/main/java/com/l4/Mutual.java @@ -0,0 +1,17 @@ +package com.l4; + +public class Mutual { + public int even(int n) { + if (n == 0) { + return 1; + } + return odd(n - 1); + } + + public int odd(int n) { + if (n == 0) { + return 0; + } + return even(n - 1); + } +}