From 0a5612dee102de5d0053c83c0d5f2d30a3ce35fe Mon Sep 17 00:00:00 2001 From: Bryan Chan Date: Fri, 21 Aug 2026 15:03:33 -0700 Subject: [PATCH 1/2] =?UTF-8?q?ADFA-4128:=20qb=2009/12=20daemon=20?= =?UTF-8?q?=E2=80=94=20Long-lived=20compile=20service=20keeping=20kotlinc?= =?UTF-8?q?=20caches=20warm:=20incremental=20Kotlin/Java,=20d8,=20aapt2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Kj9YeCDHGp9DU8LPtfWJ7W --- quickbuild/daemon/build.gradle.kts | 168 ++++ .../cotg/quickbuild/daemon/DaemonMain.kt | 132 +++ .../cotg/quickbuild/daemon/DaemonService.kt | 326 ++++++++ .../daemon/compile/IncrementalCompiler.kt | 615 ++++++++++++++ .../daemon/compile/JavaCompileStep.kt | 88 ++ .../daemon/compile/JavaSourceAbi.kt | 218 +++++ .../compile/KotlincDiagnosticsParser.kt | 57 ++ .../cotg/quickbuild/daemon/dex/DexTool.kt | 242 ++++++ .../quickbuild/daemon/dex/FinalStripper.kt | 52 ++ .../daemon/protocol/ProtocolCodec.kt | 202 +++++ .../daemon/protocol/RequestRouter.kt | 184 +++++ .../cotg/quickbuild/daemon/res/Aapt2Link.kt | 317 +++++++ .../quickbuild/daemon/DaemonLoopErrorTest.kt | 117 +++ .../cotg/quickbuild/daemon/DaemonLoopTest.kt | 83 ++ .../cotg/quickbuild/daemon/DaemonMainTest.kt | 84 ++ .../quickbuild/daemon/DaemonServiceOpsTest.kt | 279 +++++++ .../quickbuild/daemon/DaemonServiceTest.kt | 309 +++++++ .../daemon/OfflineNetworkGuardTest.kt | 69 ++ .../cotg/quickbuild/daemon/TestSdk.kt | 100 +++ .../compile/IncrementalCompilerEdgeTest.kt | 435 ++++++++++ .../daemon/compile/IncrementalCompilerTest.kt | 780 ++++++++++++++++++ .../daemon/compile/JavaCompileStepTest.kt | 59 ++ .../daemon/compile/JavaSourceAbiEdgeTest.kt | 159 ++++ .../daemon/compile/JavaSourceAbiTest.kt | 361 ++++++++ .../KotlincDiagnosticsParserEdgeTest.kt | 89 ++ .../compile/KotlincDiagnosticsParserTest.kt | 58 ++ .../quickbuild/daemon/dex/DexToolEdgeTest.kt | 181 ++++ .../cotg/quickbuild/daemon/dex/DexToolTest.kt | 95 +++ .../daemon/dex/FinalStripperInnerClassTest.kt | 59 ++ .../daemon/dex/FinalStripperTest.kt | 208 +++++ .../daemon/protocol/ProtocolCodecEdgeTest.kt | 109 +++ .../daemon/protocol/ProtocolCodecTest.kt | 330 ++++++++ .../daemon/protocol/RequestRouterErrorTest.kt | 106 +++ .../daemon/protocol/RequestRouterGuardTest.kt | 54 ++ .../daemon/protocol/RequestRouterTest.kt | 92 +++ .../daemon/res/Aapt2LinkEdgeTest.kt | 165 ++++ .../quickbuild/daemon/res/Aapt2LinkTest.kt | 483 +++++++++++ settings.gradle.kts | 1 + 38 files changed, 7466 insertions(+) create mode 100644 quickbuild/daemon/build.gradle.kts create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMain.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonService.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompiler.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStep.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbi.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParser.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexTool.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripper.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodec.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouter.kt create mode 100644 quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2Link.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopErrorTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMainTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceOpsTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/OfflineNetworkGuardTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/TestSdk.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStepTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperInnerClassTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterErrorTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterGuardTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkEdgeTest.kt create mode 100644 quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkTest.kt diff --git a/quickbuild/daemon/build.gradle.kts b/quickbuild/daemon/build.gradle.kts new file mode 100644 index 0000000000..d6e76cccf9 --- /dev/null +++ b/quickbuild/daemon/build.gradle.kts @@ -0,0 +1,168 @@ +plugins { + id("java-library") + id("org.jetbrains.kotlin.jvm") +} + +description = + "Quick Build warm compile daemon: BTA incremental Kotlin compile + d8 + aapt2, run as a CoGo child process on the bundled JDK (ADFA-4128)" + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +kotlin { + jvmToolchain(17) +} + +// The Compose compiler plugin the daemon passes as -Xplugin when the user project uses +// Compose. Its own configuration (not runtimeClasspath): it is compiler INPUT, not a +// library the daemon's JVM loads. :app's quickBuildDaemonZip stages it next to the +// daemon jar under the stable name compose-compiler-plugin.jar. +val composeCompilerPlugin: Configuration by configurations.creating { + isCanBeConsumed = false + isTransitive = false +} + +// Compose runtime for the compose compile tests' classpath. Resolved as the Android +// AAR (what a real project's compile classpath carries); classes.jar is extracted +// below. Test-only - never shipped. +val composeTestRuntimeAar: Configuration by configurations.creating { + isCanBeConsumed = false + isTransitive = false + attributes { + attribute( + Usage.USAGE_ATTRIBUTE, + objects.named(Usage::class.java, Usage.JAVA_RUNTIME), + ) + } +} + +val stageComposeTestRuntime = + tasks.register("stageComposeTestRuntime") { + val aars = composeTestRuntimeAar + from(provider { zipTree(aars.singleFile) }) { + include("classes.jar") + rename("classes.jar", "compose-runtime.jar") + } + into(layout.buildDirectory.dir("compose-test-runtime")) + } + +tasks.withType { + useJUnitPlatform() + // Real inputs, not just dependsOn: a changed plugin or runtime jar must re-run tests. + inputs.files(stageComposeTestRuntime) + inputs.files(composeCompilerPlugin) + systemProperty( + "quickbuild.test.composeRuntimeJar", + layout.buildDirectory + .dir("compose-test-runtime") + .get() + .asFile + .resolve("compose-runtime.jar") + .absolutePath, + ) + jvmArgumentProviders.add( + CommandLineArgumentProvider { + listOf("-Dquickbuild.test.composePluginJar=${composeCompilerPlugin.singleFile.absolutePath}") + }, + ) + + // Fail-if-skipped switch for the toolchain-gated tests (aapt2/d8/Compose - the + // ADFA-4128 bug 5/6/8 regression coverage). Opt in with REQUIRE_BUILD_TOOLCHAIN=1 + // (env) or -PrequireBuildToolchain: TestSdk then throws from its @EnabledIf + // predicates when the toolchain is absent, failing the tests instead of skipping. + // Also undo the root build's ignoreFailures=true (set for coverage collection) so + // the failure actually fails the build - without that, CI would stay green. + val requireToolchain = + providers.environmentVariable("REQUIRE_BUILD_TOOLCHAIN").orNull == "1" || + providers.gradleProperty("requireBuildToolchain").isPresent + systemProperty("quickbuild.test.requireToolchain", requireToolchain.toString()) + if (requireToolchain) { + ignoreFailures = false + } +} + +// DoD coverage gate: >=90% line+branch on non-UI (domain/data) code. +// The root build applies the jacoco plugin to every subproject, which auto-creates +// jacocoTestReport for JVM modules -- but with the XML report off and no dependency +// on the test task, so the gate is never actually measured. The agent's exec lands +// at the JVM default build/jacoco/test.exec (Android modules differ - see +// :quick-build's report and the ADFA-3834 learnings on silently-SKIPped reports). +tasks.named("jacocoTestReport") { + dependsOn(tasks.test) + reports { + xml.required.set(true) + html.required.set(true) + } +} + +dependencies { + // The wire DTOs/constants, shared with CoGo's client so both sides compile + // against one protocol definition. api: the router/handler signatures expose them. + api(projects.quickbuild.protocol) + + implementation(libs.kotlin.buildToolsApi) + implementation(libs.google.gson) + // ACC_FINAL stripping on recompiled payload classes (proxies extend user classes). + implementation(libs.ow2.asm) + // The BTA implementation + its runtime deps are loaded from the daemon's runtime + // classpath on device (staged alongside the jar), matched to the bundled compiler. + // kotlin-compiler-runner exists solely to launch/talk to a separate long-lived + // "Kotlin compile daemon" JVM over RMI, which IncrementalCompiler never does here + // (it always calls useInProcessStrategy()) - dead weight (~17 KB of the ~62 MB + // quickbuild-daemon.zip, ADFA-4128 size audit). + // kotlin-daemon-client and kotlin-daemon-embeddable looked like the same kind of + // dead weight but are NOT: BuildToolsApiBuildICReporter.reportCompileIteration (part + // of kotlin-build-tools-impl itself, on the in-process path) references + // org.jetbrains.kotlin.daemon.common.CompileIterationResult, which lives in + // kotlin-daemon-client - excluding it throws NoClassDefFoundError and failed 12/52 + // :quickbuild-daemon:test cases. Keep both. + runtimeOnly(libs.kotlin.buildToolsImpl) { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-compiler-runner") + } + + // Staged next to the daemon jar on device and passed as -Xplugin when the user + // project uses Compose. + composeCompilerPlugin(libs.kotlin.composeCompilerPluginEmbeddable) + // The compose compile tests resolve a classpath from this; classes.jar is extracted + // from the AAR at build time and never shipped. Names the -android artifact rather + // than the KMP umbrella, which redirects via available-at - a redirect a + // non-transitive configuration will not follow. + composeTestRuntimeAar(libs.composeRuntimeDaemonTests) + + testImplementation(libs.tests.junit.jupiter) + testImplementation(libs.tests.google.truth) + // Shared offline-guard scanner (OfflineNetworkGuardTest). + testImplementation(testFixtures(projects.quickbuild.protocol)) + testRuntimeOnly(libs.tests.junit.platformLauncher) +} + +/** Single runnable jar; the runtime classpath is staged next to it on device. */ +val daemonJar = + tasks.register("daemonJar") { + archiveBaseName.set("quickbuild-daemon") + // Not build/libs: the default jar task also writes quickbuild-daemon.jar there, + // and two tasks sharing one archive path trips Gradle's implicit-dependency + // validation in any consumer (:app:quickBuildDaemonZip). + destinationDirectory.set(layout.buildDirectory.dir("daemon-jar")) + manifest { + attributes["Main-Class"] = "org.appdevforall.cotg.quickbuild.daemon.DaemonMain" + attributes["Class-Path"] = + configurations.runtimeClasspath + .get() + .files + .joinToString(" ") { it.name } + } + from(sourceSets.main.get().output) + } + +// The manifest Class-Path above names the runtime jars by FILE NAME, resolved +// relative to the jar's own directory. This stages a complete runnable layout +// (jar + deps side by side) so `java -jar build/daemon/quickbuild-daemon.jar` +// works with no manual copy step - what the corpus harness points --daemon-jar at. +tasks.register("stageDaemon") { + from(daemonJar) + from(configurations.runtimeClasspath) + into(layout.buildDirectory.dir("daemon")) +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMain.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMain.kt new file mode 100644 index 0000000000..a456470fdf --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMain.kt @@ -0,0 +1,132 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import org.appdevforall.cotg.quickbuild.daemon.protocol.ProtocolCodec +import org.appdevforall.cotg.quickbuild.daemon.protocol.RequestRouter +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.ParseResult +import java.io.BufferedReader +import java.io.BufferedWriter +import java.io.FileDescriptor +import java.io.FileOutputStream +import java.io.OutputStreamWriter +import java.io.PrintStream +import java.io.Writer +import java.nio.charset.StandardCharsets + +/** + * Daemon entry point for the line-delimited JSON protocol. main() keeps the real stdout for + * responses and redirects System.out to stderr, since the in-process Kotlin compiler's own prints + * would otherwise corrupt the protocol stream. + * + * Exit contract (quickbuild/README.md): build errors never exit, `shutdown` or stdin EOF exit 0, + * only a fatal internal error exits non-zero. The compiler runs in this JVM, so its own + * [OutOfMemoryError] and [StackOverflowError] are build errors - see [RequestRouter.isRequestFailure]. + */ +object DaemonMain { + /** + * Wires the process to the protocol streams and serves until shutdown or EOF. + * + * @param args ignored - the daemon is configured over the protocol, not the command line, + * so a launcher need pass nothing. + */ + @JvmStatic + fun main(args: Array) { + val protocolOut = + BufferedWriter(OutputStreamWriter(FileOutputStream(FileDescriptor.out), StandardCharsets.UTF_8)) + System.setOut(PrintStream(FileOutputStream(FileDescriptor.err), true, "UTF-8")) + + logErr("started (pid=${ProcessHandle.current().pid()})") + val service = DaemonService() + serve( + input = System.`in`.bufferedReader(StandardCharsets.UTF_8), + output = protocolOut, + router = RequestRouter(service), + ) + // The session's tools outlive the request loop, so release them here rather than + // leaving it to process teardown. + service.shutdown() + logErr("exiting") + } + + /** + * Runs the request/response loop until shutdown or EOF; malformed input replies ok:false + * and keeps serving. Separated from process wiring so it unit-tests against in-memory + * streams. Single-threaded on purpose - the CoGo orchestrator serializes requests. + * + * @param input one request per line, UTF-8; a null read (EOF) ends the loop, and it is not + * closed here. + * @param output receives one encoded response line per request, flushed after each; must be + * the real stdout, never the redirected [System.out]. + * @param router dispatches each parsed request; its [RequestRouter.Routed.ReplyThenExit] + * result is what ends the loop on `shutdown`. + */ + fun serve( + input: BufferedReader, + output: Writer, + router: RequestRouter, + ) { + while (true) { + val line = input.readLine() ?: return + if (line.isBlank()) continue + + // The router guards the handlers, but parse and encode run outside it, and both + // work on request-sized data: a pathological line, or a response carrying a + // compile's whole changed-class list. An uncaught throw from either would leave the + // loop and exit the JVM, which CoGo reads as daemon death - a restart cycle on every + // save of the same file, with no diagnostic ever rendered. + var routed: RequestRouter.Routed? = null + val encoded = + try { + routed = route(line, router) + ProtocolCodec.encode(routed.response) + } catch (t: Throwable) { + if (!RequestRouter.isRequestFailure(t)) throw t + // Allocation-light on purpose: the OOM arm gets here with the failed work's + // garbage already unreachable, and this response is a few hundred bytes. + // The id is the request's own when only the encode failed, and the codec's + // unknown-id sentinel when the line never parsed. + logErr("request failed: ${t.javaClass.simpleName}") + ProtocolCodec.encode( + DaemonResponse.failure( + routed?.response?.id ?: ParseResult.Malformed.UNKNOWN_ID, + RequestRouter.describe(t), + ), + ) + } + + output.write(encoded) + output.write("\n") + output.flush() + + if (routed is RequestRouter.Routed.ReplyThenExit) return + } + } + + /** + * Parses one line and routes it, or answers a line the codec rejected. + * + * @param line one request, already known to be non-blank. + * @param router dispatches the parsed request. + * @return what to reply, and whether to keep serving afterwards. + */ + private fun route( + line: String, + router: RequestRouter, + ): RequestRouter.Routed = + when (val parsed = ProtocolCodec.parse(line)) { + is ParseResult.Malformed -> { + logErr("malformed request: ${parsed.message}") + RequestRouter.Routed.Reply( + DaemonResponse.failure(parsed.id, "malformed request: ${parsed.message}"), + ) + } + + is ParseResult.Parsed -> { + router.route(parsed.request) + } + } + + private fun logErr(message: String) { + System.err.println("[quickbuild-daemon] $message") + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonService.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonService.kt new file mode 100644 index 0000000000..48bcb4ff95 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonService.kt @@ -0,0 +1,326 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import org.appdevforall.cotg.quickbuild.daemon.compile.IncrementalCompiler +import org.appdevforall.cotg.quickbuild.daemon.dex.DexTool +import org.appdevforall.cotg.quickbuild.daemon.protocol.DaemonHandlers +import org.appdevforall.cotg.quickbuild.daemon.res.Aapt2Link +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.RequestKeys +import org.appdevforall.cotg.quickbuild.protocol.ResponseKeys +import java.io.File +import java.nio.file.Files + +/** + * Implements the build ops, holding the warm state between them: `configure` builds the + * session (classpath snapshots, tool wrappers) and `compile`/`dex`/`relink` reuse it. + * Failures become ok:false responses; the backstop for anything that still throws is + * [RequestRouter]. + * + * @property log takes one already-formatted line of human-readable progress; defaults to stderr, + * never stdout, which is protocol-only. + */ +class DaemonService( + private val log: (String) -> Unit = { System.err.println(it) }, +) : DaemonHandlers { + /** + * The warm state one `configure` builds and every later op reuses. + * + * @property compiler holds the IC caches and classpath snapshots, so it must outlive a + * single compile. + * @property dexTool owns the r8 [java.net.URLClassLoader]; closed alongside the + * compiler when the session is replaced or shut down, see [release]. + * @property aapt2Link wraps the resolved aapt2 binary and android.jar. + * @property outDir the daemon's scratch root; the `dex` and `res` work dirs hang off it. + */ + private class Session( + val compiler: IncrementalCompiler, + val dexTool: DexTool, + val aapt2Link: Aapt2Link, + val outDir: File, + ) + + private var session: Session? = null + + /** + * Checks the toolchain, then builds the session that the later ops reuse. Any unsupplied + * tool or missing input file fails here rather than mid-build. + * + * @param request the session inputs; aapt2/d8Jar/androidJar are all required - the daemon + * never guesses a tool path - and `outDir` is created if absent. + * @return ok with `durationMillis`, the protocol version and the scratch filesystem type; + * ok:false with one diagnostic per unsupplied tool, or naming every input file missing + * from disk. + */ + override fun configure(request: ConfigureRequest): DaemonResponse { + // A guessed toolchain is worse than none: it would silently compile against some other + // SDK's android.jar and only surface on device. Every path is the caller's to supply. + val unsupplied = + listOf( + RequestKeys.AAPT2 to request.aapt2, + RequestKeys.D8_JAR to request.d8Jar, + RequestKeys.ANDROID_JAR to request.androidJar, + ).filter { (_, path) -> path.isNullOrBlank() } + .map { (field, _) -> field } + if (unsupplied.isNotEmpty()) { + return DaemonResponse.failure( + request.id, + unsupplied.map { + Diagnostic( + Diagnostic.Severity.ERROR, + "configure: $it path not supplied - the daemon does not discover tool paths", + ) + }, + ) + } + val aapt2Path = requireNotNull(request.aapt2) + val d8JarPath = requireNotNull(request.d8Jar) + val androidJarPath = requireNotNull(request.androidJar) + + val missing = + (request.classpath + request.compilerPlugins + aapt2Path + d8JarPath + androidJarPath) + .filter { !File(it).exists() } + if (missing.isNotEmpty()) { + return DaemonResponse.failure(request.id, "configure: missing files: ${missing.joinToString()}") + } + val outDir = File(request.outDir) + Files.createDirectories(outDir.toPath()) + + // Re-configure replaces the session (e.g. classpath changed -> new snapshots). Build the + // replacement BEFORE releasing the old one's tools: this can throw, and closing first + // would leave the still-installed old session holding a closed r8 class loader. That + // damage is LATENT - a closed URLClassLoader still serves classes it already loaded - so + // it surfaces later as a NoClassDefFoundError from inside d8. + val startedAt = System.currentTimeMillis() + val replacement = + Session( + // androidJar goes on the compile classpath too: the variant compile + // classpath from setup.json carries libraries but not the boot jar. + compiler = + IncrementalCompiler( + (request.classpath + androidJarPath).map(::File), + outDir.toPath(), + compilerPluginJars = request.compilerPlugins.map(::File), + ), + dexTool = DexTool(File(d8JarPath), File(androidJarPath), request.minApi), + aapt2Link = Aapt2Link(File(aapt2Path), File(androidJarPath)), + outDir = outDir, + ) + val durationMillis = System.currentTimeMillis() - startedAt + session?.let(::release) + session = replacement + val fsType = scratchFilesystemType(outDir) + log( + "configured: project=${request.projectRoot} classpath=${request.classpath.size} entries, " + + "snapshots in ${durationMillis}ms, scratch fs=$fsType", + ) + return DaemonResponse.ok( + request.id, + mapOf( + ResponseKeys.DURATION_MILLIS to durationMillis, + ResponseKeys.PROTOCOL_VERSION to DaemonResponse.PROTOCOL_VERSION, + ResponseKeys.SCRATCH_FS_TYPE to fsType, + ), + ) + } + + /** + * Releases a superseded session's tools. Both closes run even if the first throws: each + * owns state that otherwise lives for the JVM's lifetime - r8's [java.net.URLClassLoader] + * and the Build Tools API engine's per-project caches - on a 2-4 GB phone. + * + * Per SESSION only. Closing the compiler per compile would discard the warm incremental + * state the whole feature rests on. + * + * @param previous the session being replaced or shut down; unusable afterwards, so it must + * already have been detached from [session] or be on its way out. + */ + private fun release(previous: Session) { + runCatching { previous.compiler.close() } + .onFailure { log("failed to release the previous session's compiler: $it") } + runCatching { previous.dexTool.close() } + .onFailure { log("failed to release the previous session's dex tool: $it") } + // Logged because WHEN a release happens is the whole correctness question here: a + // release before its replacement exists strands the live session with closed tools. + log("released the previous session's tools") + } + + /** + * Releases the live session's tools on the way out of the process, after the request loop + * has stopped serving (`shutdown` op or stdin EOF). Idempotent, and a no-op when no + * `configure` ever ran. + */ + fun shutdown() { + session?.let(::release) + session = null + } + + /** + * The work directory's filesystem type (`ext4`, `f2fs`, `fuse`, ...), reported once per + * session because it dominates every per-file step: rewriting the same class tree costs + * 52x more on Android's FUSE-backed emulated storage than on the app's own filesystem + * [measured on a56, ADFA-4128], so a timing row without it is hard to read. Any failure + * reports `unknown` rather than failing a configure over telemetry. + * + * @param outDir the scratch root, which must already exist for the file store to resolve. + * @return the filesystem type name, or `unknown` if it could not be read. + */ + private fun scratchFilesystemType(outDir: File): String = + runCatching { Files.getFileStore(outDir.toPath()).type() } + .getOrNull() + ?.takeIf { it.isNotBlank() } + ?: "unknown" + + /** + * Compiles the requested sources and reports the changed class outputs plus phase timings. + * + * @param request must list every module source in `allSources`, not only the edited ones, + * and repeat them all in `changedFiles` on a session's first compile. + * @return ok with `classesDir`, the phase timings and the `classesChanged` path list, or + * ok:false carrying the compiler diagnostics; ok:false if no `configure` ran first. + */ + override fun compile(request: CompileRequest): DaemonResponse { + val session = session ?: return notConfigured(request.id) + val startedAt = System.currentTimeMillis() + val result = + session.compiler.compile( + request.allSources.map(::File), + request.changedFiles.map(::File), + request.removedFiles.map(::File), + ) + val durationMillis = System.currentTimeMillis() - startedAt + return when (result) { + is IncrementalCompiler.Result.Success -> { + log( + "compile ok: ${request.changedFiles.size} changed of ${request.allSources.size} " + + "in ${durationMillis}ms (kotlin=${result.kotlinMillis}ms java=${result.javaMillis}ms " + + "preSnap=${result.stats.preSnapMillis}ms postSnap=${result.stats.postSnapMillis}ms " + + "abiSnap=${result.stats.javaAbiSnapMillis}ms ktToCompile=${result.stats.kotlinToCompile} " + + "ordinal=${result.stats.compileOrdinal})", + ) + DaemonResponse( + id = request.id, + ok = true, + values = + mapOf( + ResponseKeys.CLASSES_DIR to result.classesDir.absolutePath, + ResponseKeys.DURATION_MILLIS to durationMillis, + ResponseKeys.KOTLIN_MILLIS to result.kotlinMillis, + ResponseKeys.JAVA_MILLIS to result.javaMillis, + ResponseKeys.CLASSES_CHANGED to result.changedClassFiles, + ) + result.stats.toValues(), + diagnostics = result.warnings, + ) + } + + is IncrementalCompiler.Result.Failed -> { + log( + "compile failed: ${result.diagnostics.size} diagnostics in ${durationMillis}ms " + + "(ktToCompile=${result.stats.kotlinToCompile} ordinal=${result.stats.compileOrdinal})", + ) + // Built here rather than through DaemonResponse.failure, which hardcodes an empty + // values map and is shared with every other failing op. The stats ride the failure + // because this is the build they are most needed from; the response stays ok=false + // and carries the same diagnostics it always did. + DaemonResponse( + id = request.id, + ok = false, + values = mapOf(ResponseKeys.DURATION_MILLIS to durationMillis) + result.stats.toValues(), + diagnostics = result.diagnostics, + ) + } + } + } + + /** + * Dexes the requested class dirs into the session's `dex` output dir. + * + * @param request `classesDirs` are roots scanned recursively; later roots win a path + * collision, so the compile output goes first and generated proxies after. + * @return ok with `dexFile` and the strip/d8 timings, or ok:false with the d8 failure text; + * ok:false if no `configure` ran first. + */ + override fun dex(request: DexRequest): DaemonResponse { + val session = session ?: return notConfigured(request.id) + val startedAt = System.currentTimeMillis() + val outDir = File(session.outDir, "dex") + return when (val result = session.dexTool.dex(request.classesDirs.map(::File), outDir)) { + is DexTool.Result.Success -> { + val durationMillis = System.currentTimeMillis() - startedAt + log( + "dex ok: ${result.dexFile} in ${durationMillis}ms (strip=${result.stripMillis}ms " + + "d8=${result.d8Millis}ms over ${result.stats.classFiles} classes / ${result.stats.classBytes} bytes)", + ) + DaemonResponse.ok( + request.id, + mapOf( + ResponseKeys.DEX_FILE to result.dexFile.absolutePath, + ResponseKeys.DURATION_MILLIS to durationMillis, + ResponseKeys.STRIP_MILLIS to result.stripMillis, + ResponseKeys.D8_MILLIS to result.d8Millis, + ) + result.stats.toValues(), + ) + } + + is DexTool.Result.Failed -> { + log("dex failed: ${result.message}") + DaemonResponse.failure(request.id, result.message) + } + } + } + + /** + * Rebuilds the resource apk from the project's res dirs and the library resources. + * + * @param request `stableIds` and `libraryResources` are optional on the wire but omitting + * either risks a wrong-id crash or an unresolvable reference - see [Aapt2Link]'s KDoc. + * @return ok with `resourcesArsc` (the full relinked apk) and the aapt2 timings, or ok:false + * carrying the aapt2 diagnostics; ok:false if no `configure` ran first. + */ + override fun relink(request: RelinkRequest): DaemonResponse { + val session = session ?: return notConfigured(request.id) + val startedAt = System.currentTimeMillis() + val workDir = File(session.outDir, "res") + Files.createDirectories(workDir.toPath()) + val result = + session.aapt2Link.relink( + request.resDirs.map(::File), + File(request.manifest), + workDir, + stableIds = request.stableIds?.let(::File), + libraryResources = request.libraryResources.map(::File), + ) + val durationMillis = System.currentTimeMillis() - startedAt + return when (result) { + is Aapt2Link.Result.Success -> { + log( + "relink ok: ${result.resourceApk} in ${durationMillis}ms " + + "(aapt2compile=${result.compileMillis}ms link=${result.linkMillis}ms)", + ) + // The wire field is named "resourcesArsc" for protocol stability, but the payload + // is the full relinked apk rather than a bare table - see Aapt2Link's KDoc. + DaemonResponse.ok( + request.id, + mapOf( + ResponseKeys.RESOURCES_ARSC to result.resourceApk.absolutePath, + ResponseKeys.DURATION_MILLIS to durationMillis, + ResponseKeys.AAPT2_COMPILE_MILLIS to result.compileMillis, + ResponseKeys.AAPT2_LINK_MILLIS to result.linkMillis, + ), + ) + } + + is Aapt2Link.Result.Failed -> { + log("relink failed: ${result.diagnostics.size} diagnostics") + DaemonResponse.failure(request.id, result.diagnostics) + } + } + } + + private fun notConfigured(id: Long): DaemonResponse = + DaemonResponse.failure(id, "daemon is not configured: send a 'configure' request first") +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompiler.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompiler.kt new file mode 100644 index 0000000000..8eb1ad2fa1 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompiler.kt @@ -0,0 +1,615 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import org.appdevforall.cotg.quickbuild.protocol.CompileStats +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.jetbrains.kotlin.buildtools.api.CompilationResult +import org.jetbrains.kotlin.buildtools.api.CompilationService +import org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi +import org.jetbrains.kotlin.buildtools.api.KotlinLogger +import org.jetbrains.kotlin.buildtools.api.ProjectId +import org.jetbrains.kotlin.buildtools.api.SourcesChanges +import org.jetbrains.kotlin.buildtools.api.jvm.ClassSnapshotGranularity +import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.util.UUID +import java.util.zip.CRC32 + +/** One walk of the class-output tree: '/'-separated relative path -> (size, content checksum). */ +private typealias OutputSnapshot = Map> + +/** + * Compiles a module's Kotlin and Java sources incrementally, so a one-line edit recompiles + * about one file instead of the whole app. + * + * Constraints the Kotlin Build Tools API imposes, none of them visible from the calls below + * (more in quickbuild/README.md): + * - Changes must be passed as [SourcesChanges.Known]; `ToBeCalculated` silently degrades to a + * full compile, as does a shrunk snapshot path other than exactly + * `/shrunk-classpath-snapshot.bin` (it is derived from `setRootProjectDir`). + * - The caller must pass ALL sources as changed on the first compile, to seed the IC caches. + * - `assureNoClasspathSnapshotsChanges(true)` is only safe once the shrunk snapshot exists; + * before that the engine needs the full classpath comparison to seed. + * + * Java sources take two passes: kotlinc reads them for symbol resolution only, then javac + * compiles them after Kotlin into the same output dir, which is what compiles Kotlin<->Java + * cycles. javac's pass is not incremental, and [JavaSourceAbi] decides when a `.java` edit + * forces a Kotlin recompile - see [kotlinFilesToCompile]. + * + * Kotlin 2.3 deprecates this [CompilationService] entry point in favor of `KotlinToolchains`. + * This class is the only caller of it, so a migration stays contained here. + * + * @param classpathJars the module's whole compile classpath, boot jar included; snapshotted once + * in `init`, so changing it means a new instance, never an in-place edit. + * @property workDir the daemon-owned scratch root, and the BTA `rootProjectDir` that fixes where + * the shrunk snapshot lands - it must not be the user's project dir. + * @param compilerPluginJars kotlinc plugin jars, each passed as one `-Xplugin`; session-fixed + * like the classpath. + * @param compileLog takes each level-tagged compiler log line as it is produced and retains + * nothing, since a session-lifetime copy of the engine's verbose debug channel is real memory + * on a 2-4 GB phone. + */ +@OptIn(ExperimentalBuildToolsApi::class) +class IncrementalCompiler( + classpathJars: List, + private val workDir: Path, + compilerPluginJars: List = emptyList(), + private val compileLog: (String) -> Unit = {}, +) : AutoCloseable { + /** Outcome of one compile. */ + sealed interface Result { + /** + * Both passes succeeded, with the outputs they touched and what each phase cost. + * + * @property classesDir single merged output dir for Kotlin and Java classes. + * @property warnings kotlinc's and javac's warnings, already parsed into the protocol + * shape; a successful compile can still carry them. + * @property changedClassFiles the .class files this compile emitted, rewrote or deleted, + * relative to [classesDir]; the deploy policy picks restart vs recreate from it, so it + * is diffed against the last DEPLOYED state and includes deletions. + * @property kotlinMillis wall time of the Kotlin pass (0 when there are no Kotlin sources). + * @property javaMillis wall time of the javac pass (0 when there are no Java sources). + * @property stats the phases [kotlinMillis]/[javaMillis] do not cover - the two + * output-tree walks and the Java-ABI re-parse - plus this build's source and output + * counts. + */ + data class Success( + val classesDir: File, + val warnings: List, + val changedClassFiles: List, + val kotlinMillis: Long = 0, + val javaMillis: Long = 0, + val stats: CompileStats = CompileStats(), + ) : Result + + /** + * A pass failed; nothing in the output dir should be deployed. + * + * @property diagnostics the errors that stopped the compile plus any warnings collected + * before it, never empty - an unexplained failure becomes one synthetic error. + * @property stats the phases that RAN before the failure, and this build's counts. A + * failing build is the one whose numbers are most worth having: `kotlinToCompile` says + * whether the dirty set we handed the engine contained the edit at all, and 0 vs >= 1 + * separates two different causes of a stale mixed-language output. Phases that never + * ran stay 0 - `postSnapMillis` and `changedClasses` are both only reachable after a + * success, so a failure legitimately reports none. + */ + data class Failed( + val diagnostics: List, + val stats: CompileStats = CompileStats(), + ) : Result + } + + private val service = CompilationService.loadImplementation(IncrementalCompiler::class.java.classLoader) + private val projectId = ProjectId.ProjectUUID(UUID.randomUUID()) + private val icCachesDir = workDir.resolve("ic") + private val classesDir = workDir.resolve("classes") + private val shrunkSnapshot = workDir.resolve("shrunk-classpath-snapshot.bin").toFile() + private val classpathSnapshots: List + private val classpathString = classpathJars.joinToString(File.pathSeparator) { it.absolutePath } + private val classpathFiles = classpathJars + + // Compiler plugins are passed as free-form kotlinc args, one -Xplugin per jar, the same + // way a CLI invocation would. Session-fixed, like the classpath. + private val pluginArguments = compilerPluginJars.map { "-Xplugin=${it.absolutePath}" } + + /** + * Java type names whose ABI moved in the last compile, forcing a full Kotlin recompile + * (see [kotlinFilesToCompile]). Empty when the Java side stayed ABI-stable, which is what + * explains an otherwise surprising slow compile. + */ + var lastJavaAbiChange: Set = emptySet() + private set + + // Phase timings/counts measured by compileKotlin and kotlinFilesToCompile on the way past; + // compile() folds them into the returned CompileStats. Safe as fields because the compiler + // runs one compile at a time by contract. + private var javaAbiSnapMillis: Long = 0 + private var kotlinToCompileCount: Int = 0 + + /** Compiles served since construction; a `configure` builds a fresh compiler. */ + private var compileCount: Long = 0 + + /** Last successful compile's `.java` ABI; null when unknown and Kotlin must be recompiled whole. */ + private var javaAbi: Map? = null + + /** This compile's `.java` ABI, promoted to [javaAbi] only once the compile succeeds. */ + private var pendingJavaAbi: Map? = null + + /** + * The output tree as of the last compile the caller could deploy; null before the first one. + * Held across compiles for the same reason [javaAbi] is: a failed compile leaves output nobody + * deployed, so re-snapshotting at the top of the next compile would adopt those undeployed + * classes as already-live and drop them from [Result.Success.changedClassFiles]. + */ + private var deployedOutputs: OutputSnapshot? = null + + init { + Files.createDirectories(icCachesDir) + Files.createDirectories(classesDir) + val snapshotDir = workDir.resolve("cp-snap") + Files.createDirectories(snapshotDir) + // Snapshot the fixed session classpath once; a classpath change is a session + // invalidation (new configure), never an in-place mutation. + classpathSnapshots = + classpathJars.mapIndexed { index, jar -> + // Indexed, not named after the jar: every AAR-derived entry is literally + // `classes.jar`, so a basename-keyed file would have them overwrite each + // other and the list would describe only the last of them. + val snapshot = snapshotDir.resolve("$index-${jar.name}.snap").toFile() + service + .calculateClasspathSnapshot(jar, ClassSnapshotGranularity.CLASS_MEMBER_LEVEL) + .saveSnapshot(snapshot) + snapshot + } + } + + /** + * Runs one compile: the incremental Kotlin pass, then javac over any `.java` sources. + * + * @param allSources every source in the module, not just the edited ones. + * @param changedFiles sources edited since the last compile; pass all of [allSources] on + * the first compile of a session. + * @param removedFiles sources deleted since the last compile, no longer in [allSources]; + * their stale `.class` outputs are cleaned before anything is compiled. + * @return [Result.Failed] on any compile error, and also when a removed source's stale + * `.class` could not be deleted. + */ + fun compile( + allSources: List, + changedFiles: List, + removedFiles: List = emptyList(), + ): Result { + // javac never deletes outputs for sources it is no longer given, so a removed .java's + // stale .class must go before the pre-snapshot - otherwise it survives into the dex, + // or is reported as a changed output. Removed .kt outputs are the engine's job, via + // SourcesChanges.Known below. + val undeleted = deleteJavaOutputs(removedFiles) + if (undeleted.isNotEmpty()) { + // Proceeding would dex the stale classes of a deleted source, the exact thing the + // delete exists to prevent. + return Result.Failed( + undeleted.map { stale -> + Diagnostic( + Diagnostic.Severity.ERROR, + "failed to delete stale class output of a removed Java source: ${stale.absolutePath}", + ) + }, + ) + } + compileCount++ + javaAbiSnapMillis = 0 + kotlinToCompileCount = 0 + val preSnapStartedAt = System.currentTimeMillis() + val before = deployedOutputs ?: snapshotClassOutputs() + val preSnapMillis = System.currentTimeMillis() - preSnapStartedAt + val logger = CollectingLogger(compileLog) + val kotlinStartedAt = System.currentTimeMillis() + val kotlinResult = compileKotlin(allSources, changedFiles, removedFiles, logger) + val kotlinMillis = System.currentTimeMillis() - kotlinStartedAt + if (kotlinResult != CompilationResult.COMPILATION_SUCCESS) { + val diagnostics = logger.errors.map { KotlincDiagnosticsParser.parse(it, Diagnostic.Severity.ERROR) } + return Result.Failed( + diagnostics.ifEmpty { + listOf(Diagnostic(Diagnostic.Severity.ERROR, "Kotlin compilation failed: $kotlinResult")) + }, + statsSoFar(preSnapMillis, allSources.size, javaSources = 0), + ) + } + + val javaSources = allSources.filter { it.extension == "java" } + // javac rewrites the outputs of the sources it is handed but deletes none whose + // declaration is gone, so an edit that drops an anonymous or nested class leaves + // Outer$1.class behind - untouched, therefore invisible to the output diff, and dexed into + // every later payload. Sweeping the edited sources here, AFTER the pre-snapshot, both + // removes it and surfaces the deletion as a changed output for the deploy policy. Scoped + // to changedFiles because only an edited file can lose a declaration; javac regenerates + // the primary outputs immediately, since it recompiles all of them anyway. + val staleNested = deleteJavaOutputs(changedFiles) + if (staleNested.isNotEmpty()) { + return Result.Failed( + staleNested.map { stale -> + Diagnostic( + Diagnostic.Severity.ERROR, + "failed to delete stale class output of a recompiled Java source: ${stale.absolutePath}", + ) + }, + statsSoFar(preSnapMillis, allSources.size, javaSources.size), + ) + } + val javaStartedAt = System.currentTimeMillis() + val javaDiagnostics = + if (javaSources.isEmpty()) { + JavaCompileStep.Result(success = true, diagnostics = emptyList()) + } else { + JavaCompileStep.compile( + javaSources = javaSources, + classpath = classpathFiles + classesDir.toFile(), + outputDir = classesDir.toFile(), + ) + } + val javaMillis = if (javaSources.isEmpty()) 0 else System.currentTimeMillis() - javaStartedAt + val warnings = logger.warnings.map { KotlincDiagnosticsParser.parse(it, Diagnostic.Severity.WARNING) } + if (!javaDiagnostics.success) { + return Result.Failed( + javaDiagnostics.diagnostics + warnings, + statsSoFar(preSnapMillis, allSources.size, javaSources.size), + ) + } + // Only a fully successful compile may become the ABI baseline: a failed compile leaves + // output the caller never deployed, so the next compile must still see the Java side + // as changed relative to the last good state. Hence committing here, not where the + // snapshot is taken. + javaAbi = pendingJavaAbi + val postSnapStartedAt = System.currentTimeMillis() + val after = snapshotClassOutputs() + val changedClassFiles = changedClassOutputs(before, after) + val postSnapMillis = System.currentTimeMillis() - postSnapStartedAt + // Same rule as the ABI above: this output only becomes the baseline because the caller + // can now deploy it. + deployedOutputs = after + return Result.Success( + classesDir = classesDir.toFile(), + warnings = warnings + javaDiagnostics.diagnostics, + changedClassFiles = changedClassFiles, + kotlinMillis = kotlinMillis, + javaMillis = javaMillis, + stats = + CompileStats( + preSnapMillis = preSnapMillis, + postSnapMillis = postSnapMillis, + javaAbiSnapMillis = javaAbiSnapMillis, + allSources = allSources.size, + kotlinToCompile = kotlinToCompileCount, + javaSources = javaSources.size, + changedClasses = changedClassFiles.size, + compileOrdinal = compileCount, + ), + ) + } + + /** + * Snapshots every .class under [classesDir] as relative path -> (size, content checksum). + * + * Content, not mtime. javac is not incremental here - it rewrites every Java-derived .class on + * every build, byte-identical or not - so an mtime diff reported the module's whole Java half + * as changed on a Kotlin-only edit, and the deploy policy then restarted the process for a + * component nothing had touched. It also missed the reverse: a same-size rewrite inside one + * tick of a coarse-granularity filesystem read as unchanged. A checksum answers both. + * + * @return '/'-separated relative path -> (size, checksum), empty when the output dir does not + * exist yet. + */ + private fun snapshotClassOutputs(): OutputSnapshot { + val root = classesDir + if (!Files.isDirectory(root)) return emptyMap() + val snapshot = HashMap>() + Files.walk(root).use { paths -> + paths.forEach { path -> + if (Files.isRegularFile(path) && path.toString().endsWith(".class")) { + val rel = root.relativize(path).toString().replace(java.io.File.separatorChar, '/') + snapshot[rel] = Files.size(path) to checksumOf(path) + } + } + } + return snapshot + } + + /** + * CRC32 of one class file's content, paired with its size in [OutputSnapshot] so a checksum + * collision alone cannot hide a changed class from the deploy policy. + * + * @param path the .class file to read. + * @return the checksum of its bytes. + */ + private fun checksumOf(path: Path): Long { + val crc = CRC32() + crc.update(Files.readAllBytes(path)) + return crc.value + } + + /** + * Diffs two output-tree walks into the paths the deploy has to account for. + * + * @param before the last deployed state. + * @param after this compile's state. + * @return added, rewritten AND deleted paths - a deletion has to be in here, since dropping a + * nested class of a restart-sensitive component is a change the deploy policy must see and + * filtering [after] alone can never surface it. + */ + private fun changedClassOutputs( + before: OutputSnapshot, + after: OutputSnapshot, + ): List = (after.filterKeys { before[it] != after[it] }.keys + (before.keys - after.keys)).sorted() + + /** + * Deletes the `.class` outputs of the given `.java` sources - the primary class and any nested + * `Outer$Inner.class` beside it - which javac never cleans up itself. + * + * Two callers, for the two ways an output goes stale. A REMOVED source, whose whole output + * would otherwise ride into every later dex. And a RECOMPILED source, whose vanished nested and + * anonymous classes javac leaves untouched: edit away an anonymous `Runnable` and `Outer$1.class` + * stays, untouched and therefore invisible to the output diff, dexed into every later payload + * and still resolvable by name. + * + * The source may be gone, so its package comes from the path (see [javaClassStem]). A top-level + * SECONDARY class (`class Helper` beside `public class Widget` in Widget.java) compiles to + * `Helper.class`, which no stem-keyed sweep can reach; closing that needs javac's own + * emitted-file list. + * + * TODO(ADFA-4128): hook javac's emitted-file list (TaskListener/JavaFileManager) to sweep + * top-level secondary classes too. Until then a deleted one stays in the payload dex until + * the next rebaseline: dead weight and name-resolvable, but no wrong behavior for code that + * does not look it up by name. + * + * @param sources the sources to sweep; non-`.java` entries are ignored here, since the IC + * engine owns Kotlin output deletion. + * @return the `.class` files that could not be deleted, on which [compile] must fail rather + * than dex a survivor. + */ + private fun deleteJavaOutputs(sources: List): List { + val classesRoot = classesDir.toFile() + if (!classesRoot.isDirectory) return emptyList() + val undeleted = mutableListOf() + val rootPrefix = classesRoot.canonicalPath + File.separator + sources.filter { it.extension == "java" }.forEach { javaFile -> + val relStem = javaClassStem(javaFile) ?: return@forEach + // relStem is a raw join of path segments, so a `..` in the removed source's path + // would aim this delete sweep outside the output tree. The paths come from CoGo's + // own watcher, but nothing here has to trust that. + val target = File(classesRoot, relStem).canonicalFile + if (!target.path.startsWith(rootPrefix)) return@forEach + val pkgDir = target.parentFile ?: return@forEach + val stem = target.name + pkgDir.listFiles()?.forEach { candidate -> + val name = candidate.name + if (name == "$stem.class" || (name.startsWith("$stem\$") && name.endsWith(".class"))) { + if (!candidate.delete() && candidate.exists()) { + undeleted += candidate + } + } + } + } + return undeleted + } + + /** + * The output-relative class stem (`com/foo/Bar`) for a `.java` source path, or null when no + * source root is found. Path-only, since the file is gone. Prefers a `main/java` or + * `main/kotlin` root so a package segment named `java`/`kotlin` deeper in the path isn't + * mistaken for the root; otherwise falls back to the last such segment. + * + * @param javaFile the removed source's path; it need not still exist on disk. + * @return the '/'-separated stem without the `.java` suffix, or null when the path has no + * `java`/`kotlin` source root or nothing follows it. + */ + private fun javaClassStem(javaFile: File): String? { + val parts = javaFile.invariantSeparatorsPath.split('/') + val isMarker = { i: Int -> parts[i] == "java" || parts[i] == "kotlin" } + val rootIdx = + parts.indices.lastOrNull { i -> isMarker(i) && i > 0 && parts[i - 1] == "main" } + ?: parts.indices.lastOrNull(isMarker) + ?: return null + if (rootIdx >= parts.lastIndex) return null + return parts.subList(rootIdx + 1, parts.size).joinToString("/").removeSuffix(".java") + } + + /** + * Runs the incremental Kotlin pass; a module with no Kotlin sources succeeds immediately. + * + * @param allSources every module source; the `.java` ones go to kotlinc for resolution only. + * @param changedFiles this edit's changes, narrowed by [kotlinFilesToCompile] before the + * engine sees them. + * @param removedFiles this edit's removals; only the non-`.java` ones are passed on. + * @param logger collects the compiler's messages, which are the only source of diagnostics. + * @return the raw BTA result; anything but `COMPILATION_SUCCESS` fails the compile. + */ + private fun compileKotlin( + allSources: List, + changedFiles: List, + removedFiles: List, + logger: CollectingLogger, + ): CompilationResult { + val kotlinSources = allSources.filter { it.extension != "java" } + val javaSources = allSources.filter { it.extension == "java" } + if (kotlinSources.isEmpty()) { + // Nothing for a Java ABI change to invalidate; keep no baseline for it either. + pendingJavaAbi = null + return CompilationResult.COMPILATION_SUCCESS + } + + // kotlinc needs the .java sources in compileJvm's source list to resolve a Kotlin file + // that calls a same-module Java class; the `-Xjava-source-roots` flag is silently ignored + // by this entry point, and no bytecode is emitted for them (JavaCompileStep does that). + // The engine tracks no ABI over those sources, so being told a .java file changed tells it + // nothing - kotlinFilesToCompile has to decide instead. + val kotlinChanged = kotlinFilesToCompile(kotlinSources, javaSources, changedFiles) + + val strategy = service.makeCompilerExecutionStrategyConfiguration().useInProcessStrategy() + val config = service.makeJvmCompilationConfiguration().useLogger(logger) + val icConfig = config.makeClasspathSnapshotBasedIncrementalCompilationConfiguration() + icConfig.setRootProjectDir(workDir.toFile()) + icConfig.setBuildDir(classesDir.toFile()) + if (shrunkSnapshot.exists()) { + icConfig.assureNoClasspathSnapshotsChanges(true) + } + val parameters = + ClasspathSnapshotBasedIncrementalCompilationApproachParameters(classpathSnapshots, shrunkSnapshot) + // Removed Kotlin sources go in SourcesChanges.Known's removed slot: the engine deletes + // their outputs and recompiles dependents, so a dangling reference surfaces as an + // ordinary compile error. The engine tracks only Kotlin outputs, so `.java` removals + // are handled separately in deleteJavaOutputs. + val kotlinRemoved = removedFiles.filter { it.extension != "java" } + val changes = SourcesChanges.Known(kotlinChanged, kotlinRemoved) + config.useIncrementalCompilation(icCachesDir.toFile(), changes, parameters, icConfig) + + val arguments = + listOf( + "-classpath", + classpathString, + "-d", + classesDir.toString(), + "-jvm-target", + JVM_TARGET, + "-module-name", + "quickbuild-payload", + "-no-stdlib", + "-no-reflect", + "-nowarn", + ) + pluginArguments + return service.compileJvm(projectId, strategy, config, kotlinSources + javaSources, arguments) + } + + /** + * The stats for a build that did not finish: the phases that ran, and the counts already + * decided. Reads fields, computes nothing - a failure path must not do measurable work. + * + * @param preSnapMillis the pre-compile output walk, which always ran by either failure point. + * @param allSources size of the source set this compile was handed. + * @param javaSources `.java` count, or 0 from the Kotlin failure point, where javac never ran + * and the number is not yet known - 0 there means "did not get that far", not "none". + * @return stats whose unreached phases (`postSnapMillis`, `changedClasses`) are 0. + */ + private fun statsSoFar( + preSnapMillis: Long, + allSources: Int, + javaSources: Int, + ): CompileStats = + CompileStats( + preSnapMillis = preSnapMillis, + javaAbiSnapMillis = javaAbiSnapMillis, + allSources = allSources, + kotlinToCompile = kotlinToCompileCount, + javaSources = javaSources, + compileOrdinal = compileCount, + ) + + /** + * Decides which Kotlin sources this compile must treat as changed, given the engine + * tracks no dependencies over the `.java` sources it resolves against. + * + * A stable Java ABI means exactly the caller's Kotlin changes suffice; any ABI move, or an + * ABI that is unknown (first compile, no javac, an unparseable source), recompiles every + * Kotlin source - bluntly, since BTA cannot be told of a non-classpath ABI change. + * + * @param kotlinSources every Kotlin source in the module - the fallback answer. + * @param javaSources every `.java` source, fingerprinted here and compared against the last + * successful compile's baseline. + * @param changedFiles the caller's changes; the `.java` entries are dropped, since the + * fingerprint, not the caller, decides what a Java edit costs. + * @return the Kotlin sources to hand the engine as changed; also updates [lastJavaAbiChange] + * and stages the new baseline, which only a successful compile promotes. + */ + private fun kotlinFilesToCompile( + kotlinSources: List, + javaSources: List, + changedFiles: List, + ): List { + lastJavaAbiChange = emptySet() + val kotlinChanged = changedFiles.filter { it.extension != "java" } + val previous = javaAbi + val snapshotStartedAt = System.currentTimeMillis() + val current = JavaSourceAbi.snapshot(javaSources) + javaAbiSnapMillis = System.currentTimeMillis() - snapshotStartedAt + pendingJavaAbi = current + val toCompile = + when { + previous == null || current == null -> { + kotlinSources + } + + else -> { + val changedTypes = JavaSourceAbi.changedTypeNames(previous, current) + lastJavaAbiChange = changedTypes + if (changedTypes.isEmpty()) kotlinChanged else kotlinSources + } + } + kotlinToCompileCount = toCompile.size + return toCompile + } + + /** + * Releases the compilation service's state for this compiler's project. On the in-process + * strategy that state lives for the JVM's lifetime, so a session that re-configures without + * this accumulates one project's engine state per configure, on a 2-4 GB phone. + * + * Per SESSION, never per compile: the retained state IS the warm incremental cache the whole + * feature rests on. The instance cannot compile afterwards. + */ + override fun close() { + service.finishProjectCompilation(projectId) + } + + /** + * Collects compiler output per channel; the error channel feeds structured diagnostics. + * `internal` rather than private so severity routing is unit-testable - the daemon passes + * `-nowarn`, so no real compile can drive the warn channel from a test. + * + * Errors and warnings are kept because the compile's result is built from them, and they + * die with the compile. Every line is only forwarded, never accumulated. + * + * @property emit takes each line already tagged with its level. + */ + internal class CollectingLogger( + private val emit: (String) -> Unit, + ) : KotlinLogger { + val errors = mutableListOf() + val warnings = mutableListOf() + + override val isDebugEnabled: Boolean = true + + override fun error( + msg: String, + throwable: Throwable?, + ) { + errors += msg + emit("e: $msg") + } + + override fun warn( + msg: String, + throwable: Throwable?, + ) { + warnings += msg + emit("w: $msg") + } + + override fun info(msg: String) { + emit("i: $msg") + } + + override fun debug(msg: String) { + emit("d: $msg") + } + + override fun lifecycle(msg: String) { + emit("l: $msg") + } + } + + companion object { + // ART (via d8 desugaring) handles Java-17 bytecode; matches the bundled JDK. + private const val JVM_TARGET = "17" + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStep.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStep.kt new file mode 100644 index 0000000000..98a57b16e6 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStep.kt @@ -0,0 +1,88 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import java.io.File +import java.io.StringWriter +import java.nio.charset.StandardCharsets +import java.util.Locale +import javax.tools.DiagnosticCollector +import javax.tools.JavaFileObject +import javax.tools.ToolProvider + +/** + * Compiles the project's `.java` sources with the JDK's in-process javac, after Kotlin. + * javac's structured [javax.tools.Diagnostic]s map onto the protocol shape directly, so + * this path needs no text parsing. + */ +object JavaCompileStep { + /** + * Outcome of one javac run; [diagnostics] carries warnings even on success. + * + * @property success javac's own verdict; false also covers a runtime with no compiler. + * @property diagnostics every message javac produced, errors and warnings alike, so the + * caller must filter by severity rather than assume a non-empty list means failure. + */ + data class Result( + val success: Boolean, + val diagnostics: List, + ) + + /** + * Compiles [javaSources] into [outputDir]. + * + * @param javaSources every `.java` in the module, not just the edited ones - this pass is + * not incremental. + * @param classpath the compile classpath; the caller adds the Kotlin output dir so Java + * can reference Kotlin classes. + * @param outputDir the same dir the Kotlin pass wrote to, so one tree holds both languages. + * @return a failed [Result] rather than an exception when the runtime has no javac. + */ + fun compile( + javaSources: List, + classpath: List, + outputDir: File, + ): Result { + val compiler = + ToolProvider.getSystemJavaCompiler() + ?: return Result( + success = false, + diagnostics = + listOf( + Diagnostic(Diagnostic.Severity.ERROR, "no system Java compiler available (JRE-only runtime?)"), + ), + ) + val collector = DiagnosticCollector() + val fileManager = compiler.getStandardFileManager(collector, Locale.ROOT, StandardCharsets.UTF_8) + fileManager.use { manager -> + val units = manager.getJavaFileObjectsFromFiles(javaSources) + val options = + listOf( + "-classpath", + classpath.joinToString(File.pathSeparator) { it.absolutePath }, + "-d", + outputDir.absolutePath, + // Annotation processing is a full-Gradle-build concern; + // running processors here would silently diverge from the real build. + "-proc:none", + "-encoding", + "UTF-8", + ) + val task = compiler.getTask(StringWriter(), manager, collector, options, null, units) + val success = task.call() + return Result(success, collector.diagnostics.map { it.toProtocol() }) + } + } + + private fun javax.tools.Diagnostic.toProtocol(): Diagnostic = + Diagnostic( + severity = + when (kind) { + javax.tools.Diagnostic.Kind.ERROR -> Diagnostic.Severity.ERROR + else -> Diagnostic.Severity.WARNING + }, + message = getMessage(Locale.ROOT), + file = source?.name, + line = lineNumber.takeIf { it != javax.tools.Diagnostic.NOPOS }?.toInt(), + column = columnNumber.takeIf { it != javax.tools.Diagnostic.NOPOS }?.toInt(), + ) +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbi.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbi.kt new file mode 100644 index 0000000000..93e9138af1 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbi.kt @@ -0,0 +1,218 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.sun.source.tree.ClassTree +import com.sun.source.tree.CompilationUnitTree +import com.sun.source.tree.MethodTree +import com.sun.source.tree.Tree +import com.sun.source.tree.VariableTree +import com.sun.source.util.JavacTask +import java.io.File +import java.io.StringWriter +import java.nio.charset.StandardCharsets +import java.security.MessageDigest +import java.util.Locale +import javax.lang.model.element.Modifier +import javax.tools.DiagnosticCollector +import javax.tools.JavaFileObject +import javax.tools.ToolProvider + +/** + * Fingerprints the ABI - not the implementation - of the project's `.java` sources, so a + * Java edit only costs a Kotlin recompile when it could change Kotlin bytecode. + * + * kotlinc reads same-module `.java` files as raw sources (see [IncrementalCompiler]) but the + * incremental engine tracks no dependencies over them, so without a Java-side signal every + * `.java` edit would have to recompile every Kotlin file. + * + * Two things stay in the fingerprint although they look like implementation: a compile-time + * constant field's initializer, since Kotlin inlines Java constants into its callers' bytecode, + * and annotations, since they reach Kotlin's resolution (nullability especially). + * + * Parsing uses javac's own parser via [JavacTask.parse] - syntax only, no symbol resolution and + * no classpath - so it cannot fail over the unresolved cross-language references that make the + * two-pass compile necessary. Anything unparseable yields null, which callers must read as + * "assume the ABI changed". + */ +object JavaSourceAbi { + /** + * One file's ABI. + * + * @property fingerprint hash over the file's imports and declarations, method bodies excluded. + * @property declaredTypeNames every type simple name the file declares, nested included - + * the names a Kotlin source would have to write to reference it. + */ + data class FileAbi( + val fingerprint: String, + val declaredTypeNames: Set, + ) + + /** + * Fingerprints each of [javaSources]; null if any file could not be parsed. + * + * @param javaSources every `.java` in the module; an empty list is a known-empty ABI, not + * an unknown one. + * @return one entry per input file, or null - which callers must read as "assume the ABI + * changed", never as "nothing changed". + */ + fun snapshot(javaSources: List): Map? { + if (javaSources.isEmpty()) return emptyMap() + val compiler = ToolProvider.getSystemJavaCompiler() ?: return null + val collector = DiagnosticCollector() + return try { + compiler.getStandardFileManager(collector, Locale.ROOT, StandardCharsets.UTF_8).use { manager -> + val units = manager.getJavaFileObjectsFromFiles(javaSources) + val task = + compiler.getTask(StringWriter(), manager, collector, listOf("-proc:none"), null, units) + as? JavacTask ?: return null + val byPath = javaSources.associateBy { it.absolutePath } + val result = HashMap() + for (unit in task.parse()) { + val file = byPath[File(unit.sourceFile.toUri()).absolutePath] ?: continue + result[file] = unit.toAbi() + } + // A file javac declined to hand back was not parsed; do not claim to know its ABI. + if (result.size != javaSources.size) null else result + } + } catch (e: Exception) { + null + } + } + + /** + * Simple names of every type whose ABI differs between [previous] and [current], covering + * added, removed and modified files. Takes the union of old and new names, so a renamed or + * deleted type is still named for Kotlin sources that may reference it. + * + * @param previous the last successful compile's snapshot; both maps are keyed by source file. + * @param current this compile's snapshot. + * @return simple names only, nested types included; empty means the Java side is ABI-stable + * and no Kotlin bytecode can have moved because of it. + */ + fun changedTypeNames( + previous: Map, + current: Map, + ): Set { + val changed = HashSet() + for ((file, abi) in current) { + val before = previous[file] + if (before == null || before.fingerprint != abi.fingerprint) { + changed += abi.declaredTypeNames + before?.let { changed += it.declaredTypeNames } + } + } + for ((file, abi) in previous) { + if (file !in current) changed += abi.declaredTypeNames + } + return changed + } + + private fun CompilationUnitTree.toAbi(): FileAbi { + val text = StringBuilder() + val names = HashSet() + text.append("package ").append(packageName?.toString() ?: "").append('\n') + // Imports are ABI. Signatures are fingerprinted as their written source text, so + // swapping `import a.Widget` for `import b.Widget` changes the type a Kotlin caller + // links against without moving one character of `Widget make()`. Sorted, so merely + // reordering imports is not read as a change. + for (import in imports.map { it.toString().trim() }.sorted()) { + text.append(import).append('\n') + } + for (decl in typeDecls) { + if (decl is ClassTree) decl.render(text, names, prefix = "") + } + return FileAbi(sha256(text.toString()), names) + } + + /** + * Appends this type's declarations to the fingerprint text, recursing into nested types. + * + * @param out the fingerprint buffer; member order follows source order, so a pure reorder + * does read as an ABI change. + * @param names collects every simple name declared, this type and its nested ones. + * @param prefix the enclosing type's dotted name, empty at the top level. + */ + private fun ClassTree.render( + out: StringBuilder, + names: MutableSet, + prefix: String, + ) { + val name = simpleName.toString() + names += name + val qualified = if (prefix.isEmpty()) name else "$prefix.$name" + out + .append("type ") + .append(qualified) + .append(' ') + .append(modifiers.toString().trim()) + .append(" typeparams=") + .append(typeParameters.joinToString(",") { it.toString() }) + .append(" extends=") + .append(extendsClause?.toString() ?: "") + .append(" implements=") + .append(implementsClause.joinToString(",") { it.toString() }) + .append('\n') + // Interface, annotation and enum members are implicitly constant even with no + // modifiers written, so whether an initializer is ABI depends on the owner. + val constantByDefault = kind != Tree.Kind.CLASS + for (member in members) { + when (member) { + is ClassTree -> member.render(out, names, qualified) + + is MethodTree -> out.append(member.renderSignature(qualified)).append('\n') + + is VariableTree -> out.append(member.renderSignature(qualified, constantByDefault)).append('\n') + + // Initializer blocks and empty declarations carry no ABI. + else -> Unit + } + } + } + + /** + * Renders a method's signature, deliberately excluding its body. + * + * @param owner the enclosing type's dotted name, so two same-named methods do not collide. + * @return one line of fingerprint text; an annotation member's default value is included, + * because that default is itself ABI. + */ + private fun MethodTree.renderSignature(owner: String): String = + buildString { + append("method ").append(owner).append('#').append(name) + append(' ').append(modifiers.toString().trim()) + append(" typeparams=").append(typeParameters.joinToString(",") { it.toString() }) + append(" returns=").append(returnType?.toString() ?: "") + append(" params=").append(parameters.joinToString(",") { it.type.toString() + " " + it.name }) + append(" throws=").append(throws.joinToString(",") { it.toString() }) + // An annotation member's default IS its ABI. + append(" default=").append(defaultValue?.toString() ?: "") + } + + /** + * Renders a field's declaration, plus its initializer when the field is a compile-time + * constant. Kotlin bakes `static final` constant values into calling bytecode, so a changed + * value is an ABI change even though the signature did not move. An ordinary instance + * field's initializer is implementation and stays out. + * + * @param owner the enclosing type's dotted name. + * @param constantByDefault true for an interface, annotation or enum body, whose fields are + * implicitly `static final` with no modifiers written. + * @return one line of fingerprint text, carrying the initializer only for a constant. + */ + private fun VariableTree.renderSignature( + owner: String, + constantByDefault: Boolean, + ): String = + buildString { + append("field ").append(owner).append('#').append(name) + append(' ').append(modifiers.toString().trim()) + append(" type=").append(type?.toString() ?: "") + val declaredConstant = + modifiers.flags.contains(Modifier.STATIC) && modifiers.flags.contains(Modifier.FINAL) + if (declaredConstant || constantByDefault) append(" const=").append(initializer?.toString() ?: "") + } + + private fun sha256(value: String): String { + val digest = MessageDigest.getInstance("SHA-256").digest(value.toByteArray(StandardCharsets.UTF_8)) + return digest.joinToString("") { "%02x".format(it) } + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParser.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParser.kt new file mode 100644 index 0000000000..c2956db8df --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParser.kt @@ -0,0 +1,57 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic + +/** + * Turns kotlinc's rendered log messages into structured diagnostics, so the IDE can jump to + * file:line. Renderers vary across compiler versions ("file:1:2 message", "file:1:2: error: + * message"), so the location prefix is matched leniently and anything unrecognized degrades + * to a location-less diagnostic rather than being dropped. + */ +object KotlincDiagnosticsParser { + // .kt:: optionally followed by ":", optionally "error:"/"warning:". + // Matched against the message's FIRST LINE only: `.` must not cross a newline here, or a + // multi-line message whose location sits on a later line has its first line swallowed into + // the file group - losing the primary error text and yielding a path no editor can open. + private val LOCATION = + Regex("""^(.+?\.(?:kt|kts|java)):(\d+):(\d+):?\s+(?:(error|warning):\s*)?(.*)$""") + + /** + * Parses one compiler message into a diagnostic, with location when the text carries one. + * + * @param message one rendered compiler message, trimmed here; only its first line can carry a + * location, any further lines being kept as message body. + * @param severity the severity implied by the logger channel the message arrived on + * (error() -> ERROR, warn() -> WARNING); an explicit "error:"/"warning:" prefix in the + * text wins over it. + * @return a diagnostic with file/line/column when the first line carried a location, and the + * whole trimmed message with none when it did not - input is never dropped. + */ + fun parse( + message: String, + severity: Diagnostic.Severity, + ): Diagnostic { + val trimmed = message.trim() + val firstLine = trimmed.substringBefore('\n') + val body = trimmed.substringAfter('\n', missingDelimiterValue = "") + val match = + LOCATION.find(firstLine) + ?: return Diagnostic(severity, trimmed) + val (file, line, column, severityWord, text) = match.destructured + val effectiveSeverity = + when (severityWord) { + "error" -> Diagnostic.Severity.ERROR + "warning" -> Diagnostic.Severity.WARNING + else -> severity + } + return Diagnostic( + severity = effectiveSeverity, + message = if (body.isEmpty()) text.trim() else (text.trim() + "\n" + body).trim(), + // kotlinc 2.x renders locations as file:// URIs; the IDE jump-to-editor + // path (and the protocol example) wants a plain filesystem path. + file = file.removePrefix("file://"), + line = line.toIntOrNull(), + column = column.toIntOrNull(), + ) + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexTool.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexTool.kt new file mode 100644 index 0000000000..5b7dcc3746 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexTool.kt @@ -0,0 +1,242 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import org.appdevforall.cotg.quickbuild.protocol.DexStats +import java.io.File +import java.lang.reflect.InvocationTargetException +import java.net.URLClassLoader +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.extension + +/** + * Runs D8 over compiled class files to produce `classes.dex`. The r8 jar comes from the + * device's provisioned build-tools at configure time and is loaded through its own + * [URLClassLoader], with every call made reflectively, so the daemon needs no AGP or r8 build + * dependency and works against whatever build-tools version the device ships. + * + * @param d8Jar the device's `lib/d8.jar`; opened into a private class loader here and not + * retained, so the caller may not swap it without a new [DexTool]. + * @property androidJar the platform jar, passed to d8 as library (not program) input. + * @property minApi the payload's `minSdkVersion`, which decides what d8 desugars. + */ +class DexTool( + d8Jar: File, + private val androidJar: File, + private val minApi: Int, +) : AutoCloseable { + /** Outcome of one dex run. */ + sealed interface Result { + /** + * D8 produced a dex, with the timings and counts the run cost. + * + * @property dexFile the emitted `classes.dex`, verified to exist before this is built. + * @property stripMillis wall time of the ACC_FINAL-stripping mirror pass. + * @property d8Millis wall time of the d8 invocation itself. + * @property stats what the run processed; both steps cover the whole class tree every + * build, so their cost scales with these counts rather than with the edit's size. + */ + data class Success( + val dexFile: File, + val stripMillis: Long = 0, + val d8Millis: Long = 0, + val stats: DexStats = DexStats(), + ) : Result + + /** + * The run produced no usable dex. + * + * @property message caller-facing reason - no input classes, a d8 error, a payload d8 + * had to split across several dex files, or an r8 jar whose layout does not match + * what the reflective calls expect. + */ + data class Failed( + val message: String, + ) : Result + } + + private val loader = URLClassLoader(arrayOf(d8Jar.toURI().toURL()), DexTool::class.java.classLoader) + + /** + * Dexes every `.class` under [classesDirs] into `/classes.dex`, first clearing + * ACC_FINAL from each class ([FinalStripper]) so the payload matches the gen-0 baseline's + * opened classes and the proxies' `extends` stays verifiable. + * + * @param classesDirs roots walked recursively; a non-directory entry is skipped, and a later + * root overwrites an earlier one on the same relative path. + * @param outDir created if absent; receives `classes.dex` and the `opened-classes` mirror, + * both wiped at the start of every run. + * @return [Result.Failed] when no `.class` was found, when d8 threw, when d8 exited clean + * without writing a dex, or when d8 split the payload across more than one dex. + */ + fun dex( + classesDirs: List, + outDir: File, + ): Result { + outDir.mkdirs() + // The dex count after the run is the only signal that d8 split the payload, so the dir + // must hold nothing but this run's output. The r8 jar comes from whatever build-tools + // the device provisioned, and while the ones measured here do clear stale dex files + // themselves, that is not a documented guarantee to inherit a correctness check from. + dexFilesIn(outDir).forEach { it.delete() } + val stripStartedAt = System.currentTimeMillis() + val opened = openClasses(classesDirs, File(outDir, "opened-classes")) + val stripMillis = System.currentTimeMillis() - stripStartedAt + val classFiles = opened.paths + if (classFiles.isEmpty()) { + return Result.Failed("no .class files found under: ${classesDirs.joinToString()}") + } + return try { + val d8StartedAt = System.currentTimeMillis() + runD8(classFiles, outDir.toPath()) + val d8Millis = System.currentTimeMillis() - d8StartedAt + val dexFiles = dexFilesIn(outDir) + val failure = dexFailureReason(dexFiles, outDir) + if (failure != null) { + Result.Failed(failure) + } else { + Result.Success( + dexFiles.single(), + stripMillis = stripMillis, + d8Millis = d8Millis, + stats = DexStats(classFiles = classFiles.size, classBytes = opened.bytes), + ) + } + } catch (e: InvocationTargetException) { + Result.Failed("d8 failed: ${e.cause?.message ?: e.cause?.javaClass?.name ?: e.message}") + } catch (e: ReflectiveOperationException) { + Result.Failed("d8 jar is not usable (wrong build-tools layout?): ${e.message}") + } + } + + /** + * Builds and runs a D8 command reflectively against the device's r8 jar. + * + * @param classFiles the already-stripped `.class` copies, passed as d8 program inputs. + * @param outDir d8's output dir, written in `DexIndexed` mode. + * @throws java.lang.reflect.InvocationTargetException wrapping any d8 compilation error. + * @throws ReflectiveOperationException when the r8 jar does not expose the expected API. + */ + private fun runD8( + classFiles: List, + outDir: Path, + ) { + val commandClass = loader.loadClass("com.android.tools.r8.D8Command") + val outputModeClass = loader.loadClass("com.android.tools.r8.OutputMode") + val dexIndexed = outputModeClass.enumConstants.first { (it as Enum<*>).name == "DexIndexed" } + + val builder = commandClass.getMethod("builder").invoke(null) + val builderClass = builder.javaClass + builderClass + .getMethod("addProgramFiles", Collection::class.java) + .invoke(builder, classFiles) + builderClass + .getMethod("addLibraryFiles", Collection::class.java) + .invoke(builder, listOf(androidJar.toPath())) + builderClass + .getMethod("setMinApiLevel", Int::class.javaPrimitiveType) + .invoke(builder, minApi) + builderClass + .getMethod("setOutput", Path::class.java, outputModeClass) + .invoke(builder, outDir, dexIndexed) + val command = builderClass.getMethod("build").invoke(builder) + + loader + .loadClass("com.android.tools.r8.D8") + .getMethod("run", commandClass) + .invoke(null, command) + } + + /** + * Mirrors every `.class` under [classesDirs] into [openedRoot] with ACC_FINAL + * cleared. Later roots overwrite earlier ones on a path collision (compile output + * first, proxy classes second - no overlap in practice). + * + * @param classesDirs roots to mirror, in precedence order; non-directories are skipped. + * @param openedRoot deleted recursively first, so it must not be a caller-owned dir. + * @return the stripped copies in first-seen path order, and the total bytes read. + */ + private fun openClasses( + classesDirs: List, + openedRoot: File, + ): Opened { + openedRoot.deleteRecursively() + val opened = LinkedHashMap() + var bytes = 0L + for (dir in classesDirs.filter { it.isDirectory }) { + val base = dir.toPath() + Files.walk(base).use { stream -> + stream.filter { it.extension == "class" }.forEach { classFile -> + val target = openedRoot.toPath().resolve(base.relativize(classFile)) + Files.createDirectories(target.parent) + val original = Files.readAllBytes(classFile) + bytes += original.size + Files.write(target, FinalStripper.strip(original)) + opened[base.relativize(classFile)] = target + } + } + } + return Opened(opened.values.toList(), bytes) + } + + /** + * What one [openClasses] pass produced: the stripped copies, and the bytes it read. + * + * @property paths absolute paths under the opened root, deduplicated by relative path. + * @property bytes size of the originals read, not of the rewritten copies. + */ + private data class Opened( + val paths: List, + val bytes: Long, + ) + + /** Closes the r8 class loader; the instance cannot dex afterwards. */ + override fun close() { + loader.close() + } + + companion object { + /** `classes.dex`, `classes2.dex`, ... - d8's DexIndexed output names, and nothing else. */ + private val DEX_FILE_NAME = Regex("""classes\d*\.dex""") + + /** + * The dex files d8 has written into [outDir], `classes.dex` first. + * + * @param outDir the run's output dir; a dir that does not exist yet reads as empty. + */ + private fun dexFilesIn(outDir: File): List = + outDir + .listFiles { file -> file.isFile && DEX_FILE_NAME.matches(file.name) } + ?.sortedBy { it.name } + .orEmpty() + + /** + * Why [dexFiles] is not a deployable result, or null when it is the one dex the deploy path + * can carry. `internal` so the split case is testable - real d8 needs 64K method refs to split. + * + * A split payload has to fail: d8 splits silently and exits clean past the per-dex method-ref + * limit, and the runtime only ever loads `classes.dex`, so shipping it would surface as + * `NoClassDefFoundError` against a green build. + * + * @param dexFiles what [dexFilesIn] found after the d8 run. + * @param outDir named in the message, since the caller sees only the message. + */ + internal fun dexFailureReason( + dexFiles: List, + outDir: File, + ): String? = + when { + dexFiles.isEmpty() -> { + "d8 reported success but produced no classes.dex in $outDir" + } + + dexFiles.size > 1 -> { + "payload too large for one dex: d8 split it into ${dexFiles.joinToString { it.name }}. " + + "Quick Build deploys a single dex, so this payload needs a standard build." + } + + else -> { + null + } + } + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripper.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripper.kt new file mode 100644 index 0000000000..b3de0060a8 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripper.kt @@ -0,0 +1,52 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes + +/** + * Clears ACC_FINAL from a class file, matching the proxy app build's ClassOpener in the + * gradle-plugin. The generated Proxy*Activity classes extend the user's activities and the + * dex verifier enforces superclass finality at load time, so every payload dex must carry the + * recompiled user classes with finality stripped, exactly as the gen-0 baseline did. Kotlin + * classes are final by default, so this runs on every hot recompile rather than once. + */ +object FinalStripper { + /** + * Returns [classBytes] rewritten with ACC_FINAL cleared on the class and its inner classes. + * + * @param classBytes one whole `.class` file; read, never modified in place. + * @return a freshly allocated class file, semantically the input minus ACC_FINAL but not + * byte-comparable with it, since ASM rebuilds the constant pool on the way through. + */ + fun strip(classBytes: ByteArray): ByteArray { + val reader = ClassReader(classBytes) + val writer = ClassWriter(0) + reader.accept( + object : ClassVisitor(Opcodes.ASM9, writer) { + override fun visit( + version: Int, + access: Int, + name: String?, + signature: String?, + superName: String?, + interfaces: Array?, + ) { + super.visit(version, access and Opcodes.ACC_FINAL.inv(), name, signature, superName, interfaces) + } + + override fun visitInnerClass( + name: String?, + outerName: String?, + innerName: String?, + access: Int, + ) { + super.visitInnerClass(name, outerName, innerName, access and Opcodes.ACC_FINAL.inv()) + } + }, + 0, + ) + return writer.toByteArray() + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodec.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodec.kt new file mode 100644 index 0000000000..cee3d7a4b6 --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodec.kt @@ -0,0 +1,202 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.google.gson.JsonPrimitive +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonOps +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.ParseResult +import org.appdevforall.cotg.quickbuild.protocol.PingRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.RequestKeys +import org.appdevforall.cotg.quickbuild.protocol.ResponseKeys +import org.appdevforall.cotg.quickbuild.protocol.ShutdownRequest + +/** + * Encodes and decodes the line-delimited JSON protocol. Pure functions over strings, no IO, so + * malformed-input handling is exhaustively unit-testable. Gson escapes newlines inside strings, + * so an encoded response is always exactly one line. + */ +object ProtocolCodec { + /** + * Parses one request line. Never throws: broken input becomes [ParseResult.Malformed]. + * + * @param line exactly one JSON object, without its trailing newline; blank lines are the + * caller's to skip. + * @return [ParseResult.Parsed] with the typed request, or [ParseResult.Malformed] carrying + * the id when one could be read and [ParseResult.Malformed.UNKNOWN_ID] when it could not. + */ + fun parse(line: String): ParseResult { + val root = + try { + val element = JsonParser.parseString(line) + if (!element.isJsonObject) { + return ParseResult.Malformed(ParseResult.Malformed.UNKNOWN_ID, "request is not a JSON object") + } + element.asJsonObject + } catch (e: Exception) { + return ParseResult.Malformed(ParseResult.Malformed.UNKNOWN_ID, "invalid JSON: ${e.message}") + } + + val id = + root.longOrNull(RequestKeys.ID) ?: return ParseResult.Malformed( + ParseResult.Malformed.UNKNOWN_ID, + "missing or non-numeric 'id'", + ) + + return try { + when (val op = root.stringOrNull(RequestKeys.OP)) { + DaemonOps.CONFIGURE -> { + ParseResult.Parsed( + ConfigureRequest( + id = id, + projectRoot = root.requireString(RequestKeys.PROJECT_ROOT), + classpath = root.requireStringList(RequestKeys.CLASSPATH), + outDir = root.requireString(RequestKeys.OUT_DIR), + aapt2 = root.stringOrNull(RequestKeys.AAPT2), + d8Jar = root.stringOrNull(RequestKeys.D8_JAR), + androidJar = root.stringOrNull(RequestKeys.ANDROID_JAR), + minApi = root.longOrNull(RequestKeys.MIN_API)?.toInt() ?: ConfigureRequest.DEFAULT_MIN_API, + compilerPlugins = root.optionalStringList(RequestKeys.COMPILER_PLUGINS), + ), + ) + } + + DaemonOps.COMPILE -> { + ParseResult.Parsed( + CompileRequest( + id = id, + allSources = root.requireStringList(RequestKeys.ALL_SOURCES), + changedFiles = root.requireStringList(RequestKeys.CHANGED_FILES), + removedFiles = root.optionalStringList(RequestKeys.REMOVED_FILES), + ), + ) + } + + DaemonOps.DEX -> { + ParseResult.Parsed( + DexRequest(id = id, classesDirs = root.requireStringList(RequestKeys.CLASSES_DIRS)), + ) + } + + DaemonOps.RELINK -> { + ParseResult.Parsed( + RelinkRequest( + id = id, + resDirs = root.requireStringList(RequestKeys.RES_DIRS), + manifest = root.requireString(RequestKeys.MANIFEST), + stableIds = root.stringOrNull(RequestKeys.STABLE_IDS), + libraryResources = root.optionalStringList(RequestKeys.LIBRARY_RESOURCES), + ), + ) + } + + DaemonOps.PING -> { + ParseResult.Parsed(PingRequest(id)) + } + + DaemonOps.SHUTDOWN -> { + ParseResult.Parsed(ShutdownRequest(id)) + } + + null -> { + ParseResult.Malformed(id, "missing 'op'") + } + + else -> { + ParseResult.Malformed(id, "unknown op '$op'") + } + } + } catch (e: MissingFieldException) { + ParseResult.Malformed(id, e.message ?: "malformed request") + } + } + + /** + * Encodes a response as one JSON line (no trailing newline). + * + * @param response its `values` may hold numbers, booleans, collections of strings, or + * anything else, which is written as its `toString`. + * @return a single line - Gson escapes any newline inside a string - that the caller must + * terminate itself. + */ + fun encode(response: DaemonResponse): String { + val root = JsonObject() + root.addProperty(ResponseKeys.ID, response.id) + root.addProperty(ResponseKeys.OK, response.ok) + for ((key, value) in response.values) { + when (value) { + is Number -> { + root.addProperty(key, value) + } + + is Boolean -> { + root.addProperty(key, value) + } + + is Collection<*> -> { + val array = JsonArray() + value.forEach { array.add(it.toString()) } + root.add(key, array) + } + + else -> { + root.addProperty(key, value.toString()) + } + } + } + if (response.diagnostics.isNotEmpty()) { + val array = JsonArray() + for (diagnostic in response.diagnostics) { + val obj = JsonObject() + obj.addProperty(ResponseKeys.Diagnostics.SEVERITY, diagnostic.severity.name) + obj.addProperty(ResponseKeys.Diagnostics.MESSAGE, diagnostic.message) + diagnostic.file?.let { obj.addProperty(ResponseKeys.Diagnostics.FILE, it) } + diagnostic.line?.let { obj.addProperty(ResponseKeys.Diagnostics.LINE, it) } + diagnostic.column?.let { obj.addProperty(ResponseKeys.Diagnostics.COLUMN, it) } + array.add(obj) + } + root.add(ResponseKeys.DIAGNOSTICS, array) + } + return root.toString() + } + + private class MissingFieldException( + message: String, + ) : Exception(message) + + private fun JsonObject.longOrNull(name: String): Long? { + val element = get(name) ?: return null + val primitive = element as? JsonPrimitive ?: return null + if (!primitive.isNumber) return null + return primitive.asLong + } + + private fun JsonObject.stringOrNull(name: String): String? { + val element = get(name) ?: return null + val primitive = element as? JsonPrimitive ?: return null + if (!primitive.isString) return null + return primitive.asString + } + + private fun JsonObject.requireString(name: String): String = + stringOrNull(name) ?: throw MissingFieldException("missing or non-string '$name'") + + private fun JsonObject.optionalStringList(name: String): List = if (has(name)) requireStringList(name) else emptyList() + + private fun JsonObject.requireStringList(name: String): List { + val element = get(name) ?: throw MissingFieldException("missing '$name'") + if (!element.isJsonArray) throw MissingFieldException("'$name' is not an array") + return element.asJsonArray.map { item -> + val primitive = item as? JsonPrimitive + if (primitive == null || !primitive.isString) { + throw MissingFieldException("'$name' contains a non-string element") + } + primitive.asString + } + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouter.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouter.kt new file mode 100644 index 0000000000..bfbbba3d7c --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouter.kt @@ -0,0 +1,184 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.PingRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.ResponseKeys +import org.appdevforall.cotg.quickbuild.protocol.ShutdownRequest + +/** + * The build ops the daemon serves. Implementations report tool failures as ok:false responses; + * a throw that escapes anyway is caught by [RequestRouter] when it is a failure of the request + * rather than of the process ([RequestRouter.isRequestFailure]), so a build problem can never + * kill it (the daemon exits only on shutdown, EOF, or a fatal internal error). + */ +interface DaemonHandlers { + /** + * Builds the session state - toolchain, classpath snapshots - that the other ops reuse. + * + * @param request the session inputs; unset tool paths are discovered by the implementation. + * @return the response to write back, ok:false when a tool or input file is missing. + */ + fun configure(request: ConfigureRequest): DaemonResponse + + /** + * Compiles the requested sources and reports which class outputs changed. + * + * @param request the full source list plus this edit's changed and removed files. + * @return the response to write back, ok:false carrying diagnostics on a compile error. + */ + fun compile(request: CompileRequest): DaemonResponse + + /** + * Dexes the requested class dirs into a single `classes.dex`. + * + * @param request the class-output roots to dex, in precedence order. + * @return the response to write back, ok:false when d8 fails or emits no dex. + */ + fun dex(request: DexRequest): DaemonResponse + + /** + * Rebuilds the resource apk from the project's resources. + * + * @param request the res dirs, manifest, and the optional stable-ids and library inputs. + * @return the response to write back, ok:false carrying aapt2's diagnostics on failure. + */ + fun relink(request: RelinkRequest): DaemonResponse +} + +/** + * Routes a parsed request to its handler and keeps handler exceptions from escaping. Pure + * logic, no IO, so routing and the exception backstop unit-test with scripted fakes. + * + * @property handlers the build ops; `ping` and `shutdown` never reach it, and anything it throws + * is converted to an ok:false response rather than propagated. + */ +class RequestRouter( + private val handlers: DaemonHandlers, +) { + /** What the main loop should do with the routed result. */ + sealed interface Routed { + val response: DaemonResponse + + /** + * Reply and keep serving - the ordinary case. + * + * @property response the line to write back before reading the next request. + */ + data class Reply( + override val response: DaemonResponse, + ) : Routed + + /** + * Reply, then exit the process cleanly (shutdown op). + * + * @property response must still be written and flushed before the loop returns. + */ + data class ReplyThenExit( + override val response: DaemonResponse, + ) : Routed + } + + /** + * Dispatches [request] to its handler; ping and shutdown are answered here directly. + * + * @param request an already-parsed request; malformed input never gets this far. + * @return [Routed.ReplyThenExit] only for `shutdown`, [Routed.Reply] for everything else. + */ + fun route(request: DaemonRequest): Routed = + when (request) { + is ShutdownRequest -> { + Routed.ReplyThenExit(DaemonResponse.ok(request.id)) + } + + is PingRequest -> { + Routed.Reply( + DaemonResponse.ok(request.id, mapOf(ResponseKeys.PROTOCOL_VERSION to DaemonResponse.PROTOCOL_VERSION)), + ) + } + + is ConfigureRequest -> { + Routed.Reply(guarded(request.id) { handlers.configure(request) }) + } + + is CompileRequest -> { + Routed.Reply(guarded(request.id) { handlers.compile(request) }) + } + + is DexRequest -> { + Routed.Reply(guarded(request.id) { handlers.dex(request) }) + } + + is RelinkRequest -> { + Routed.Reply(guarded(request.id) { handlers.relink(request) }) + } + } + + /** + * Turns a handler failure into an ok:false response, including the two [Error]s the + * in-process compiler throws on the user's own source. + * + * @param id the request id to echo, so a failed call is still correlatable by the caller. + * @param body the handler call to run; a throw that [isRequestFailure] rejects propagates. + * @return the handler's own response, or a synthesized failure naming what went wrong. + */ + private inline fun guarded( + id: Long, + body: () -> DaemonResponse, + ): DaemonResponse = + try { + body() + } catch (t: Throwable) { + if (!isRequestFailure(t)) throw t + DaemonResponse.failure(id, describe(t)) + } + + companion object { + /** + * Text for an [OutOfMemoryError], pre-built so the failure path allocates no string. + * + * Catching an OOM and carrying on is only sound while the unwind allocates almost + * nothing: the compiler's own garbage is unreachable by the time this is read, so the + * small response below is affordable, and anything larger would not be. + */ + private const val OUT_OF_MEMORY = + "the compiler ran out of memory on this change. Try a smaller edit, or restart the " + + "Quick Build session for a fresh compiler." + + /** Text for a [StackOverflowError], pre-built for the same reason as [OUT_OF_MEMORY]. */ + private const val STACK_OVERFLOW = + "the compiler ran out of stack on this change - an expression or type here nests too " + + "deeply for it." + + /** + * Whether a throw is a failure of the requested work rather than a broken process. + * + * The compiler runs in this JVM, so an out-of-memory or a parser stack overflow is an + * outcome of compiling the user's source - a build error, which the exit contract + * (see `DaemonMain`) says must never exit. A `LinkageError` is a genuine internal fault + * and still exits, so the two are named rather than [Error] caught wholesale. + * + * @param t what escaped the handler. + * @return true to reply ok:false and keep serving, false to let it kill the process. + */ + fun isRequestFailure(t: Throwable): Boolean = t is Exception || t is OutOfMemoryError || t is StackOverflowError + + /** + * Renders a request failure as the one diagnostic the reply carries. + * + * @param t a throw [isRequestFailure] accepted. + * @return user-facing text for the two compiler [Error]s, else the exception's class + * and message, which are for whoever reads the Build Output of an internal fault. + */ + fun describe(t: Throwable): String = + when (t) { + is OutOfMemoryError -> OUT_OF_MEMORY + is StackOverflowError -> STACK_OVERFLOW + else -> "internal: ${t.javaClass.simpleName}: ${t.message}" + } + } +} diff --git a/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2Link.kt b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2Link.kt new file mode 100644 index 0000000000..368acb155c --- /dev/null +++ b/quickbuild/daemon/src/main/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2Link.kt @@ -0,0 +1,317 @@ +package org.appdevforall.cotg.quickbuild.daemon.res + +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import java.io.File +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.zip.ZipFile + +/** + * Rebuilds the app's resource apk with the device-provisioned aapt2 after a resource edit: + * compiles every res dir to `.flat`, then links them against android.jar with the proxy app + * manifest. Every call recompiles and relinks everything, which costs single-digit seconds on a + * phone-sized res tree (see [DEFAULT_TIMEOUT_MILLIS]). + * + * The payload is the whole linked apk, not a bare extracted table: `ResourcesProvider.loadFromTable` + * (API 30+) and the API 28/29 addAssetPath shim both need a file-typed resource's bytes reachable + * from the same archive as the table, so a stripped arsc throws `Resources$NotFoundException` on + * the next activity recreate. + * + * A relink links a strict subset of what the proxy app build's resource merge produced (library + * AAR resources are absent), so three rules keep it safe: + * + * 1. **[stableIds] is mandatory.** aapt2 assigns type ids by declaration order, so a type absent + * here shifts every later type down, and the proxy app's manifest still encodes `android:icon` + * as a fixed numeric id against the baseline table. `--stable-ids` pins each resource to the + * id AGP gave it. + * + * 2. **[libraryResources] must carry both of AGP's library-resource mechanisms.** VALUES + * resources are flattened transitively into the project's own `intermediates/merged_res/`; + * FILE-based ones are not, each library being compiled separately under + * `AndroidArtifacts.ArtifactType.COMPILED_DEPENDENCIES_RESOURCES`. A theme's item values + * reference both kinds, so either piece missing on its own fails the link. + * `--auto-add-overlay` does not help: it only relaxes duplicate checks among the caller's + * own inputs. + * + * 3. **The freshly compiled project resources go in as `-R`, ordered last.** A bare positional + * input always loses to any `-R` input for the same resource whatever the command-line order, + * and only among `-R` inputs does textual order decide - so passing the fresh compile + * positionally would serve merged_res's build-time value for every resource just edited. + * + * @property aapt2 the device-provisioned aapt2 binary, run as a subprocess; must be executable. + * @property androidJar the platform jar, passed to every link as `-I`. + * @property timeoutMillis per-invocation ceiling; an aapt2 that outlasts it is killed and the + * relink fails, and it is injectable so the timeout path is testable in milliseconds. + */ +class Aapt2Link( + private val aapt2: File, + private val androidJar: File, + private val timeoutMillis: Long = DEFAULT_TIMEOUT_MILLIS, +) { + companion object { + /** + * Two minutes per aapt2 invocation. A relink's aapt2 phases cost single-digit seconds on + * a phone-sized res tree [measured on a56, ADFA-4128], so this is ~20x headroom for a + * throttled 2 GB device, while staying under the client's 300 s per-request ceiling + * (`DaemonProcessClient.DEFAULT_REQUEST_TIMEOUT_MILLIS`) - the daemon has to free itself + * before the client gives up, or the next request meets a still-wedged daemon. + */ + const val DEFAULT_TIMEOUT_MILLIS = 120_000L + } + + /** Outcome of one relink. */ + sealed interface Result { + /** + * aapt2 linked a resource apk, with the timings the two phases cost. + * + * @property resourceApk the whole linked apk, verified to contain a `resources.arsc`; + * this is the payload, not a bare table (see class KDoc). + * @property compileMillis wall time of the per-dir `aapt2 compile` loop. + * @property linkMillis wall time of the `aapt2 link` run. + */ + data class Success( + val resourceApk: File, + val compileMillis: Long = 0, + val linkMillis: Long = 0, + ) : Result + + /** + * The relink did not produce a usable apk. + * + * @property diagnostics aapt2's own messages where they parsed, and always at least one + * ERROR - a non-zero exit never reports clean. + */ + data class Failed( + val diagnostics: List, + ) : Result + } + + /** + * Compiles [resDirs] and links the result into a fresh resource apk under [workDir]. + * + * @param resDirs the project's own `res/` roots, each compiled whole; empty means the link + * carries only [libraryResources]. + * @param manifest the proxy app's manifest, already compiled against the baseline table - + * which is why [stableIds] matters (see class KDoc, rule 1). + * @param workDir the daemon-owned scratch dir; its `res-compiled` subdir is wiped on every + * call and `linked-res.apk` is overwritten. + * @param stableIds AGP's `stableIds.txt` mapping (`pkg:type/name = 0x7f0xxxxx`) from the proxy + * app build, passed as `--stable-ids` when readable; null falls back to unpinned + * declaration-order ids (see class KDoc). + * @param libraryResources pre-compiled `.flat` units from the proxy app build - the + * `intermediates/merged_res/` closure plus each AAR's separately-compiled file-based + * resources - without which a library-provided reference fails to link (see class KDoc). + * @return [Result.Failed] when the scratch dir could not be reset, when either aapt2 phase + * exited non-zero, or when the output carries no resource table. + */ + fun relink( + resDirs: List, + manifest: File, + workDir: File, + stableIds: File? = null, + libraryResources: List = emptyList(), + ): Result { + // The compiled dir must start empty: the link globs every .flat in it, so a leftover + // from a previous run - a since-deleted resource's .flat, say - would be linked in as + // a stale resource. A failed reset therefore fails the relink. + val compiledDir = File(workDir, "res-compiled") + if (!compiledDir.deleteRecursively() && compiledDir.listFiles()?.isNotEmpty() == true) { + return Result.Failed( + listOf( + Diagnostic( + Diagnostic.Severity.ERROR, + "failed to clear compiled-resource dir ${compiledDir.absolutePath}; " + + "leftover entries would leak stale .flat files into the link", + ), + ), + ) + } + if (!compiledDir.mkdirs() && !compiledDir.isDirectory) { + return Result.Failed( + listOf( + Diagnostic( + Diagnostic.Severity.ERROR, + "failed to create compiled-resource dir ${compiledDir.absolutePath}", + ), + ), + ) + } + + val compileStartedAt = System.currentTimeMillis() + for (resDir in resDirs) { + val compileResult = + run(listOf(aapt2.absolutePath, "compile", "--dir", resDir.absolutePath, "-o", compiledDir.absolutePath)) + if (compileResult.exitCode != 0) { + return Result.Failed(parseDiagnostics(compileResult.output, "aapt2 compile failed")) + } + } + val compileMillis = System.currentTimeMillis() - compileStartedAt + + val flatFiles = compiledDir.listFiles { file -> file.name.endsWith(".flat") }.orEmpty() + val linkedApk = File(workDir, "linked-res.apk") + linkedApk.delete() + val linkArguments = buildLinkArguments(linkedApk, manifest, flatFiles.toList(), stableIds, libraryResources) + val linkStartedAt = System.currentTimeMillis() + val linkResult = run(linkArguments) + val linkMillis = System.currentTimeMillis() - linkStartedAt + if (linkResult.exitCode != 0) { + return Result.Failed(parseDiagnostics(linkResult.output, "aapt2 link failed")) + } + + return try { + Result.Success(verifyHasTable(linkedApk), compileMillis = compileMillis, linkMillis = linkMillis) + } catch (e: Exception) { + Result.Failed( + listOf(Diagnostic(Diagnostic.Severity.ERROR, "linked apk has no resources.arsc: ${e.message}")), + ) + } + } + + /** + * Assembles the `aapt2 link` command line, with every resource input passed as `-R` and + * [flatFiles] last so the user's fresh edit wins over the baseline (see class KDoc, rule 3). + * `internal` rather than private so the `--stable-ids` behavior is unit-testable without an + * aapt2 binary on the test host, unlike [relink] itself. + * + * @param linkedApk the `-o` target; not created here, only named. + * @param manifest the proxy app's manifest, passed verbatim as `--manifest`; neither read + * nor rewritten here. + * @param flatFiles this run's freshly compiled `.flat` units, appended last so they win. + * @param stableIds null, or a path that does not exist, omits `--stable-ids` entirely. + * @param libraryResources baseline `-R` inputs, emitted ahead of [flatFiles]. + * @return the full argv, aapt2's own path included as element 0. + */ + internal fun buildLinkArguments( + linkedApk: File, + manifest: File, + flatFiles: List, + stableIds: File?, + libraryResources: List = emptyList(), + ): List { + val arguments = + mutableListOf( + aapt2.absolutePath, + "link", + "-o", + linkedApk.absolutePath, + "--manifest", + manifest.absolutePath, + "-I", + androidJar.absolutePath, + "--auto-add-overlay", + ) + if (stableIds != null && stableIds.isFile) { + arguments += listOf("--stable-ids", stableIds.absolutePath) + } + libraryResources.forEach { arguments += listOf("-R", it.absolutePath) } + flatFiles.forEach { arguments += listOf("-R", it.absolutePath) } + return arguments + } + + /** + * Checks that [linkedApk] actually contains a resource table before it ships as the + * payload - a missing entry means aapt2 produced malformed output despite exit 0. Entry + * lookup only, no extraction. + * + * @param linkedApk aapt2's link output, already known to have exited 0. + * @return [linkedApk] unchanged, so the check reads inline at the call site. + * @throws IllegalStateException when the archive holds no `resources.arsc`; [relink] turns + * it, and any zip-level failure, into a [Result.Failed]. + */ + private fun verifyHasTable(linkedApk: File): File { + ZipFile(linkedApk).use { zip -> + zip.getEntry("resources.arsc") + ?: throw IllegalStateException("link output ${linkedApk.name} has no resources.arsc") + } + return linkedApk + } + + private data class ProcessResult( + val exitCode: Int, + val output: String, + ) + + /** + * Runs an aapt2 command, capturing its merged output; a launch failure becomes exit -1. + * + * The output is drained to EOF before the exit code is waited on, since aapt2 can outrun the + * pipe buffer and waiting first would deadlock against a full pipe. That drain is itself + * unbounded, so a wedged aapt2 would stop the single-threaded daemon loop from answering ANY + * request, `ping` and `shutdown` included - hence the watchdog, which kills the child at + * [timeoutMillis] and thereby closes the pipe and releases the read. + * + * @param command the full argv, executable first; run to completion, so the caller blocks. + * @return the exit code and the merged stdout/stderr text, never null and never thrown; a + * timeout reports exit -1 with a message [parseDiagnostics] renders as an ERROR. + */ + private fun run(command: List): ProcessResult { + val process = + try { + // aapt2 reports errors on stderr and notes on stdout, so both are captured + // together. The daemon's own stdout stays protocol-only either way. + ProcessBuilder(command).redirectErrorStream(true).start() + } catch (e: Exception) { + return ProcessResult(-1, "failed to run ${command.firstOrNull()}: ${e.message}") + } + val timedOut = AtomicBoolean(false) + // Daemon thread, so a watchdog still waiting cannot hold up JVM exit. It ends on its + // own as soon as the child does, so nothing interrupts it. + Thread { + if (!process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)) { + timedOut.set(true) + process.destroyForcibly() + } + }.apply { + isDaemon = true + name = "aapt2-watchdog" + start() + } + return try { + val output = process.inputStream.bufferedReader().use { it.readText() } + val exitCode = process.waitFor() + if (timedOut.get()) { + ProcessResult(-1, "aapt2 timed out after $timeoutMillis ms and was killed: ${command.joinToString(" ")}") + } else { + ProcessResult(exitCode, output) + } + } catch (e: Exception) { + ProcessResult(-1, "failed to run ${command.firstOrNull()}: ${e.message}") + } finally { + // A failure on the read path must not orphan the child. + process.destroy() + } + } + + // aapt2 messages look like ":: error: " or "error: ". + private val aapt2Line = Regex("""^(?:(.+?):(?:(\d+):)?\s*)?(error|warn(?:ing)?):\s*(.*)$""") + + /** + * Parses aapt2's output into diagnostics, appending a [fallback] error carrying the raw + * output when nothing in it parsed as an error - a non-zero exit must never report clean. + * + * @param output aapt2's merged stdout/stderr, parsed line by line; unrecognized lines drop. + * @param fallback prefix for the synthesized error, naming which phase failed. + * @return at least one ERROR diagnostic; the fallback carries the raw output, truncated to + * 2000 characters. + */ + private fun parseDiagnostics( + output: String, + fallback: String, + ): List { + val diagnostics = + output + .lineSequence() + .mapNotNull { line -> + val match = aapt2Line.find(line.trim()) ?: return@mapNotNull null + val (file, lineNumber, severity, message) = match.destructured + Diagnostic( + severity = if (severity.startsWith("warn")) Diagnostic.Severity.WARNING else Diagnostic.Severity.ERROR, + message = message, + file = file.ifEmpty { null }, + line = lineNumber.toIntOrNull(), + ) + }.toList() + if (diagnostics.any { it.severity == Diagnostic.Severity.ERROR }) return diagnostics + return diagnostics + Diagnostic(Diagnostic.Severity.ERROR, "$fallback: ${output.trim().take(2000)}") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopErrorTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopErrorTest.kt new file mode 100644 index 0000000000..caab22a50f --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopErrorTest.kt @@ -0,0 +1,117 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.daemon.protocol.DaemonHandlers +import org.appdevforall.cotg.quickbuild.daemon.protocol.RequestRouter +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.io.BufferedReader +import java.io.StringReader +import java.io.StringWriter + +/** + * The loop's own backstop, outside the router's: parse and encode both run on request-sized data + * and neither was wrapped, so a throw from either exited the JVM and CoGo reported daemon death. + * + * Driven through encode, because a value whose `toString` throws is a deterministic way to break + * it - no real memory pressure, no pathological input, and it exercises the exact arm a compile + * response with a huge changed-class list would hit. + */ +class DaemonLoopErrorTest { + /** A response value the codec must stringify, which throws instead. */ + private class ExplodingValue( + private val boom: () -> Nothing, + ) { + override fun toString(): String = boom() + } + + private class RespondingHandlers( + private val response: (Long) -> DaemonResponse, + ) : DaemonHandlers { + override fun configure(request: ConfigureRequest): DaemonResponse = response(request.id) + + override fun compile(request: CompileRequest): DaemonResponse = response(request.id) + + override fun dex(request: DexRequest): DaemonResponse = response(request.id) + + override fun relink(request: RelinkRequest): DaemonResponse = response(request.id) + } + + private fun serve( + boom: () -> Nothing, + vararg lines: String, + ): List { + val output = StringWriter() + DaemonMain.serve( + input = BufferedReader(StringReader(lines.joinToString("\n"))), + output = output, + router = + RequestRouter( + RespondingHandlers { id -> + DaemonResponse.ok(id, mapOf("classesDir" to ExplodingValue(boom))) + }, + ), + ) + return output.toString().lines().filter { it.isNotBlank() } + } + + private val compile = """{"id": 41, "op": "compile", "allSources": [], "changedFiles": []}""" + private val ping = """{"id": 42, "op": "ping"}""" + + @Test + fun `an out-of-memory while encoding replies ok-false on that id and keeps serving`() { + val responses = serve({ throw OutOfMemoryError("Java heap space") }, compile, ping) + + assertThat(responses).hasSize(2) + val failed = JsonParser.parseString(responses[0]).asJsonObject + assertThat(failed.get("ok").asBoolean).isFalse() + assertThat(failed.get("id").asLong).isEqualTo(41) + val message = + failed + .getAsJsonArray("diagnostics") + .single() + .asJsonObject + .get("message") + .asString + assertThat(message).contains("ran out of memory") + + // The half that matters: the loop is still alive to answer the next request. + val served = JsonParser.parseString(responses[1]).asJsonObject + assertThat(served.get("ok").asBoolean).isTrue() + assertThat(served.get("id").asLong).isEqualTo(42) + } + + @Test + fun `a stack overflow while encoding replies ok-false and keeps serving`() { + val responses = serve({ throw StackOverflowError() }, compile, ping) + + assertThat(responses).hasSize(2) + assertThat( + JsonParser + .parseString(responses[0]) + .asJsonObject + .get("ok") + .asBoolean, + ).isFalse() + assertThat( + JsonParser + .parseString(responses[1]) + .asJsonObject + .get("ok") + .asBoolean, + ).isTrue() + } + + @Test + fun `a fatal error still ends the loop, so the exit contract keeps its teeth`() { + assertThrows { + serve({ throw NoClassDefFoundError("com/example/Gone") }, compile, ping) + } + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopTest.kt new file mode 100644 index 0000000000..399663c22d --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonLoopTest.kt @@ -0,0 +1,83 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.daemon.protocol.RequestRouter +import org.junit.jupiter.api.Test +import java.io.BufferedReader +import java.io.StringReader +import java.io.StringWriter + +/** Drives [DaemonMain.serve] over in-memory streams: the protocol loop end to end. */ +class DaemonLoopTest { + private fun serve(vararg lines: String): List { + val output = StringWriter() + DaemonMain.serve( + input = BufferedReader(StringReader(lines.joinToString("\n"))), + output = output, + router = RequestRouter(DaemonService(log = {})), + ) + return output.toString().lines().filter { it.isNotBlank() } + } + + @Test + fun `ping round-trips over the wire`() { + val responses = serve("""{"id": 1, "op": "ping"}""") + + assertThat(responses).hasSize(1) + val root = JsonParser.parseString(responses[0]).asJsonObject + assertThat(root.get("id").asLong).isEqualTo(1) + assertThat(root.get("ok").asBoolean).isTrue() + } + + @Test + fun `malformed request replies ok-false and the loop keeps serving`() { + val responses = + serve( + "not json at all", + """{"id": 2, "op": "ping"}""", + ) + + assertThat(responses).hasSize(2) + val malformed = JsonParser.parseString(responses[0]).asJsonObject + assertThat(malformed.get("ok").asBoolean).isFalse() + assertThat(malformed.get("id").asLong).isEqualTo(-1) + val ping = JsonParser.parseString(responses[1]).asJsonObject + assertThat(ping.get("ok").asBoolean).isTrue() + } + + @Test + fun `blank lines are skipped without a response`() { + val responses = serve("", " ", """{"id": 3, "op": "ping"}""") + + assertThat(responses).hasSize(1) + } + + @Test + fun `shutdown replies then stops serving later requests`() { + val responses = + serve( + """{"id": 4, "op": "shutdown"}""", + """{"id": 5, "op": "ping"}""", + ) + + assertThat(responses).hasSize(1) + val root = JsonParser.parseString(responses[0]).asJsonObject + assertThat(root.get("id").asLong).isEqualTo(4) + assertThat(root.get("ok").asBoolean).isTrue() + } + + @Test + fun `EOF ends the loop cleanly after serving everything`() { + val responses = + serve( + """{"id": 6, "op": "ping"}""", + """{"id": 7, "op": "compile", "allSources": [], "changedFiles": []}""", + ) + + // compile before configure: served (ok:false), then EOF returned normally. + assertThat(responses).hasSize(2) + val compile = JsonParser.parseString(responses[1]).asJsonObject + assertThat(compile.get("ok").asBoolean).isFalse() + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMainTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMainTest.kt new file mode 100644 index 0000000000..cba85d53cc --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonMainTest.kt @@ -0,0 +1,84 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.junit.jupiter.api.Assertions.assertTimeoutPreemptively +import org.junit.jupiter.api.Test +import java.io.ByteArrayInputStream +import java.io.File +import java.time.Duration + +/** + * The process entry point's exit and stream contracts (README): `shutdown` and stdin EOF + * end the loop instead of hanging, and System.out gets redirected away from the protocol + * stream before serving. The serve loop itself is covered stream-by-stream in + * DaemonLoopTest; these run the real main() wiring around it. + */ +class DaemonMainTest { + private fun runMain(stdin: String) { + val originalIn = System.`in` + val originalOut = System.out + try { + System.setIn(ByteArrayInputStream(stdin.toByteArray(Charsets.UTF_8))) + // The exit contract is "returns", and the failure mode is "hangs forever + // waiting on stdin" - so the assertion is a hard timeout around main(). + assertTimeoutPreemptively(Duration.ofSeconds(30)) { DaemonMain.main(emptyArray()) } + // Stdout is protocol-only: anything the compiler prints via System.out must + // have been redirected off the protocol stream. + assertThat(System.out).isNotSameInstanceAs(originalOut) + } finally { + System.setIn(originalIn) + System.setOut(originalOut) + } + } + + @Test + fun `main serves until shutdown, then exits the loop`() { + runMain("""{"id": 1, "op": "shutdown"}""" + "\n") + } + + @Test + fun `main exits cleanly on stdin EOF without any request`() { + runMain("") + } + + /** + * In-process, the redirect is all that can be seen: main() captures the real stdout BEFORE + * redirecting System.out, so both ends live in this same JVM and writing responses to the + * redirected System.out instead - the mutation the DaemonMain KDoc warns about - looks + * identical from here. It is not: responses would land on stderr and CoGo would read an + * empty protocol stream. Only a child process can tell the two file descriptors apart. + */ + @Test + fun `responses reach the process stdout, never the redirected System out`() { + val java = File(File(System.getProperty("java.home"), "bin"), "java") + val process = + ProcessBuilder( + java.absolutePath, + "-cp", + System.getProperty("java.class.path"), + DaemonMain::class.java.name, + ).start() + + try { + assertTimeoutPreemptively(Duration.ofSeconds(60)) { + process.outputStream.writer(Charsets.UTF_8).use { it.write("""{"id": 7, "op": "shutdown"}""" + "\n") } + val stdout = process.inputStream.readBytes().toString(Charsets.UTF_8) + val stderr = process.errorStream.readBytes().toString(Charsets.UTF_8) + + assertThat(process.waitFor()).isEqualTo(0) + // One line on stdout and it IS the response: nothing else may share the stream, + // and an EMPTY stdout is the redirect-swallowed-it failure this test exists for. + val lines = stdout.lines().filter { it.isNotBlank() } + assertThat(lines).hasSize(1) + val response = JsonParser.parseString(lines.single()).asJsonObject + assertThat(response.get("id").asLong).isEqualTo(7) + assertThat(response.get("ok").asBoolean).isTrue() + // The daemon's own logging went the other way, where it cannot corrupt anything. + assertThat(stderr).contains("[quickbuild-daemon] started") + } + } finally { + process.destroyForcibly() + } + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceOpsTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceOpsTest.kt new file mode 100644 index 0000000000..15466dd92f --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceOpsTest.kt @@ -0,0 +1,279 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.DexStats +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * The configured-session op paths of [DaemonService]: how each op's tool result becomes a + * protocol response - failures as ok:false with diagnostics, successes carrying the + * artifact paths and timings the client deploys and logs from. Complements + * DaemonServiceTest, which covers configure validation and the compile happy path. + */ +class DaemonServiceOpsTest { + @TempDir + lateinit var tempDir: File + + private val service = DaemonService(log = {}) + + private fun configure( + aapt2: File = TestSdk.kotlinStdlib(), + d8Jar: File = TestSdk.kotlinStdlib(), + androidJar: File = TestSdk.kotlinStdlib(), + compilerPlugins: List = emptyList(), + service: DaemonService = this.service, + ) { + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(TestSdk.kotlinStdlib().absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = aapt2.absolutePath, + d8Jar = d8Jar.absolutePath, + androidJar = androidJar.absolutePath, + compilerPlugins = compilerPlugins, + ), + ) + check(response.ok) { "fixture configure failed: ${response.diagnostics}" } + } + + @Test + fun `a compile failure responds ok-false with the compiler's diagnostics`() { + configure() + val broken = File(tempDir, "Broken.kt").apply { writeText("package demo\n\nfun broken(: Int\n") } + + val response = service.compile(CompileRequest(2, listOf(broken.absolutePath), listOf(broken.absolutePath))) + + assertThat(response.ok).isFalse() + assertThat(response.diagnostics).isNotEmpty() + assertThat(response.diagnostics.all { it.severity == Diagnostic.Severity.ERROR }).isTrue() + } + + @Test + fun `a dex failure responds ok-false with the tool's message`() { + configure() + val emptyDir = File(tempDir, "no-classes").apply { mkdirs() } + + val response = service.dex(DexRequest(3, listOf(emptyDir.absolutePath))) + + assertThat(response.ok).isFalse() + assertThat(response.diagnostics.single().message).contains("no .class files") + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `compile then dex produces a classes dex under the session's out dir`() { + configure(d8Jar = TestSdk.d8Jar()!!, androidJar = TestSdk.androidJar()!!) + val source = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + val compile = service.compile(CompileRequest(2, listOf(source.absolutePath), listOf(source.absolutePath))) + check(compile.ok) { "fixture compile failed: ${compile.diagnostics}" } + + val response = service.dex(DexRequest(3, listOf(compile.values["classesDir"] as String))) + + assertThat(response.ok).isTrue() + val dexFile = File(response.values["dexFile"] as String) + assertThat(dexFile.isFile).isTrue() + assertThat(dexFile.name).isEqualTo("classes.dex") + assertThat(dexFile.absolutePath).startsWith(File(tempDir, "out").absolutePath) + // The timing/stat fields a slow row is read by. + assertThat((response.values["durationMillis"] as Long)).isAtLeast(0) + assertThat((response.values["stripMillis"] as Long)).isAtLeast(0) + assertThat((response.values["d8Millis"] as Long)).isAtLeast(0) + val stats = DexStats.fromValues { key -> (response.values[key] as? Number)?.toLong() }!! + assertThat(stats.classFiles).isEqualTo(1) + assertThat(stats.classBytes).isGreaterThan(0) + } + + @Test + fun `a relink failure responds ok-false with error diagnostics`() { + // The stdlib jar stands in for aapt2: it exists (passes configure) but cannot be + // executed, so the relink's aapt2 compile step fails and must surface as a + // response, never a throw. + configure() + val resDir = File(tempDir, "res/values").apply { mkdirs() }.parentFile + File(resDir, "values/strings.xml").writeText("") + val manifest = File(tempDir, "AndroidManifest.xml").apply { writeText("") } + val stableIds = File(tempDir, "stableIds.txt").apply { writeText("demo:string/app_name = 0x7f010000") } + + // stableIds and libraryResources ride through to the tool even on a failing run. + val response = + service.relink( + RelinkRequest( + 4, + listOf(resDir.absolutePath), + manifest.absolutePath, + stableIds = stableIds.absolutePath, + libraryResources = listOf(File(tempDir, "lib.flat").absolutePath), + ), + ) + + assertThat(response.ok).isFalse() + assertThat(response.diagnostics).isNotEmpty() + assertThat(response.diagnostics.any { it.severity == Diagnostic.Severity.ERROR }).isTrue() + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `a relink success carries the linked resource apk and the aapt2 phase timings`() { + configure(aapt2 = TestSdk.aapt2()!!, androidJar = TestSdk.androidJar()!!) + val resDir = File(tempDir, "res/values").apply { mkdirs() }.parentFile + File(resDir, "values/strings.xml").writeText( + """ + + + Quick Build Demo + + """.trimIndent(), + ) + val manifest = + File(tempDir, "AndroidManifest.xml").apply { + writeText( + """ + + + + + """.trimIndent(), + ) + } + + val response = service.relink(RelinkRequest(5, listOf(resDir.absolutePath), manifest.absolutePath)) + + assertThat(response.ok).isTrue() + // Wire name kept as "resourcesArsc" for protocol stability; payload is the full apk. + val resourceApk = File(response.values["resourcesArsc"] as String) + assertThat(resourceApk.isFile).isTrue() + assertThat(resourceApk.length()).isGreaterThan(0) + assertThat(resourceApk.absolutePath).startsWith(File(tempDir, "out").absolutePath) + assertThat((response.values["durationMillis"] as Long)).isAtLeast(0) + assertThat((response.values["aapt2CompileMillis"] as Long)).isAtLeast(0) + assertThat((response.values["aapt2LinkMillis"] as Long)).isAtLeast(0) + } + + @Test + fun `configure accepts session-fixed compiler plugins that exist on disk`() { + // The jar's content is irrelevant at configure time - only existence is validated; + // a MISSING plugin path must fail configure like any other missing input. + configure(compilerPlugins = listOf(TestSdk.kotlinStdlib().absolutePath)) + + val missing = + service.configure( + ConfigureRequest( + id = 9, + projectRoot = tempDir.absolutePath, + classpath = emptyList(), + outDir = File(tempDir, "out").absolutePath, + aapt2 = TestSdk.kotlinStdlib().absolutePath, + d8Jar = TestSdk.kotlinStdlib().absolutePath, + androidJar = TestSdk.kotlinStdlib().absolutePath, + compilerPlugins = listOf(File(tempDir, "no-such-plugin.jar").absolutePath), + ), + ) + + assertThat(missing.ok).isFalse() + assertThat(missing.diagnostics.single().message).contains("no-such-plugin.jar") + } + + @Test + fun `a configure that throws does not release the live session's tools`() { + // A session's tools are released only once its replacement exists. Releasing first + // stranded the still-installed session with a closed r8 class loader and a finished + // compilation project - and the damage is LATENT, because a closed URLClassLoader still + // serves the classes it already loaded, so it surfaces later as a NoClassDefFoundError + // from inside d8 rather than at the close. The ordering is therefore asserted directly. + val lines = mutableListOf() + val loggingService = DaemonService(log = { lines += it }) + configure(service = loggingService) + val source = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + val compile = { id: Long -> + loggingService.compile(CompileRequest(id, listOf(source.absolutePath), listOf(source.absolutePath))) + } + check(compile(2).ok) { "fixture compile failed" } + // A classpath entry that exists but is not a zip: passes configure's existence check, + // then throws inside classpath snapshotting - the realistic corrupt-AAR shape. + val corruptJar = File(tempDir, "corrupt.jar").apply { writeText("not a jar") } + + val reconfigure = + runCatching { + loggingService.configure( + ConfigureRequest( + id = 3, + projectRoot = tempDir.absolutePath, + classpath = listOf(corruptJar.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = TestSdk.kotlinStdlib().absolutePath, + d8Jar = TestSdk.kotlinStdlib().absolutePath, + androidJar = TestSdk.kotlinStdlib().absolutePath, + ), + ) + } + + // Assert the THROWING path specifically: an ok:false return exercises none of this, so + // the test would quietly stop covering the bug if snapshotting ever stopped throwing. + assertThat(reconfigure.isFailure).isTrue() + assertThat(lines.none { it.contains("released the previous session") }).isTrue() + assertThat(compile(4).ok).isTrue() + // A re-configure that SUCCEEDS must still release, or the leak this guards is real in + // the other direction. + configure(service = loggingService) + assertThat(lines.any { it.contains("released the previous session") }).isTrue() + } + + @Test + fun `shutdown releases the session and is safe to repeat`() { + configure() + + service.shutdown() + service.shutdown() + + val response = service.compile(CompileRequest(2, emptyList(), emptyList())) + assertThat(response.ok).isFalse() + assertThat(response.diagnostics.single().message).contains("not configured") + } + + @Test + fun `the default logger writes session lines to stderr, not stdout`() { + // Stdout is protocol-only (README): a stray log line there would corrupt the + // stream. The default log sink must therefore be stderr. + val defaultLogService = DaemonService() + val originalOut = System.out + val originalErr = System.err + val capturedOut = java.io.ByteArrayOutputStream() + val capturedErr = java.io.ByteArrayOutputStream() + try { + System.setOut(java.io.PrintStream(capturedOut, true, "UTF-8")) + System.setErr(java.io.PrintStream(capturedErr, true, "UTF-8")) + val response = + defaultLogService.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = emptyList(), + outDir = File(tempDir, "out").absolutePath, + aapt2 = TestSdk.kotlinStdlib().absolutePath, + d8Jar = TestSdk.kotlinStdlib().absolutePath, + androidJar = TestSdk.kotlinStdlib().absolutePath, + ), + ) + assertThat(response.ok).isTrue() + } finally { + System.setOut(originalOut) + System.setErr(originalErr) + } + assertThat(capturedOut.toString("UTF-8")).isEmpty() + // Asserting stderr received the line is what makes this a logging test: without + // it, deleting the logging entirely would still pass "nothing on stdout". + assertThat(capturedErr.toString("UTF-8")).contains("configure") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceTest.kt new file mode 100644 index 0000000000..045cc1c8c8 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/DaemonServiceTest.kt @@ -0,0 +1,309 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.CompileStats +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.ResponseKeys +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +class DaemonServiceTest { + @TempDir + lateinit var tempDir: File + + private val service = DaemonService(log = {}) + + @Test + fun `build ops before configure fail with a clear message`() { + val compile = service.compile(CompileRequest(1, emptyList(), emptyList())) + val dex = service.dex(DexRequest(2, emptyList())) + val relink = service.relink(RelinkRequest(3, emptyList(), "/M.xml")) + + for (response in listOf(compile, dex, relink)) { + assertThat(response.ok).isFalse() + assertThat(response.diagnostics.single().message).contains("configure") + } + } + + @Test + fun `configure with missing files fails and names them`() { + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(File(tempDir, "no-such.jar").absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = File(tempDir, "no-such-aapt2").absolutePath, + d8Jar = File(tempDir, "no-such-r8.jar").absolutePath, + androidJar = File(tempDir, "no-such-android.jar").absolutePath, + ), + ) + + assertThat(response.ok).isFalse() + assertThat(response.diagnostics.single().message).contains("no-such.jar") + assertThat(response.diagnostics.single().message).contains("no-such-aapt2") + } + + @Test + fun `configure then compile runs the real pipeline`() { + val stdlib = TestSdk.kotlinStdlib() + // aapt2/d8Jar/androidJar only need to exist for configure; use the stdlib jar + // as a stand-in so this test runs without an Android SDK. + val configure = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + assertThat(configure.ok).isTrue() + + val source = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + val compile = + service.compile(CompileRequest(2, listOf(source.absolutePath), listOf(source.absolutePath))) + + assertThat(compile.ok).isTrue() + val classesDir = File(compile.values["classesDir"] as String) + assertThat(File(classesDir, "demo/HelloKt.class").isFile).isTrue() + assertThat(compile.values["durationMillis"]).isNotNull() + // The deploy-policy signal: this run's emitted class files. + assertThat(compile.values["classesChanged"]).isEqualTo(listOf("demo/HelloKt.class")) + } + + @Test + fun `configure success stamps the protocol version`() { + val stdlib = TestSdk.kotlinStdlib() + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + + assertThat(response.ok).isTrue() + assertThat(response.values["protocolVersion"]).isEqualTo(DaemonResponse.PROTOCOL_VERSION) + } + + @Test + fun `configure reports the scratch tree's filesystem`() { + // Session-constant context for every later timing: per-file work costs ~52x more on + // FUSE-backed emulated storage than on a real one (measured under ADFA-4128). + val stdlib = TestSdk.kotlinStdlib() + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + + assertThat(response.ok).isTrue() + val fsType = response.values[ResponseKeys.SCRATCH_FS_TYPE] as String + // The value is host-dependent (apfs here, f2fs/fuse on device); what must hold is + // that a real type was resolved rather than the unknown fallback. + assertThat(fsType).isNotEmpty() + assertThat(fsType).isNotEqualTo("unknown") + } + + @Test + fun `compile reports the phases kotlinMillis and javaMillis do not cover`() { + val stdlib = TestSdk.kotlinStdlib() + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + val source = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + + val first = service.compile(CompileRequest(2, listOf(source.absolutePath), listOf(source.absolutePath))) + source.writeText("package demo\n\nfun hello() = \"hello\"\n") + val second = service.compile(CompileRequest(3, listOf(source.absolutePath), listOf(source.absolutePath))) + + val firstStats = CompileStats.fromValues { key -> (first.values[key] as? Number)?.toLong() }!! + assertThat(firstStats.allSources).isEqualTo(1) + assertThat(firstStats.javaSources).isEqualTo(0) + assertThat(firstStats.kotlinToCompile).isEqualTo(1) + assertThat(firstStats.changedClasses).isEqualTo(1) + // The cold build of the session - the distinction that keeps a first build from + // being read as a per-edit cost. + assertThat(firstStats.compileOrdinal).isEqualTo(1) + assertThat(firstStats.preSnapMillis).isAtLeast(0) + assertThat(firstStats.postSnapMillis).isAtLeast(0) + + val secondStats = CompileStats.fromValues { key -> (second.values[key] as? Number)?.toLong() }!! + assertThat(secondStats.compileOrdinal).isEqualTo(2) + } + + @Test + fun `a FAILED compile still reports the stats, which is the build we most need them from`() { + // The field that identifies a mixed-language staleness bug is kotlinToCompile: 0 means the + // .kt never reached the declared changed set, >= 1 means it did and the staleness is + // elsewhere. Those are different fixes. Dropping the stats on the failure path is what + // makes them indistinguishable, so this is the build the numbers matter most on. + val stdlib = TestSdk.kotlinStdlib() + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + val kotlin = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + // Fails in JAVAC, not kotlinc - the branch the mixed-language defect actually takes. + val java = + File(tempDir, "Broken.java").apply { + writeText("package demo;\n\npublic class Broken { public int broken() { return \"nope\"; } }\n") + } + val sources = listOf(kotlin.absolutePath, java.absolutePath) + + val response = service.compile(CompileRequest(2, sources, sources)) + + assertThat(response.ok).isFalse() + // fromValues returns null when the keys are ABSENT, so this asserts the stats were + // carried at all - the actual defect - rather than that some value is right. + val stats = CompileStats.fromValues { key -> (response.values[key] as? Number)?.toLong() } + assertThat(stats).isNotNull() + assertThat(stats!!.allSources).isEqualTo(2) + assertThat(stats.javaSources).isEqualTo(1) + assertThat(stats.kotlinToCompile).isEqualTo(1) + assertThat(stats.compileOrdinal).isEqualTo(1) + // The diagnostics must survive the change that adds the stats. + assertThat(response.diagnostics.any { it.file?.endsWith("Broken.java") == true }).isTrue() + } + + @Test + fun `a compile failing in kotlinc also reports its stats`() { + val stdlib = TestSdk.kotlinStdlib() + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + val broken = File(tempDir, "Broken.kt").apply { writeText("package demo\n\nfun oops(: Int\n") } + + val response = service.compile(CompileRequest(2, listOf(broken.absolutePath), listOf(broken.absolutePath))) + + assertThat(response.ok).isFalse() + val stats = CompileStats.fromValues { key -> (response.values[key] as? Number)?.toLong() } + assertThat(stats).isNotNull() + assertThat(stats!!.allSources).isEqualTo(1) + assertThat(stats.kotlinToCompile).isEqualTo(1) + } + + @Test + fun `a fresh configure restarts the compile ordinal`() { + // A respawn re-pays the cold cost, so its next compile is a cold build again. + val stdlib = TestSdk.kotlinStdlib() + val configure = { + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = listOf(stdlib.absolutePath), + outDir = File(tempDir, "out").absolutePath, + aapt2 = stdlib.absolutePath, + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + } + val source = File(tempDir, "Hello.kt").apply { writeText("package demo\n\nfun hello() = \"hi\"\n") } + val compile = { id: Long -> + service.compile(CompileRequest(id, listOf(source.absolutePath), listOf(source.absolutePath))) + } + + configure() + compile(2) + compile(3) + configure() + val afterReconfigure = compile(4) + + val stats = CompileStats.fromValues { key -> (afterReconfigure.values[key] as? Number)?.toLong() }!! + assertThat(stats.compileOrdinal).isEqualTo(1) + } + + @Test + fun `configure without aapt2, d8Jar or androidJar fails naming each unsupplied path`() { + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = emptyList(), + outDir = File(tempDir, "out").absolutePath, + ), + ) + + // The daemon never guesses a tool path, so an omission has to say which field is + // missing - the alternative is a silently wrong SDK that only fails on device. + assertThat(response.ok).isFalse() + val messages = response.diagnostics.map { it.message } + assertThat(messages).hasSize(3) + assertThat(messages.any { it.contains("aapt2") }).isTrue() + assertThat(messages.any { it.contains("d8Jar") }).isTrue() + assertThat(messages.any { it.contains("androidJar") }).isTrue() + assertThat(messages.all { it.contains("not supplied") }).isTrue() + } + + @Test + fun `configure with a blank tool path is treated as unsupplied, not as a missing file`() { + val stdlib = TestSdk.kotlinStdlib() + + val response = + service.configure( + ConfigureRequest( + id = 1, + projectRoot = tempDir.absolutePath, + classpath = emptyList(), + outDir = File(tempDir, "out").absolutePath, + aapt2 = "", + d8Jar = stdlib.absolutePath, + androidJar = stdlib.absolutePath, + ), + ) + + assertThat(response.ok).isFalse() + val messages = response.diagnostics.map { it.message } + assertThat(messages).hasSize(1) + assertThat(messages.single()).contains("aapt2") + assertThat(messages.single()).contains("not supplied") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/OfflineNetworkGuardTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/OfflineNetworkGuardTest.kt new file mode 100644 index 0000000000..9e0bfdb9a1 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/OfflineNetworkGuardTest.kt @@ -0,0 +1,69 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.appdevforall.cotg.quickbuild.testfixtures.OfflineGuard +import org.junit.jupiter.api.Test + +/** + * Offline guard (ADFA-4128 offline-test-plan touchpoints 7-10): the hot loop must make zero network + * calls, so this scans the module's compiled production classes for constant-pool references to a + * network API and fails naming the offending class and constant. Running in the normal `test` task + * catches e.g. a new OkHttp call in CI, not on a device walk. `java.net.URL`/`URI`/`URLClassLoader` + * are allowed: the daemon loads the bundled local `d8.jar` from a `file:` URI (see [dex.DexTool]). + */ +class OfflineNetworkGuardTest { + @Test + fun productionClassesReferenceNoNetworkApis() { + val buildDir = OfflineGuard.moduleBuildDir(javaClass) + val classFiles = OfflineGuard.productionClassFiles(buildDir) + + // Anti-vacuous: a mis-location must fail loudly, never pass by scanning nothing. + assertWithMessage("no production .class files found under $buildDir -- guard self-location is broken") + .that(classFiles) + .isNotEmpty() + + val violations = OfflineGuard.scanForBannedReferences(buildDir, classFiles) + assertWithMessage( + "Quick Build must be network-free offline, but production classes reference banned network APIs:\n" + + violations.joinToString("\n") { " - $it" } + + "\n(scanned ${classFiles.size} classes under $buildDir)", + ).that(violations) + .isEmpty() + } + + /** + * Proves the detector would genuinely fail if a banned reference appeared, and that + * the allow-listed local-URL APIs do NOT trip it -- so a green result above is a real + * signal, not a scanner that can never fire. + */ + @Test + fun detectorFiresOnBannedBytesAndNotOnAllowedBytes() { + val banned = + "prefix Lokhttp3/OkHttpClient; and java/net/Socket suffix" + .toByteArray(Charsets.US_ASCII) + assertThat(OfflineGuard.BANNED.filter { OfflineGuard.containsAscii(banned, it) }) + .containsExactly("okhttp3/", "java/net/Socket") + + val allowed = + "Ljava/net/URL; Ljava/net/URLClassLoader; Ljava/net/URI;" + .toByteArray(Charsets.US_ASCII) + assertThat(OfflineGuard.BANNED.filter { OfflineGuard.containsAscii(allowed, it) }) + .isEmpty() + } + + /** + * The daemon really does load d8 via a `file:` `URLClassLoader`, so the allow-listed + * constant is present in production bytes. Asserting it doubles as proof the scanner + * reads real class bytes (not an empty set) for this module. + */ + @Test + fun documentedLocalUrlClassLoaderExceptionIsPresentInProductionBytes() { + val buildDir = OfflineGuard.moduleBuildDir(javaClass) + val hasUrlClassLoader = + OfflineGuard.productionClassFiles(buildDir).any { f -> + OfflineGuard.containsAscii(f.readBytes(), "java/net/URLClassLoader") + } + assertThat(hasUrlClassLoader).isTrue() + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/TestSdk.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/TestSdk.kt new file mode 100644 index 0000000000..32f296450f --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/TestSdk.kt @@ -0,0 +1,100 @@ +package org.appdevforall.cotg.quickbuild.daemon + +import java.io.File + +/** + * Locates a host Android SDK for the d8/aapt2 tests, which are assumption-guarded (`@EnabledIf`) + * because hosts without an SDK can't run them. On device the paths arrive in the configure request; + * the daemon never uses this. `REQUIRE_BUILD_TOOLCHAIN=1` / `-PrequireBuildToolchain` (both wired + * to `quickbuild.test.requireToolchain`) turn an absent toolchain from a silent skip into a test + * error, so CI can never skip the aapt2/d8/Compose regressions (ADFA-4128 bugs 5/6/8). + */ +object TestSdk { + private fun toolchainRequired(): Boolean = System.getProperty("quickbuild.test.requireToolchain").toBoolean() + + private fun requireOrSkip( + available: Boolean, + what: String, + ): Boolean { + check(available || !toolchainRequired()) { + "REQUIRE_BUILD_TOOLCHAIN is set but the $what is unavailable on this host - " + + "these tests must run, not skip (SDK roots tried: ANDROID_HOME, ANDROID_SDK_ROOT, " + + "~/Android/Sdk, ~/Library/Android/sdk; Compose jars are staged by the build)." + } + return available + } + + private val sdkRoot: File? by lazy { + sequenceOf( + System.getenv("ANDROID_HOME"), + System.getenv("ANDROID_SDK_ROOT"), + System.getProperty("user.home") + "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/Android/Sdk", + System.getProperty("user.home") + "/Library/Android/sdk", + ).filterNotNull() + .map(::File) + .firstOrNull { it.isDirectory } + } + + /** + * Orders an SDK directory name by its numeric components, so `35.0.0` beats `9.0.0` and + * `android-36` beats `android-9`. A lexical max gets both backwards, and picks a toolchain + * old enough that the failure reads as a daemon bug rather than a test-helper one. + */ + private fun versionKey(name: String): List = Regex("\\d+").findAll(name).map { it.value.toInt() }.toList() + + private val byVersion: Comparator = + Comparator { left, right -> + val a = versionKey(left.name) + val b = versionKey(right.name) + var result = 0 + for (i in 0 until maxOf(a.size, b.size)) { + result = (a.getOrElse(i) { 0 }).compareTo(b.getOrElse(i) { 0 }) + if (result != 0) break + } + result + } + + private fun newestBuildTools(): File? = + sdkRoot + ?.resolve("build-tools") + ?.listFiles { file -> file.isDirectory } + ?.maxWithOrNull(byVersion) + + fun d8Jar(): File? = newestBuildTools()?.resolve("lib/d8.jar")?.takeIf { it.isFile } + + fun aapt2(): File? = newestBuildTools()?.resolve("aapt2")?.takeIf { it.canExecute() } + + fun androidJar(): File? = + sdkRoot + ?.resolve("platforms") + ?.listFiles { file -> file.isDirectory && file.name.startsWith("android-") } + ?.maxWithOrNull(byVersion) + ?.resolve("android.jar") + ?.takeIf { it.isFile } + + @JvmStatic + fun dexToolchainAvailable(): Boolean = requireOrSkip(d8Jar() != null && androidJar() != null, "d8/android.jar toolchain") + + @JvmStatic + fun aapt2ToolchainAvailable(): Boolean = requireOrSkip(aapt2() != null && androidJar() != null, "aapt2/android.jar toolchain") + + /** The kotlin-stdlib jar the test JVM itself runs against; compile-test classpath. */ + fun kotlinStdlib(): File = + System + .getProperty("java.class.path") + .split(File.pathSeparator) + .map(::File) + .first { it.name.startsWith("kotlin-stdlib") && it.extension == "jar" } + + /** The Compose compiler plugin jar; staged by the build (see build.gradle.kts). */ + fun composePluginJar(): File? = fileProperty("quickbuild.test.composePluginJar") + + /** Compose runtime classes.jar extracted from the AAR by the build. */ + fun composeRuntimeJar(): File? = fileProperty("quickbuild.test.composeRuntimeJar") + + @JvmStatic + fun composeToolchainAvailable(): Boolean = + requireOrSkip(composePluginJar() != null && composeRuntimeJar() != null, "staged Compose compiler/runtime") + + private fun fileProperty(name: String): File? = System.getProperty(name)?.let(::File)?.takeIf { it.isFile } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerEdgeTest.kt new file mode 100644 index 0000000000..ea5ab37095 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerEdgeTest.kt @@ -0,0 +1,435 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.daemon.TestSdk +import org.appdevforall.cotg.quickbuild.daemon.protocol.ProtocolCodec +import org.appdevforall.cotg.quickbuild.protocol.CompileStats +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * Edges around IncrementalCompilerTest's happy paths: language-subset source sets, the + * conservative fallback when the Java ABI cannot be known, and the removed-Java output + * cleanup's path mapping (nested classes, unusual source roots, unrelated paths). + */ +class IncrementalCompilerEdgeTest { + @TempDir + lateinit var tempDir: File + + private lateinit var srcDir: File + private lateinit var workDir: File + + @BeforeEach + fun setUp() { + srcDir = File(tempDir, "src").apply { mkdirs() } + workDir = File(tempDir, "work").apply { mkdirs() } + } + + private fun compiler() = IncrementalCompiler(listOf(TestSdk.kotlinStdlib()), workDir.toPath()) + + private fun writeJava( + relativePath: String, + content: String, + ): File = + File(srcDir, relativePath).apply { + parentFile!!.mkdirs() + writeText(content) + } + + private fun widgetJava(relativePath: String = "main/java/demo/Widget.java"): File = + writeJava(relativePath, "package demo;\n\npublic class Widget { public int v() { return 1; } }") + + private fun kotlinSource(greeting: String = "hi"): File = + File(srcDir, "Greeter.kt").apply { + writeText("package demo\n\nclass Greeter { fun hi() = \"$greeting\" }\n") + } + + @Test + fun `a java-only source set compiles through javac alone`() { + val widget = widgetJava() + val compiler = compiler() + + val result = compiler.compile(listOf(widget), changedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val success = result as IncrementalCompiler.Result.Success + assertThat(File(success.classesDir, "demo/Widget.class").isFile).isTrue() + assertThat(success.stats.javaSources).isEqualTo(1) + // No Kotlin sources: nothing for kotlinc to do, and the stat must say so. + assertThat(success.stats.kotlinToCompile).isEqualTo(0) + } + + @Test + fun `a java source that disappears from disk fails the compile, not the daemon`() { + val widget = widgetJava() + val kotlin = kotlinSource() + val compiler = compiler() + val sources = listOf(kotlin, widget) + val first = compiler.compile(sources, changedFiles = sources) + check(first is IncrementalCompiler.Result.Success) { "fixture compile failed" } + + // Still listed in allSources but gone from disk (an editor race CoGo cannot + // prevent): the missing file must surface as an ordinary compile failure the + // client can render, never as a daemon-killing throw. + assertThat(widget.delete()).isTrue() + val result = compiler.compile(sources, changedFiles = emptyList()) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + assertThat((result as IncrementalCompiler.Result.Failed).diagnostics).isNotEmpty() + } + + @Test + fun `a success without timings encodes as numeric zeros the client reads back as measured`() { + // "0 means unmeasured, never -1 and never a string" is a wire contract, so assert it on + // the wire: the same keys DaemonService.compile writes, through the real encoder, read + // back the way DaemonProcessClient reads them (JSON-number guard, else null). + val success = + IncrementalCompiler.Result.Success( + classesDir = File("/classes"), + warnings = emptyList(), + changedClassFiles = emptyList(), + ) + + val encoded = + ProtocolCodec.encode( + DaemonResponse.ok( + id = 7L, + values = + mapOf( + "classesDir" to success.classesDir.absolutePath, + "kotlinMillis" to success.kotlinMillis, + "javaMillis" to success.javaMillis, + ) + success.stats.toValues(), + ), + ) + + val json = JsonParser.parseString(encoded).asJsonObject + val readLong = { key: String -> json.get(key)?.takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isNumber }?.asLong } + assertThat(readLong("kotlinMillis")).isEqualTo(0L) + assertThat(readLong("javaMillis")).isEqualTo(0L) + // Present-and-zero, not absent: null here would tell the client this daemon predates + // the stats group, and a -1 sentinel in any field would fail the equality. + assertThat(CompileStats.fromValues(readLong)).isEqualTo(CompileStats()) + } + + @Test + fun `the logger routes each channel to its collection with a level tag`() { + val emitted = mutableListOf() + val logger = IncrementalCompiler.CollectingLogger(emitted::add) + + logger.error("boom", null) + logger.warn("careful", null) + logger.info("fyi") + logger.debug("details") + logger.lifecycle("phase") + + // errors/warnings feed structured diagnostics; every line is forwarded to the sink + // and nothing else is retained. + assertThat(logger.errors).containsExactly("boom") + assertThat(logger.warnings).containsExactly("careful") + assertThat(emitted) + .containsExactly("e: boom", "w: careful", "i: fyi", "d: details", "l: phase") + .inOrder() + assertThat(logger.isDebugEnabled).isTrue() + } + + @Test + fun `removing a java source deletes its nested classes but not a sibling's outputs`() { + val widget = + writeJava( + "main/java/demo/Widget.java", + "package demo;\n\npublic class Widget {\n\tpublic class Inner {}\n}\n", + ) + val sibling = writeJava("main/java/demo/Widget2.java", "package demo;\n\npublic class Widget2 {}\n") + val compiler = compiler() + val first = compiler.compile(listOf(widget, sibling), changedFiles = listOf(widget, sibling)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget\$Inner.class").isFile).isTrue() + // A non-class file sharing the nested-class prefix must survive the sweep. + val notes = File(classesDir, "demo/Widget\$notes.txt").apply { writeText("keep") } + + assertThat(widget.delete()).isTrue() + val result = compiler.compile(listOf(sibling), changedFiles = emptyList(), removedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + assertThat(File(classesDir, "demo/Widget\$Inner.class").exists()).isFalse() + assertThat(File(classesDir, "demo/Widget2.class").isFile).isTrue() + assertThat(notes.isFile).isTrue() + } + + @Test + fun `a nested class edited out of a surviving java source leaves no stale output`() { + // javac deletes nothing for a source it recompiles, so a declaration edited away leaves + // its output behind - untouched, therefore invisible to the output diff, and re-dexed into + // every later payload. Dead classes then accumulate against the single-dex ceiling, which + // is a hard failure, and the removed class still resolves by name through the payload + // loader. + val widget = + writeJava( + "main/java/demo/Widget.java", + "package demo;\n\npublic class Widget {\n" + + "\tpublic class Helper {}\n" + + "\tpublic Runnable r = new Runnable() { public void run() {} };\n" + + "}\n", + ) + val sibling = writeJava("main/java/demo/Widget2.java", "package demo;\n\npublic class Widget2 {}\n") + val compiler = compiler() + val first = compiler.compile(listOf(widget, sibling), changedFiles = listOf(widget, sibling)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget\$Helper.class").isFile).isTrue() + assertThat(File(classesDir, "demo/Widget\$1.class").isFile).isTrue() + + widget.writeText("package demo;\n\npublic class Widget {}\n") + val result = compiler.compile(listOf(widget, sibling), changedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget\$Helper.class").exists()).isFalse() + assertThat(File(classesDir, "demo/Widget\$1.class").exists()).isFalse() + // The primary class is swept too, but javac regenerates it in the same build. + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + // An untouched sibling must not be swept. + assertThat(File(classesDir, "demo/Widget2.class").isFile).isTrue() + // The deploy policy has to SEE the removals, or it cannot know a component's nested class + // went away - which is why the sweep runs after the pre-snapshot. + val changed = (result as IncrementalCompiler.Result.Success).changedClassFiles + assertThat(changed).contains("demo/Widget\$Helper.class") + assertThat(changed).contains("demo/Widget\$1.class") + } + + @Test + fun `a removed java path with no source-root marker is skipped without touching outputs`() { + val widget = widgetJava() + val compiler = compiler() + val first = compiler.compile(listOf(widget), changedFiles = listOf(widget)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + + // No java/kotlin segment anywhere: the stem cannot be derived, so nothing may be + // guessed at and deleted. + val unrooted = File(tempDir, "flat/demo/Widget.java") + val result = compiler.compile(listOf(widget), changedFiles = emptyList(), removedFiles = listOf(unrooted)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + } + + @Test + fun `a removed java under a non-main java root falls back to the last root marker`() { + val widget = widgetJava("custom/java/demo/Widget.java") + val compiler = compiler() + val first = compiler.compile(listOf(widget), changedFiles = listOf(widget)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + + assertThat(widget.delete()).isTrue() + val result = compiler.compile(emptyList(), changedFiles = emptyList(), removedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + } + + @Test + fun `a removed java under a kotlin source root maps its package the same way`() { + // Mixed layouts put .java files under src/main/kotlin too; the root marker + // accepts either directory name. + val widget = widgetJava("main/kotlin/demo/Widget.java") + val compiler = compiler() + val first = compiler.compile(listOf(widget), changedFiles = listOf(widget)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + + assertThat(widget.delete()).isTrue() + val result = compiler.compile(emptyList(), changedFiles = emptyList(), removedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + } + + @Test + fun `a rootless relative removed path still maps its package via the leading marker`() { + val widget = widgetJava() + val compiler = compiler() + val first = compiler.compile(listOf(widget), changedFiles = listOf(widget)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + + assertThat(widget.delete()).isTrue() + val result = + compiler.compile( + emptyList(), + changedFiles = emptyList(), + removedFiles = listOf(File("java/demo/Widget.java")), + ) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + } + + @Test + fun `a vanished classes dir mid-session is rebuilt, not tripped over`() { + // External cleanup (or a first-ever build) can leave the output tree absent when a + // compile starts: the pre-snapshot and the removed-java sweep must both treat + // "no tree" as "no outputs" and the compile must recreate it. + val kotlin = kotlinSource() + val compiler = compiler() + val ghostRemoved = File(srcDir, "main/java/demo/Old.java") + File(workDir, "classes").deleteRecursively() + + val result = + compiler.compile(listOf(kotlin), changedFiles = listOf(kotlin), removedFiles = listOf(ghostRemoved)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val success = result as IncrementalCompiler.Result.Success + assertThat(File(success.classesDir, "demo/Greeter.class").isFile).isTrue() + assertThat(success.changedClassFiles).contains("demo/Greeter.class") + } + + @Test + fun `output a failed compile left behind is still reported by the next successful one`() { + // Save 0: both sides good. This is the state the caller actually deployed. + val widget = widgetJava() + val greeter = kotlinSource() + val compiler = compiler() + val sources = listOf(greeter, widget) + check(compiler.compile(sources, changedFiles = sources) is IncrementalCompiler.Result.Success) + val greeterClass = File(File(workDir, "classes"), "demo/Greeter.class") + val deployedLength = greeterClass.length() + + // Save A edits both sides. Kotlin succeeds and rewrites Greeter.class; the Java edit is a + // body-only error, so javac fails and NOTHING from this compile is deployed. + kotlinSource("a considerably longer greeting") + writeJava( + "main/java/demo/Widget.java", + "package demo;\n\npublic class Widget { public int v() { return \"nope\"; } }", + ) + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + // The premise of the whole sequence: the failed compile left new bytecode on disk. + assertThat(greeterClass.length()).isNotEqualTo(deployedLength) + + // Save B fixes only the Java body, leaving the Java ABI equal to the last SUCCESSFUL + // compile's - so no Kotlin recompiles and Greeter.class is not touched again. + widgetJava() + val result = compiler.compile(sources, changedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val success = result as IncrementalCompiler.Result.Success + assertThat(success.stats.kotlinToCompile).isEqualTo(0) + // This is the first compile whose output the caller can deploy, so it owns save A's + // class too. Re-snapshotting the tree at the top of every compile adopts those + // undeployed classes as already-live and drops them here, and the deploy policy then + // answers recreate where a changed component needs a restart. + assertThat(success.changedClassFiles).contains("demo/Greeter.class") + } + + @Test + fun `a deleted class output is reported as changed, not silently dropped`() { + val widget = widgetJava() + val greeter = kotlinSource() + val compiler = compiler() + val first = compiler.compile(listOf(greeter, widget), changedFiles = listOf(greeter, widget)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + check(File(classesDir, "demo/Widget.class").isFile) { "fixture compile produced no Widget.class" } + + assertThat(widget.delete()).isTrue() + val result = compiler.compile(listOf(greeter), changedFiles = emptyList(), removedFiles = listOf(widget)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + // A deletion exists only in the before-snapshot, so filtering the post-snapshot alone can + // never surface it - and dropping a restart-sensitive component's nested class is exactly + // the change the deploy policy has to see. + assertThat((result as IncrementalCompiler.Result.Success).changedClassFiles) + .contains("demo/Widget.class") + } + + @Test + fun `a removed path that climbs out of the output tree deletes nothing`() { + val widget = widgetJava() + val compiler = compiler() + check(compiler.compile(listOf(widget), changedFiles = listOf(widget)) is IncrementalCompiler.Result.Success) + // The output tree is /classes, so two levels up from it is tempDir. + val victim = File(tempDir, "outside/Bar.class").apply { parentFile!!.mkdirs() } + victim.writeText("keep") + val escaping = File(srcDir, "main/java/../../outside/Bar.java") + + val result = compiler.compile(listOf(widget), changedFiles = emptyList(), removedFiles = listOf(escaping)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + // The stem is a raw join of the segments after the source root, so without a containment + // check this sweep lists and deletes outside the output tree it owns. + assertThat(victim.isFile).isTrue() + assertThat(File(File(workDir, "classes"), "demo/Widget.class").isFile).isTrue() + } + + @Test + fun `a removed java under a package named java maps against the main source root`() { + // `main/java` wins over the deeper `java` package segment; resolving to the last marker + // instead would map this to a bare `Bar` at the output root and leave the real output + // behind as stale bytecode. + val bar = writeJava("main/java/com/foo/java/Bar.java", "package com.foo.java;\n\npublic class Bar {}\n") + val compiler = compiler() + val first = compiler.compile(listOf(bar), changedFiles = listOf(bar)) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + check(File(classesDir, "com/foo/java/Bar.class").isFile) { "fixture compile produced no Bar.class" } + + assertThat(bar.delete()).isTrue() + val result = compiler.compile(emptyList(), changedFiles = emptyList(), removedFiles = listOf(bar)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "com/foo/java/Bar.class").exists()).isFalse() + } + + @Test + fun `two classpath jars with the same basename get a snapshot each`() { + // Every AAR-derived classpath entry is literally `classes.jar`. Named after the basename, + // each snapshot overwrote the last, so the list handed to the IC engine held one path N + // times and described only the final jar. + val stdlib = TestSdk.kotlinStdlib() + val fromFirstAar = File(tempDir, "aar-a/classes.jar").apply { parentFile!!.mkdirs() } + val fromSecondAar = File(tempDir, "aar-b/classes.jar").apply { parentFile!!.mkdirs() } + stdlib.copyTo(fromFirstAar, overwrite = true) + stdlib.copyTo(fromSecondAar, overwrite = true) + + IncrementalCompiler(listOf(fromFirstAar, fromSecondAar), workDir.toPath()).use { compiler -> + assertThat(File(workDir, "cp-snap").listFiles()!!.map { it.name }.toSet()).hasSize(2) + + val kotlin = kotlinSource() + assertThat(compiler.compile(listOf(kotlin), changedFiles = listOf(kotlin))) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + } + } + + @Test + fun `closing hands the compilation service's project state back`() { + // The BTA contract wants a project finished once it is done with, and on the in-process + // strategy the retained state otherwise lives for the JVM's lifetime - one project's + // worth per re-configure, on a 2-4 GB phone. There is nothing observable left behind to + // assert on; what this pins is that close() exists, is reached through AutoCloseable, and + // carries a projectId the service accepts. + val kotlin = kotlinSource() + + IncrementalCompiler(listOf(TestSdk.kotlinStdlib()), workDir.toPath()).use { compiler -> + assertThat(compiler.compile(listOf(kotlin), changedFiles = listOf(kotlin))) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + } + } + + @Test + fun `a removed java whose package never produced output is a no-op`() { + val kotlin = kotlinSource() + val compiler = compiler() + val first = compiler.compile(listOf(kotlin), changedFiles = listOf(kotlin)) + check(first is IncrementalCompiler.Result.Success) { "fixture compile failed" } + + val ghost = File(srcDir, "main/java/ghost/Gone.java") + val result = compiler.compile(listOf(kotlin), changedFiles = emptyList(), removedFiles = listOf(ghost)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerTest.kt new file mode 100644 index 0000000000..20bcdf4e8f --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/IncrementalCompilerTest.kt @@ -0,0 +1,780 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.daemon.TestSdk +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * End-to-end on the host JVM: real BTA CompilationService, real kotlinc, real IC caches. + * The incremental assertions pin the README gotchas - if the engine silently falls back + * to a full compile (the failure mode the shrunk-snapshot path and SourcesChanges.Known + * exist to prevent), these tests go red. + */ +class IncrementalCompilerTest { + @TempDir + lateinit var tempDir: File + + private lateinit var srcDir: File + private lateinit var workDir: File + + @BeforeEach + fun setUp() { + srcDir = File(tempDir, "src").apply { mkdirs() } + workDir = File(tempDir, "work").apply { mkdirs() } + } + + /** Every line the compiler emitted; cleared between compiles to read one compile's log. */ + private val compileLog = mutableListOf() + + private fun compiler() = IncrementalCompiler(listOf(TestSdk.kotlinStdlib()), workDir.toPath(), compileLog = { compileLog += it }) + + private fun writeSource( + name: String, + content: String, + ): File = File(srcDir, name).apply { writeText(content) } + + private fun greeterKt(greeting: String = "Hello") = + writeSource( + "Greeter.kt", + """ + package demo + + class Greeter(private val name: String) { + fun greet(): String = "$greeting, ${'$'}name!" + } + """.trimIndent(), + ) + + private fun mainKt() = + writeSource( + "Main.kt", + """ + package demo + + fun main() { + println(Greeter("world").greet()) + } + """.trimIndent(), + ) + + @Test + fun `first build compiles all sources and seeds the IC caches`() { + val sources = listOf(greeterKt(), mainKt()) + val compiler = compiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Greeter.class").isFile).isTrue() + assertThat(File(classesDir, "demo/MainKt.class").isFile).isTrue() + // The seed build must leave the shrunk snapshot at EXACTLY this path - a + // mismatch means every later build silently degrades to non-incremental. + assertThat(File(workDir, "shrunk-classpath-snapshot.bin").isFile).isTrue() + } + + @Test + fun `editing one file recompiles incrementally, not a full rebuild`() { + val greeter = greeterKt() + val sources = listOf(greeter, mainKt()) + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + greeterKt(greeting = "Howdy") + // The seed compile above legitimately recompiles everything; only the edit's own log + // says whether THIS compile was incremental. + compileLog.clear() + val result = compiler.compile(sources, changedFiles = listOf(greeter)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val log = compileLog.joinToString("\n") + // The IC engine reports each compile iteration with the files it actually + // recompiled: the changed file must be there, and no fallback marker may appear. + assertThat(log).contains("Greeter.kt") + assertThat(log).contains("compile iteration") + assertThat(log).doesNotContainMatch("(?i)non-incremental") + assertThat(log).doesNotContain("CLASSPATH_SNAPSHOT_NOT_FOUND") + assertThat(log).doesNotContain("UNKNOWN_CHANGES_IN_GRADLE_INPUTS") + val iterationLines = compileLog.filter { it.contains("compile iteration") } + assertThat(iterationLines).isNotEmpty() + for (line in iterationLines) { + assertThat(line).doesNotContain("Main.kt") + } + } + + @Test + fun `changed class files list the seed build's outputs, then only the recompiled ones`() { + val greeter = greeterKt() + val sources = listOf(greeter, mainKt()) + val compiler = compiler() + + val first = compiler.compile(sources, changedFiles = sources) + assertThat(first).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat((first as IncrementalCompiler.Result.Success).changedClassFiles) + .containsAtLeast("demo/Greeter.class", "demo/MainKt.class") + + greeterKt(greeting = "Howdy") + val second = compiler.compile(sources, changedFiles = listOf(greeter)) + + assertThat(second).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val changed = (second as IncrementalCompiler.Result.Success).changedClassFiles + // The recompiled file is reported; the untouched one is not - an over- or + // under-report here would skew the CoGo-side restart decision. + assertThat(changed).contains("demo/Greeter.class") + assertThat(changed).doesNotContain("demo/MainKt.class") + } + + @Test + fun `a removed kotlin source has its output deleted`() { + val orphan = writeSource("Orphan.kt", "package demo\n\nclass Orphan") + val sources = listOf(greeterKt(), mainKt(), orphan) + val compiler = compiler() + val first = compiler.compile(sources, changedFiles = sources) + assertThat(first).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Orphan.class").isFile).isTrue() + + // Orphan.kt is deleted: gone from allSources AND passed as a removal. Threaded into + // SourcesChanges.Known's removed slot, the engine must delete its stale output so a + // deleted class can't survive into the dex. + assertThat(orphan.delete()).isTrue() + val result = + compiler.compile( + listOf(greeterKt(), mainKt()), + changedFiles = emptyList(), + removedFiles = listOf(orphan), + ) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Orphan.class").exists()).isFalse() + } + + @Test + fun `a removed java source has its class deleted before it can reach the dex`() { + // javac never deletes outputs for sources it's no longer handed, so the daemon must + // delete a removed .java's .class explicitly. The path mirrors its package under a + // main/java root, exactly as the enforced project layout does. + val widget = + File(srcDir, "main/java/demo/Widget.java").apply { + parentFile!!.mkdirs() + writeText("package demo;\n\npublic class Widget { public int v() { return 1; } }") + } + val sources = listOf(greeterKt(), mainKt(), widget) + val compiler = compiler() + val first = compiler.compile(sources, changedFiles = sources) + assertThat(first).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/Widget.class").isFile).isTrue() + + assertThat(widget.delete()).isTrue() + val result = + compiler.compile( + listOf(greeterKt(), mainKt()), + changedFiles = emptyList(), + removedFiles = listOf(widget), + ) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(File(classesDir, "demo/Widget.class").exists()).isFalse() + } + + @Test + fun `a stale java class that cannot be deleted fails the compile instead of riding into the dex`() { + // POSIX: deleting a file needs write permission on its DIRECTORY - a read-only + // package dir makes File.delete() return false with the file still present, + // exactly the "stubborn stale output" this guard exists for. + val widget = + File(srcDir, "main/java/demo/Widget.java").apply { + parentFile!!.mkdirs() + writeText("package demo;\n\npublic class Widget { public int v() { return 1; } }") + } + val sources = listOf(greeterKt(), mainKt(), widget) + val compiler = compiler() + val first = compiler.compile(sources, changedFiles = sources) + assertThat(first).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (first as IncrementalCompiler.Result.Success).classesDir + val staleClass = File(classesDir, "demo/Widget.class") + assertThat(staleClass.isFile).isTrue() + + assertThat(widget.delete()).isTrue() + val pkgDir = staleClass.parentFile!! + assertThat(pkgDir.setWritable(false)).isTrue() + try { + val result = + compiler.compile( + listOf(greeterKt(), mainKt()), + changedFiles = emptyList(), + removedFiles = listOf(widget), + ) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + val diagnostics = (result as IncrementalCompiler.Result.Failed).diagnostics + assertThat(diagnostics).isNotEmpty() + assertThat(diagnostics.any { it.severity == Diagnostic.Severity.ERROR }).isTrue() + // The diagnostic must NAME the stubborn path so the failure is actionable. + assertThat(diagnostics.any { it.message.contains(staleClass.absolutePath) }).isTrue() + assertThat(staleClass.exists()).isTrue() + } finally { + pkgDir.setWritable(true) + } + } + + @Test + fun `syntax error yields structured diagnostics with file and line`() { + val sources = listOf(greeterKt(), mainKt()) + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + val broken = + writeSource( + "Greeter.kt", + """ + package demo + + class Greeter(private val name: String) { + fun greet(): String = "Hello, ${'$'}name!" + + } + """.trimIndent(), + ) + val result = compiler.compile(sources, changedFiles = listOf(broken)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + val diagnostics = (result as IncrementalCompiler.Result.Failed).diagnostics + assertThat(diagnostics).isNotEmpty() + val located = diagnostics.firstOrNull { it.file?.endsWith("Greeter.kt") == true } + assertThat(located).isNotNull() + assertThat(located!!.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(located.line).isAtLeast(1) + } + + @Test + fun `recovering from a syntax error compiles cleanly again`() { + val greeter = greeterKt() + val sources = listOf(greeter, mainKt()) + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + writeSource("Greeter.kt", "package demo\n\nclass Greeter(private val name: String) {\n") + assertThat(compiler.compile(sources, changedFiles = listOf(greeter))) + .isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + + greeterKt(greeting = "Fixed") + val result = compiler.compile(sources, changedFiles = listOf(greeter)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + } + + @Test + fun `java sources compile against kotlin output into the same classes dir`() { + val javaSource = + writeSource( + "JavaUser.java", + """ + package demo; + + public class JavaUser { + public String use() { + return new Greeter("java").greet(); + } + } + """.trimIndent(), + ) + val sources = listOf(greeterKt(), mainKt(), javaSource) + val compiler = compiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/JavaUser.class").isFile).isTrue() + assertThat(File(classesDir, "demo/Greeter.class").isFile).isTrue() + } + + private fun composeCompiler() = + IncrementalCompiler( + listOf(TestSdk.kotlinStdlib(), TestSdk.composeRuntimeJar()!!), + workDir.toPath(), + compilerPluginJars = listOf(TestSdk.composePluginJar()!!), + compileLog = { compileLog += it }, + ) + + private fun composablesKt(marker: String = "MARKER_V1") = + writeSource( + "Composables.kt", + """ + package demo + + import androidx.compose.runtime.Composable + import androidx.compose.runtime.getValue + import androidx.compose.runtime.mutableStateOf + import androidx.compose.runtime.remember + import androidx.compose.runtime.setValue + + @Composable + fun Greeting(name: String) { + var count by remember { mutableStateOf(0) } + Label("$marker hello, ${'$'}name (${'$'}count)") + count += 1 + } + + @Composable + fun Label(text: String) { + Recorder.record(text) + } + """.trimIndent(), + ) + + private fun recorderKt() = + writeSource( + "Recorder.kt", + """ + package demo + + object Recorder { + val seen = mutableListOf() + + fun record(text: String) { + seen += text + } + } + """.trimIndent(), + ) + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#composeToolchainAvailable") + fun `compose plugin transforms composable functions`() { + val sources = listOf(composablesKt(), recorderKt()) + val compiler = composeCompiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + val composables = File(classesDir, "demo/ComposablesKt.class") + assertThat(composables.isFile).isTrue() + // The Compose transform rewrites @Composable functions to take a Composer + // parameter; its type name in the constant pool is the proof the plugin ran + // (without the plugin the same source compiles to a plain static method). + assertThat(String(composables.readBytes(), Charsets.ISO_8859_1)) + .contains("androidx/compose/runtime/Composer") + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#composeToolchainAvailable") + fun `composable edit recompiles incrementally with the plugin active`() { + val composables = composablesKt() + val sources = listOf(composables, recorderKt()) + val compiler = composeCompiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + composablesKt(marker = "MARKER_V2") + // Read the edit's own log, not the seed's. + compileLog.clear() + val result = compiler.compile(sources, changedFiles = listOf(composables)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + assertThat(String(File(classesDir, "demo/ComposablesKt.class").readBytes(), Charsets.ISO_8859_1)) + .contains("MARKER_V2") + val log = compileLog.joinToString("\n") + assertThat(log).doesNotContainMatch("(?i)non-incremental") + assertThat(log).doesNotContain("CLASSPATH_SNAPSHOT_NOT_FOUND") + assertThat(log).doesNotContain("UNKNOWN_CHANGES_IN_GRADLE_INPUTS") + val iterationLines = compileLog.filter { it.contains("compile iteration") } + assertThat(iterationLines).isNotEmpty() + for (line in iterationLines) { + assertThat(line).doesNotContain("Recorder.kt") + } + } + + @Test + fun `kotlin source resolves a same-module java class it calls`() { + // Without javaSources in compileJvm's source list, kotlinc has zero visibility into + // a sibling .java file that isn't precompiled onto the classpath yet, and the + // baseline compile fails outright with "Unresolved reference". + val javaSource = + writeSource( + "JavaCalculator.java", + """ + package demo; + + public class JavaCalculator { + public int computeTotal(int a, int b) { return a + b; } + } + """.trimIndent(), + ) + val callerSource = + writeSource( + "OrderService.kt", + """ + package demo + + class OrderService { + fun total(a: Int, b: Int) = JavaCalculator().computeTotal(a, b) + } + """.trimIndent(), + ) + val sources = listOf(javaSource, callerSource) + val compiler = compiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/JavaCalculator.class").isFile).isTrue() + assertThat(File(classesDir, "demo/OrderService.class").isFile).isTrue() + } + + @Test + fun `a java-only signature change recompiles its unedited kotlin caller`() { + // The regression this guards: SourcesChanges.Known filtered out .java entries, so a + // changedFiles list containing ONLY a .java path told the incremental engine "nothing + // kotlin changed" and it skipped OrderService.kt entirely - leaving its .class calling + // the OLD Java descriptor even after JavaCalculator's signature changed underneath it. + val javaSource = + writeSource( + "JavaCalculator.java", + """ + package demo; + + public class JavaCalculator { + public int computeTotal(int a, int b) { return a + b; } + } + """.trimIndent(), + ) + val callerSource = + writeSource( + "OrderService.kt", + """ + package demo + + class OrderService { + fun total(a: Int, b: Int) = JavaCalculator().computeTotal(a, b) + } + """.trimIndent(), + ) + val sources = listOf(javaSource, callerSource) + val compiler = compiler() + val baseline = compiler.compile(sources, changedFiles = sources) + assertThat(baseline).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (baseline as IncrementalCompiler.Result.Success).classesDir + val before = File(classesDir, "demo/OrderService.class").readBytes() + + // Widen the return type: OrderService's call-site descriptor must change to match, even + // though OrderService.kt itself is untouched on disk and NOT in changedFiles. + writeSource( + "JavaCalculator.java", + """ + package demo; + + public class JavaCalculator { + public long computeTotal(int a, int b) { return (long) a + b; } + } + """.trimIndent(), + ) + val result = compiler.compile(sources, changedFiles = listOf(javaSource)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val after = File(classesDir, "demo/OrderService.class").readBytes() + assertThat(after).isNotEqualTo(before) + } + + /** + * A genuine Kotlin<->Java cycle: mutual calls, plus a Java class whose supertype is a + * Kotlin source in the same compile. Neither language can be compiled first in + * isolation, so this is the shape the corpus's `mixed-lang-cyclic` app pins end to end. + */ + private fun cyclicSources(rendererBody: String = """return "Node(" + node.getLabel() + ")";"""): List { + val node = + writeSource( + "TreeNode.kt", + """ + package demo + + open class TreeNode(val label: String) { + open fun describe() = NodeRenderer.render(this) + + companion object { + fun leaf(label: String): TreeNode = JavaLeafNode(label) + } + } + """.trimIndent(), + ) + val renderer = + writeSource( + "NodeRenderer.java", + """ + package demo; + + public final class NodeRenderer { + public static String render(TreeNode node) { $rendererBody } + } + """.trimIndent(), + ) + val leaf = + writeSource( + "JavaLeafNode.java", + """ + package demo; + + public class JavaLeafNode extends TreeNode { + public JavaLeafNode(String label) { super(label); } + + @Override + public String describe() { return "Leaf[" + getLabel() + "]"; } + } + """.trimIndent(), + ) + return listOf(node, renderer, leaf) + } + + @Test + fun `mutually referencing kotlin and java sources compile in one pass`() { + val sources = cyclicSources() + val compiler = compiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (result as IncrementalCompiler.Result.Success).classesDir + assertThat(File(classesDir, "demo/TreeNode.class").isFile).isTrue() + assertThat(File(classesDir, "demo/NodeRenderer.class").isFile).isTrue() + // The Java subclass is the sharp end: javac could only resolve its supertype + // because kotlinc had already emitted TreeNode into the same output dir. + assertThat(File(classesDir, "demo/JavaLeafNode.class").isFile).isTrue() + } + + @Test + fun `a java body-only edit leaves kotlin untouched`() { + val sources = cyclicSources() + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + cyclicSources(rendererBody = """return "Node[" + node.getLabel() + "]";""") + val result = compiler.compile(sources, changedFiles = listOf(sources[1])) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + // No Java signature moved, so no Kotlin class can differ - and none may be rewritten. + assertThat(compiler.lastJavaAbiChange).isEmpty() + val changed = (result as IncrementalCompiler.Result.Success).changedClassFiles + assertThat(changed).contains("demo/NodeRenderer.class") + assertThat(changed).doesNotContain("demo/TreeNode.class") + } + + @Test + fun `a kotlin-only edit does not report the untouched java half as changed`() { + // javac is not incremental here: it recompiles and rewrites every .java on every build, + // byte-identical or not. An mtime-keyed output snapshot therefore reported every + // Java-derived class as changed on a Kotlin-only edit, and DeployPolicy restarts the + // process whenever the changed set reaches a Service, Provider or Application closure - + // so a project with one Java component paid a full restart, and lost app state, on every + // save. The snapshot is keyed on content for exactly this. + val sources = cyclicSources() + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + writeSource( + "TreeNode.kt", + """ + package demo + + open class TreeNode(val label: String) { + open fun describe() = NodeRenderer.render(this) + + fun depth(): Int = 1 + + companion object { + fun leaf(label: String): TreeNode = JavaLeafNode(label) + } + } + """.trimIndent(), + ) + val result = compiler.compile(sources, changedFiles = listOf(sources[0])) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val changed = (result as IncrementalCompiler.Result.Success).changedClassFiles + assertThat(changed).contains("demo/TreeNode.class") + // Neither Java class's source was touched, so neither may appear - even though javac + // rewrote both output files. + assertThat(changed).doesNotContain("demo/NodeRenderer.class") + assertThat(changed).doesNotContain("demo/JavaLeafNode.class") + } + + private fun limitsSources(max: String): List { + val limits = + writeSource( + "JavaLimits.java", + """ + package demo; + + public class JavaLimits { + public static final int MAX = $max; + } + """.trimIndent(), + ) + val caller = + writeSource( + "LimitUser.kt", + """ + package demo + + class LimitUser { + fun ceiling(): Int = JavaLimits.MAX + } + """.trimIndent(), + ) + return listOf(limits, caller) + } + + @Test + fun `a java constant's new value reaches its kotlin caller's bytecode`() { + // Kotlin inlines Java compile-time constants, so nothing about this edit shows up in + // a signature - if the ABI fingerprint ignored constant VALUES, the Java-ABI shortcut + // would skip LimitUser and leave it returning 5 forever. + val sources = limitsSources("5") + val compiler = compiler() + val baseline = compiler.compile(sources, changedFiles = sources) + assertThat(baseline).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (baseline as IncrementalCompiler.Result.Success).classesDir + val before = File(classesDir, "demo/LimitUser.class").readBytes() + + limitsSources("7") + val result = compiler.compile(sources, changedFiles = listOf(sources[0])) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(compiler.lastJavaAbiChange).contains("JavaLimits") + assertThat(File(classesDir, "demo/LimitUser.class").readBytes()).isNotEqualTo(before) + } + + @Test + fun `a failed compile does not become the java ABI baseline`() { + // Otherwise the next compile compares against an ABI whose bytecode was never + // emitted, and silently skips the Kotlin recompile the Java change still needs. + val sources = limitsSources("5") + val compiler = compiler() + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Success::class.java) + + limitsSources("7") + writeSource("LimitUser.kt", "package demo\n\nclass LimitUser { fun ceiling(): Int = ") + assertThat(compiler.compile(sources, changedFiles = sources)) + .isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + + // Repair only the Kotlin file; the Java constant is still 7, still unaccounted for. + writeSource( + "LimitUser.kt", + """ + package demo + + class LimitUser { + fun ceiling(): Int = JavaLimits.MAX + } + """.trimIndent(), + ) + val result = compiler.compile(sources, changedFiles = listOf(sources[1])) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + assertThat(compiler.lastJavaAbiChange).contains("JavaLimits") + } + + private fun labelsKt(suffix: String = "MY_LABEL_V1") = + writeSource( + "Labels.kt", + """ + package demo + + object Labels { + inline fun label(prefix: String): String = prefix + "$suffix" + } + """.trimIndent(), + ) + + private fun labelUserKt() = + writeSource( + "LabelUser.kt", + """ + package demo + + class LabelUser { + fun render(): String = Labels.label("prefix: ") + } + """.trimIndent(), + ) + + @Test + fun `an inline function's body edit recompiles its unedited caller`() { + // An inline function's BODY is part of its ABI - it is copied into every call site - + // so an edit that moves no signature must still recompile untouched callers. That's + // a different invalidation rule from the signature-change cases above, and Kotlin's + // IC has historically got it wrong: the caller then keeps running the old inlined + // body while its source says otherwise. + val labels = labelsKt() + val sources = listOf(labels, labelUserKt()) + val compiler = compiler() + val baseline = compiler.compile(sources, changedFiles = sources) + assertThat(baseline).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val classesDir = (baseline as IncrementalCompiler.Result.Success).classesDir + val before = File(classesDir, "demo/LabelUser.class").readBytes() + // Premise check: the literal only lands in the CALLER's constant pool if the body + // really was inlined. Without this the edit assertion below could pass vacuously. + assertThat(String(before, Charsets.ISO_8859_1)).contains("MY_LABEL_V1") + + labelsKt(suffix = "MY_LABEL_V2") + // The seed compile legitimately compiles everything; only the edit's own log says + // whether THIS compile recompiled the caller by invalidation or by falling back. + compileLog.clear() + val result = compiler.compile(sources, changedFiles = listOf(labels)) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Success::class.java) + val after = File(classesDir, "demo/LabelUser.class").readBytes() + assertThat(after).isNotEqualTo(before) + // The untouched caller's own bytecode must now carry the new body, and not the old. + assertThat(String(after, Charsets.ISO_8859_1)).contains("MY_LABEL_V2") + assertThat(String(after, Charsets.ISO_8859_1)).doesNotContain("MY_LABEL_V1") + // The caller is reported as changed, which is what feeds CoGo's restart decision. + assertThat((result as IncrementalCompiler.Result.Success).changedClassFiles) + .contains("demo/LabelUser.class") + // A full-rebuild fallback would satisfy everything above for the wrong reason, so + // require that the caller was reached by invalidation. + val log = compileLog.joinToString("\n") + assertThat(log).doesNotContainMatch("(?i)non-incremental") + assertThat(log).doesNotContain("CLASSPATH_SNAPSHOT_NOT_FOUND") + assertThat(log).doesNotContain("UNKNOWN_CHANGES_IN_GRADLE_INPUTS") + } + + @Test + fun `java error yields structured diagnostics and fails the compile`() { + val javaSource = + writeSource( + "Broken.java", + """ + package demo; + + public class Broken { + public int broken() { return "not an int"; } + } + """.trimIndent(), + ) + val sources = listOf(greeterKt(), javaSource) + val compiler = compiler() + + val result = compiler.compile(sources, changedFiles = sources) + + assertThat(result).isInstanceOf(IncrementalCompiler.Result.Failed::class.java) + val diagnostics = (result as IncrementalCompiler.Result.Failed).diagnostics + val located = diagnostics.firstOrNull { it.file?.endsWith("Broken.java") == true } + assertThat(located).isNotNull() + assertThat(located!!.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(located.line).isEqualTo(4) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStepTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStepTest.kt new file mode 100644 index 0000000000..622ef6877d --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaCompileStepTest.kt @@ -0,0 +1,59 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * javac's structured diagnostics mapped onto the protocol shape: errors block with a + * location, advisory notes pass through as warnings without one - the severity split is + * what lets the client fail a build on ERROR while still showing the rest. + */ +class JavaCompileStepTest { + @TempDir + lateinit var tempDir: File + + private fun outputDir(): File = File(tempDir, "classes").apply { mkdirs() } + + @Test + fun `a compile error fails with an ERROR diagnostic locating the problem`() { + val broken = + File(tempDir, "Broken.java").apply { + writeText("package demo;\n\npublic class Broken {\n\tint x = ;\n}\n") + } + + val result = JavaCompileStep.compile(listOf(broken), emptyList(), outputDir()) + + assertThat(result.success).isFalse() + val error = result.diagnostics.first { it.severity == Diagnostic.Severity.ERROR } + assertThat(error.file).contains("Broken.java") + assertThat(error.line).isEqualTo(4) + assertThat(error.column).isNotNull() + } + + @Test + fun `an advisory javac note compiles successfully as a WARNING without a fabricated location`() { + // Raw-type use draws javac's file-level "uses unchecked or unsafe operations" + // note: no position exists, so line/column must read back null - inventing one + // would send the IDE's jump-to-diagnostic somewhere wrong. + val rawUser = + File(tempDir, "RawUser.java").apply { + writeText( + "package demo;\n\n" + + "public class RawUser {\n" + + "\tpublic void fill(java.util.List list) { list.add(\"x\"); }\n" + + "}\n", + ) + } + + val result = JavaCompileStep.compile(listOf(rawUser), emptyList(), outputDir()) + + assertThat(result.success).isTrue() + assertThat(File(outputDir(), "demo/RawUser.class").isFile).isTrue() + assertThat(result.diagnostics).isNotEmpty() + assertThat(result.diagnostics.map { it.severity }).doesNotContain(Diagnostic.Severity.ERROR) + assertThat(result.diagnostics.any { it.line == null && it.column == null }).isTrue() + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiEdgeTest.kt new file mode 100644 index 0000000000..9cf050a4b6 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiEdgeTest.kt @@ -0,0 +1,159 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * Declaration kinds beyond JavaSourceAbiTest's classes-and-methods core: whether each + * kind's edit is IN the fingerprint decides between a stale-bytecode bug (ignored when it + * shouldn't be) and a needless full Kotlin recompile (included when it needn't be). + */ +class JavaSourceAbiEdgeTest { + @TempDir + lateinit var tempDir: File + + private fun write( + name: String, + content: String, + ): File = File(tempDir, name).apply { writeText(content.trimIndent()) } + + private fun fingerprintOf(file: File): String { + val snapshot = JavaSourceAbi.snapshot(listOf(file)) + assertThat(snapshot).isNotNull() + return snapshot!!.getValue(file).fingerprint + } + + @Test + fun `a source that becomes unreadable still flags its old types as changed`() { + // javac error-recovers instead of throwing: an unreadable file parses to an + // EMPTY declaration set, so its fingerprint moves and changedTypeNames names the + // types it used to declare - which is exactly what forces the conservative full + // Kotlin recompile. (The snapshot's null path is reserved for real exceptions.) + val locked = write("Locked.java", "package demo;\n\npublic class Locked {}") + val previous = JavaSourceAbi.snapshot(listOf(locked))!! + check(locked.setReadable(false)) { "could not revoke read permission" } + try { + val current = JavaSourceAbi.snapshot(listOf(locked))!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, current)).containsExactly("Locked") + } finally { + locked.setReadable(true) + } + } + + @Test + fun `the package declaration is part of the ABI`() { + val without = write("A.java", "public class Widget {}") + val with = write("B.java", "package demo;\n\npublic class Widget {}") + + assertThat(fingerprintOf(without)).isNotEqualTo(fingerprintOf(with)) + assertThat(JavaSourceAbi.snapshot(listOf(without))!!.getValue(without).declaredTypeNames) + .containsExactly("Widget") + } + + @Test + fun `an interface constant's value is ABI even without static final modifiers`() { + // Interface fields are implicitly constant; Kotlin inlines them like any other + // compile-time constant. + val before = fingerprintOf(write("Limits.java", "package demo;\n\npublic interface Limits { int MAX = 5; }")) + val after = fingerprintOf(write("Limits.java", "package demo;\n\npublic interface Limits { int MAX = 7; }")) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `an annotation member's default value is ABI`() { + val before = + fingerprintOf( + write("Marker.java", "package demo;\n\npublic @interface Marker { String value() default \"x\"; }"), + ) + val after = + fingerprintOf( + write("Marker.java", "package demo;\n\npublic @interface Marker { String value() default \"y\"; }"), + ) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `a constructor's parameter list is ABI`() { + val before = fingerprintOf(write("Box.java", "package demo;\n\npublic class Box {\n\tpublic Box() {}\n}")) + val after = fingerprintOf(write("Box.java", "package demo;\n\npublic class Box {\n\tpublic Box(int size) {}\n}")) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `a static initializer block is not ABI`() { + val without = fingerprintOf(write("Init.java", "package demo;\n\npublic class Init {\n\tstatic int x;\n}")) + val with = + fingerprintOf( + write("Init.java", "package demo;\n\npublic class Init {\n\tstatic int x;\n\tstatic { x = 3; }\n}"), + ) + + assertThat(with).isEqualTo(without) + } + + @Test + fun `an extends clause is ABI`() { + val plain = fingerprintOf(write("Leaf.java", "package demo;\n\npublic class Leaf {}")) + val extending = + fingerprintOf( + write("Leaf.java", "package demo;\n\npublic class Leaf extends java.util.ArrayList {}"), + ) + + assertThat(extending).isNotEqualTo(plain) + } + + @Test + fun `a non-final static field's initializer is not ABI`() { + // Only static AND final makes a Java compile-time constant Kotlin can inline; a + // mutable static's initializer is implementation, and charging a full Kotlin + // recompile for editing it would make the ABI shortcut pointless. + val before = + fingerprintOf(write("Counter.java", "package demo;\n\npublic class Counter { static int next = 1; }")) + val after = + fingerprintOf(write("Counter.java", "package demo;\n\npublic class Counter { static int next = 2; }")) + + assertThat(after).isEqualTo(before) + } + + @Test + fun `an explicitly static final interface constant is still a constant`() { + // Redundant modifiers spelled out must not change the classification. + val before = + fingerprintOf(write("Caps.java", "package demo;\n\npublic interface Caps { static final int M = 1; }")) + val after = + fingerprintOf(write("Caps.java", "package demo;\n\npublic interface Caps { static final int M = 2; }")) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `a stray top-level semicolon is not ABI`() { + val without = fingerprintOf(write("Tidy.java", "package demo;\n\npublic class Tidy {}")) + val with = fingerprintOf(write("Tidy.java", "package demo;\n\npublic class Tidy {};")) + + assertThat(with).isEqualTo(without) + } + + @Test + fun `a duplicated source entry yields null - the per-file map cannot attribute it`() { + // Conservative contract: when the snapshot cannot represent the input faithfully + // it must say "unknown" (forcing a full Kotlin recompile), never half an answer. + val file = write("Dup.java", "package demo;\n\npublic class Dup {}") + + assertThat(JavaSourceAbi.snapshot(listOf(file, file))).isNull() + } + + @Test + fun `an enum's constant set is ABI`() { + // Kotlin `when` exhaustiveness and constant references both see enum constants. + val before = fingerprintOf(write("Color.java", "package demo;\n\npublic enum Color { RED }")) + val after = fingerprintOf(write("Color.java", "package demo;\n\npublic enum Color { RED, BLUE }")) + + assertThat(after).isNotEqualTo(before) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiTest.kt new file mode 100644 index 0000000000..8a3e81e96d --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/JavaSourceAbiTest.kt @@ -0,0 +1,361 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * The fingerprint decides whether a `.java` edit costs a full Kotlin recompile, so what it + * ignores matters as much as what it captures: ignore too much and Kotlin bytecode goes + * stale, ignore too little and every Java keystroke pays for a recompile it does not need. + */ +class JavaSourceAbiTest { + @TempDir + lateinit var tempDir: File + + private fun write( + name: String, + content: String, + ): File = File(tempDir, name).apply { writeText(content.trimIndent()) } + + private fun fingerprintOf(file: File): String { + val snapshot = JavaSourceAbi.snapshot(listOf(file)) + assertThat(snapshot).isNotNull() + return snapshot!!.getValue(file).fingerprint + } + + private fun calculator(body: String) = + write( + "Calculator.java", + """ + package demo; + + public class Calculator { + public int compute(int a, int b) { $body } + } + """, + ) + + @Test + fun `a method body edit leaves the fingerprint unchanged`() { + val before = fingerprintOf(calculator("return a + b;")) + + val after = fingerprintOf(calculator("int sum = a + b; return sum;")) + + assertThat(after).isEqualTo(before) + } + + @Test + fun `a return type change moves the fingerprint`() { + val before = fingerprintOf(calculator("return a + b;")) + + val after = + fingerprintOf( + write( + "Calculator.java", + """ + package demo; + + public class Calculator { + public long compute(int a, int b) { return (long) a + b; } + } + """, + ), + ) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `a parameter list change moves the fingerprint`() { + val before = fingerprintOf(calculator("return a + b;")) + + val after = + fingerprintOf( + write( + "Calculator.java", + """ + package demo; + + public class Calculator { + public int compute(int a, int b, int c) { return a + b + c; } + } + """, + ), + ) + + assertThat(after).isNotEqualTo(before) + } + + private fun limits(value: String) = + write( + "Limits.java", + """ + package demo; + + public class Limits { + public static final int MAX = $value; + private int scratch = 1; + } + """, + ) + + @Test + fun `a static final constant's VALUE is part of the ABI`() { + // Kotlin inlines Java compile-time constants into its callers, so the value moving + // is an ABI change even though no signature did. Dropping this would let the + // Java-ABI shortcut leave Kotlin callers holding the old constant. + val before = fingerprintOf(limits("5")) + + val after = fingerprintOf(limits("7")) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `an instance field's initializer is not part of the ABI`() { + val before = fingerprintOf(limits("5")) + + val after = + fingerprintOf( + write( + "Limits.java", + """ + package demo; + + public class Limits { + public static final int MAX = 5; + private int scratch = 42; + } + """, + ), + ) + + assertThat(after).isEqualTo(before) + } + + @Test + fun `an annotation change moves the fingerprint`() { + val before = + fingerprintOf( + write( + "Annotated.java", + """ + package demo; + + public class Annotated { + public String value() { return "x"; } + } + """, + ), + ) + + val after = + fingerprintOf( + write( + "Annotated.java", + """ + package demo; + + public class Annotated { + @Deprecated + public String value() { return "x"; } + } + """, + ), + ) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `a supertype change moves the fingerprint`() { + val before = + fingerprintOf( + write( + "Leaf.java", + """ + package demo; + + public class Leaf { + } + """, + ), + ) + + val after = + fingerprintOf( + write( + "Leaf.java", + """ + package demo; + + public class Leaf implements java.io.Serializable { + } + """, + ), + ) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `declared type names cover nested types`() { + val file = + write( + "Outer.java", + """ + package demo; + + public class Outer { + public static class Inner { + public interface Deep {} + } + } + """, + ) + + val abi = JavaSourceAbi.snapshot(listOf(file))!!.getValue(file) + + assertThat(abi.declaredTypeNames).containsExactly("Outer", "Inner", "Deep") + } + + @Test + fun `changedTypeNames reports a modified file's types`() { + val file = calculator("return a + b;") + val previous = JavaSourceAbi.snapshot(listOf(file))!! + val current = + JavaSourceAbi.snapshot( + listOf( + write( + "Calculator.java", + """ + package demo; + + public class Calculator { + public long compute(int a, int b) { return a; } + } + """, + ), + ), + )!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, current)).containsExactly("Calculator") + } + + @Test + fun `changedTypeNames reports nothing when only bodies moved`() { + val previous = JavaSourceAbi.snapshot(listOf(calculator("return a + b;")))!! + val current = JavaSourceAbi.snapshot(listOf(calculator("return b + a;")))!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, current)).isEmpty() + } + + @Test + fun `changedTypeNames reports a deleted file's types, which callers may still reference`() { + val gone = calculator("return a + b;") + val previous = JavaSourceAbi.snapshot(listOf(gone))!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, emptyMap())).containsExactly("Calculator") + } + + @Test + fun `changedTypeNames reports an added file's types`() { + val added = calculator("return a + b;") + val current = JavaSourceAbi.snapshot(listOf(added))!! + + assertThat(JavaSourceAbi.changedTypeNames(emptyMap(), current)).containsExactly("Calculator") + } + + @Test + fun `a rename reports both the old and the new name`() { + val file = write("Renamed.java", "package demo;\n\npublic class Before {}") + val previous = JavaSourceAbi.snapshot(listOf(file))!! + val current = + JavaSourceAbi.snapshot(listOf(write("Renamed.java", "package demo;\n\npublic class After {}")))!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, current)).containsExactly("Before", "After") + } + + private fun repository(dateImport: String) = + write( + "Repository.java", + """ + package demo; + + import $dateImport; + + public class Repository { + public Date created() { return null; } + } + """, + ) + + @Test + fun `swapping an import for a same-simple-name type moves the fingerprint`() { + // The signature text does not move - it still reads `Date created()` - but the type a + // Kotlin caller links against does. Miss this and changedTypeNames comes back empty, + // no Kotlin file recompiles, and the un-recompiled caller keeps a checkcast against + // the old class: ClassCastException in the running app. + val before = fingerprintOf(repository("java.util.Date")) + + val after = fingerprintOf(repository("java.sql.Date")) + + assertThat(after).isNotEqualTo(before) + } + + @Test + fun `an import swap names the declaring type as changed, forcing a Kotlin recompile`() { + val previous = JavaSourceAbi.snapshot(listOf(repository("java.util.Date")))!! + val current = JavaSourceAbi.snapshot(listOf(repository("java.sql.Date")))!! + + assertThat(JavaSourceAbi.changedTypeNames(previous, current)).containsExactly("Repository") + } + + @Test + fun `reordering imports leaves the fingerprint unchanged`() { + // Imports are hashed sorted, so a formatter's reorder must not cost a full Kotlin + // recompile - only a change to the set of imported types does. + val before = + fingerprintOf( + write( + "Ordered.java", + """ + package demo; + + import java.util.List; + import java.util.Map; + + public class Ordered { + public List> rows() { return null; } + } + """, + ), + ) + + val after = + fingerprintOf( + write( + "Ordered.java", + """ + package demo; + + import java.util.Map; + import java.util.List; + + public class Ordered { + public List> rows() { return null; } + } + """, + ), + ) + + assertThat(after).isEqualTo(before) + } + + @Test + fun `no java sources is a known-empty ABI, not an unknown one`() { + assertThat(JavaSourceAbi.snapshot(emptyList())).isEmpty() + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserEdgeTest.kt new file mode 100644 index 0000000000..3a0f6634f4 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserEdgeTest.kt @@ -0,0 +1,89 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.Test + +/** + * The severity-word override in the direction KotlincDiagnosticsParserTest doesn't pin, plus + * how a multi-line message is split between location and body. + */ +class KotlincDiagnosticsParserEdgeTest { + @Test + fun `an explicit warning prefix downgrades a message from the error channel`() { + // Some renderers deliver warnings through the error() logger channel; the text's + // own "warning:" must win, or the client would fail builds over warnings. + val diagnostic = + KotlincDiagnosticsParser.parse( + "/p/src/A.kt:3:5: warning: unused variable 'x'", + Diagnostic.Severity.ERROR, + ) + + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.WARNING) + assertThat(diagnostic.message).isEqualTo("unused variable 'x'") + assertThat(diagnostic.file).isEqualTo("/p/src/A.kt") + assertThat(diagnostic.line).isEqualTo(3) + assertThat(diagnostic.column).isEqualTo(5) + } + + @Test + fun `a location line keeps its multi-line body in the message`() { + // kotlinc renders inference failures as a headline plus indented candidate lines; the + // body is what makes the error actionable, so it must survive on the diagnostic. + val diagnostic = + KotlincDiagnosticsParser.parse( + "/p/src/A.kt:3:5: error: none of the following candidates is applicable:\n" + + " fun of(value: Int): Wrapper\n" + + " fun of(value: String): Wrapper", + Diagnostic.Severity.WARNING, + ) + + assertThat(diagnostic.file).isEqualTo("/p/src/A.kt") + assertThat(diagnostic.line).isEqualTo(3) + assertThat(diagnostic.column).isEqualTo(5) + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).startsWith("none of the following candidates is applicable:") + assertThat(diagnostic.message).contains("fun of(value: String): Wrapper") + } + + @Test + fun `a message whose location is on a later line keeps its first line`() { + // Matching the location across newlines swallowed the headline into the file group, + // producing a path with a newline in it and dropping the primary error text. + val diagnostic = + KotlincDiagnosticsParser.parse( + "inference failure: candidate not applicable\n/p/src/A.kt:3:5: error: boom", + Diagnostic.Severity.ERROR, + ) + + assertThat(diagnostic.message).contains("inference failure: candidate not applicable") + assertThat(diagnostic.file).isNull() + assertThat(diagnostic.line).isNull() + assertThat(diagnostic.column).isNull() + } + + @Test + fun `a compiler crash dump keeps its headline and its stack trace`() { + val diagnostic = + KotlincDiagnosticsParser.parse( + "e: java.lang.AssertionError: no descriptor for Foo\n" + + "\tat org.jetbrains.kotlin.Fir.resolve(Fir.kt:120)", + Diagnostic.Severity.ERROR, + ) + + assertThat(diagnostic.file).isNull() + assertThat(diagnostic.message).startsWith("e: java.lang.AssertionError: no descriptor for Foo") + assertThat(diagnostic.message).contains("Fir.kt:120") + } + + @Test + fun `a windows path parses despite the drive-letter colon`() { + val diagnostic = + KotlincDiagnosticsParser.parse("""C:\src\A.kt:3:5: error: boom""", Diagnostic.Severity.WARNING) + + assertThat(diagnostic.file).isEqualTo("""C:\src\A.kt""") + assertThat(diagnostic.line).isEqualTo(3) + assertThat(diagnostic.column).isEqualTo(5) + assertThat(diagnostic.message).isEqualTo("boom") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserTest.kt new file mode 100644 index 0000000000..251bc6d49c --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/compile/KotlincDiagnosticsParserTest.kt @@ -0,0 +1,58 @@ +package org.appdevforall.cotg.quickbuild.daemon.compile + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.Test + +class KotlincDiagnosticsParserTest { + @Test + fun `parses path line column with explicit severity`() { + val diagnostic = + KotlincDiagnosticsParser.parse( + "/p/src/B.kt:7:13: error: expecting an expression", + Diagnostic.Severity.WARNING, + ) + + assertThat(diagnostic.file).isEqualTo("/p/src/B.kt") + assertThat(diagnostic.line).isEqualTo(7) + assertThat(diagnostic.column).isEqualTo(13) + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).isEqualTo("expecting an expression") + } + + @Test + fun `parses renderer variant without severity word`() { + val diagnostic = + KotlincDiagnosticsParser.parse("/p/src/B.kt:7:13 unresolved reference: foo", Diagnostic.Severity.ERROR) + + assertThat(diagnostic.file).isEqualTo("/p/src/B.kt") + assertThat(diagnostic.line).isEqualTo(7) + assertThat(diagnostic.column).isEqualTo(13) + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).isEqualTo("unresolved reference: foo") + } + + @Test + fun `file URI locations normalize to plain paths`() { + val diagnostic = + KotlincDiagnosticsParser.parse( + "file:///p/src/Greeter.kt:4:41: error: Syntax error: Expecting an element.", + Diagnostic.Severity.ERROR, + ) + + assertThat(diagnostic.file).isEqualTo("/p/src/Greeter.kt") + assertThat(diagnostic.line).isEqualTo(4) + assertThat(diagnostic.column).isEqualTo(41) + assertThat(diagnostic.message).isEqualTo("Syntax error: Expecting an element.") + } + + @Test + fun `unparseable text degrades to a location-less diagnostic, never drops`() { + val diagnostic = KotlincDiagnosticsParser.parse("something exploded internally", Diagnostic.Severity.ERROR) + + assertThat(diagnostic.file).isNull() + assertThat(diagnostic.line).isNull() + assertThat(diagnostic.message).isEqualTo("something exploded internally") + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolEdgeTest.kt new file mode 100644 index 0000000000..c7ba28b48e --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolEdgeTest.kt @@ -0,0 +1,181 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.daemon.TestSdk +import org.appdevforall.cotg.quickbuild.daemon.compile.JavaCompileStep +import org.appdevforall.cotg.quickbuild.daemon.protocol.ProtocolCodec +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexStats +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** The `final` bit in a dex `class_def_item`'s access flags. */ +private const val ACC_FINAL = 0x10 + +/** DexTool failure surfacing and result defaults beyond DexToolTest's happy paths. */ +class DexToolEdgeTest { + @TempDir + lateinit var tempDir: File + + private fun compileTinyClass(): File = compile("Tiny", "public class Tiny", "classes") + + private fun compile( + name: String, + declaration: String, + outputDirName: String, + ): File { + val source = + File(tempDir, "$name.java").apply { + writeText("package demo;\n\n$declaration {\n\tpublic int two() { return 2; }\n}\n") + } + val classesDir = File(tempDir, outputDirName).apply { mkdirs() } + val result = JavaCompileStep.compile(listOf(source), emptyList(), classesDir) + check(result.success) { "fixture compile failed: ${result.diagnostics}" } + return classesDir + } + + /** + * The class-level access flags of every `class_def_item` in a dex, read out of the header: + * `class_defs_size`/`class_defs_off` at 0x60/0x64, then `access_flags` one uint into each + * 32-byte item. Little-endian, as the format specifies. + */ + private fun dexClassAccessFlags(dexFile: File): List { + val dex = ByteBuffer.wrap(dexFile.readBytes()).order(ByteOrder.LITTLE_ENDIAN) + val classDefs = dex.getInt(0x60) + val classDefsOffset = dex.getInt(0x64) + return (0 until classDefs).map { index -> dex.getInt(classDefsOffset + index * 32 + 4) } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `a d8 compilation failure surfaces d8's own message, not a throw`() { + val classesDir = compileTinyClass() + + // A missing library archive makes D8 itself fail (CompilationFailedException + // through the reflective call) - the daemon must relay the cause's message. + DexTool(TestSdk.d8Jar()!!, File(tempDir, "no-such-android.jar"), minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), File(tempDir, "dex")) + + assertThat(result).isInstanceOf(DexTool.Result.Failed::class.java) + assertThat((result as DexTool.Result.Failed).message).contains("d8 failed") + } + } + + @Test + fun `a dex left by an earlier run is cleared by this one, before d8 is reached`() { + // Asserted on a run that bails on empty input, so d8 never starts: the r8 jars measured + // here clear stale dex files themselves, which makes an end-to-end assertion pass whether + // or not this code clears anything. The dex count after the run is the only signal that + // d8 split the payload, so that clearing cannot be left to the device's build-tools. + val outDir = File(tempDir, "dex").apply { mkdirs() } + val stale = File(outDir, "classes2.dex").apply { writeText("stale") } + + DexTool(File(tempDir, "unopened-d8.jar"), File(tempDir, "unopened-android.jar"), minApi = 30).use { tool -> + val result = tool.dex(listOf(File(tempDir, "empty").apply { mkdirs() }), outDir) + + assertThat(result).isInstanceOf(DexTool.Result.Failed::class.java) + assertThat(stale.exists()).isFalse() + } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `a run whose payload fits one dex leaves exactly that one dex behind`() { + val classesDir = compileTinyClass() + val outDir = File(tempDir, "dex").apply { mkdirs() } + File(outDir, "classes2.dex").writeText("what a bigger earlier payload left") + + DexTool(TestSdk.d8Jar()!!, TestSdk.androidJar()!!, minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), outDir) + + // Success is only reachable on a single dex, so a leftover second one would have to + // fail the run rather than ride along into the deploy. + assertThat(result).isInstanceOf(DexTool.Result.Success::class.java) + assertThat(outDir.listFiles { file -> file.name.endsWith(".dex") }!!.map { it.name }) + .containsExactly("classes.dex") + } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `the emitted dex carries no final class, so a proxy can extend it`() { + // The gen-0 baseline shipped these classes opened by the gradle-plugin's ClassOpener, and + // the dex verifier enforces superclass finality at load time: a payload that kept + // ACC_FINAL would fail to load under the Proxy*Activity extending it. Asserted on the dex + // d8 emitted rather than on FinalStripper, because what is untested is whether DexTool + // runs the strip at all. + val classesDir = compile("TinyFinal", "public final class TinyFinal", "final-classes") + + DexTool(TestSdk.d8Jar()!!, TestSdk.androidJar()!!, minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), File(tempDir, "dex")) as DexTool.Result.Success + + val accessFlags = dexClassAccessFlags(result.dexFile) + // Without this the "none are final" assertion below passes on an empty dex. + assertThat(accessFlags).isNotEmpty() + assertThat(accessFlags.filter { it and ACC_FINAL != 0 }).isEmpty() + } + } + + @Test + fun `a payload d8 split across several dex files fails instead of shipping half of it`() { + // The split decision is asserted directly: d8 only splits past 64K method references, + // which is not a payload a unit test can build. Reaching Success here would deploy + // classes.dex alone and surface as NoClassDefFoundError against a green build. + val outDir = File(tempDir, "dex") + + val reason = + DexTool.dexFailureReason( + listOf(File(outDir, "classes.dex"), File(outDir, "classes2.dex")), + outDir, + ) + + assertThat(reason).isNotNull() + assertThat(reason).contains("classes2.dex") + // The message has to tell the user what to do instead, not just what went wrong. + assertThat(reason).contains("standard build") + } + + @Test + fun `a clean d8 exit that wrote no dex at all still fails`() { + val outDir = File(tempDir, "dex") + + assertThat(DexTool.dexFailureReason(emptyList(), outDir)).contains("no classes.dex") + // Exactly one dex is the only deployable answer. + assertThat(DexTool.dexFailureReason(listOf(File(outDir, "classes.dex")), outDir)).isNull() + } + + @Test + fun `a success without timings encodes as numeric zeros the client reads back as measured`() { + // "0 means unmeasured, never -1 and never a string" is a wire contract, so assert it on + // the wire: the same keys DaemonService.dex writes, through the real encoder, read back + // the way DaemonProcessClient reads them (JSON-number guard, else null). + val success = DexTool.Result.Success(File("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/dex/classes.dex")) + + val encoded = + ProtocolCodec.encode( + DaemonResponse.ok( + id = 7L, + values = + mapOf( + "dexFile" to success.dexFile.absolutePath, + "stripMillis" to success.stripMillis, + "d8Millis" to success.d8Millis, + ) + success.stats.toValues(), + ), + ) + + val json = JsonParser.parseString(encoded).asJsonObject + val readLong = { key: String -> json.get(key)?.takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isNumber }?.asLong } + assertThat(readLong("stripMillis")).isEqualTo(0L) + assertThat(readLong("d8Millis")).isEqualTo(0L) + // Present-and-zero, not absent: null here would tell the client this daemon predates + // the stats group and the row would be dropped rather than read as a measured zero. + assertThat(DexStats.fromValues(readLong)).isEqualTo(DexStats(classFiles = 0, classBytes = 0)) + assertThat(json.get("dexFile").asString).endsWith("classes.dex") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolTest.kt new file mode 100644 index 0000000000..b73df2767b --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/DexToolTest.kt @@ -0,0 +1,95 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.daemon.TestSdk +import org.appdevforall.cotg.quickbuild.daemon.compile.JavaCompileStep +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import org.junit.jupiter.api.io.TempDir +import java.io.File + +/** + * The dex paths that need a host SDK are guarded per-test: build-tools' d8.jar carries the + * same com.android.tools.r8.D8 the device-provisioned r8.jar does, so those exercise the + * exact reflective path. The two failure paths below never reach d8 and so must run + * everywhere - a class-level guard would skip them on an SDK-less host. + */ +class DexToolTest { + @TempDir + lateinit var tempDir: File + + private fun compileTinyClass(): File { + val source = + File(tempDir, "Tiny.java").apply { + writeText("package demo;\n\npublic class Tiny {\n\tpublic int two() { return 2; }\n}\n") + } + val classesDir = File(tempDir, "classes").apply { mkdirs() } + val result = JavaCompileStep.compile(listOf(source), emptyList(), classesDir) + check(result.success) { "fixture compile failed: ${result.diagnostics}" } + return classesDir + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `dexes compiled classes into a valid classes dex`() { + val classesDir = compileTinyClass() + val outDir = File(tempDir, "dex") + + DexTool(TestSdk.d8Jar()!!, TestSdk.androidJar()!!, minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), outDir) + + assertThat(result).isInstanceOf(DexTool.Result.Success::class.java) + val dexFile = (result as DexTool.Result.Success).dexFile + assertThat(dexFile.name).isEqualTo("classes.dex") + assertThat(dexFile.length()).isGreaterThan(0) + // The dex magic: "dex\n" then the version. + val magic = dexFile.readBytes().take(4).toByteArray() + assertThat(magic).isEqualTo(byteArrayOf(0x64, 0x65, 0x78, 0x0a)) + } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#dexToolchainAvailable") + fun `reports how many classes and bytes the pass moved`() { + // The strip pass rewrites the WHOLE tree every build, so these counts - not the + // edit's size - are what its cost scales with, and they are what makes a slow + // stripMillis readable. + val classesDir = compileTinyClass() + val classFile = File(classesDir, "demo/Tiny.class") + + DexTool(TestSdk.d8Jar()!!, TestSdk.androidJar()!!, minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), File(tempDir, "dex")) as DexTool.Result.Success + + assertThat(result.stats.classFiles).isEqualTo(1) + assertThat(result.stats.classBytes).isEqualTo(classFile.length()) + } + } + + @Test + fun `empty classes dirs fail with a message, not a throw`() { + val emptyDir = File(tempDir, "empty").apply { mkdirs() } + + // No SDK anywhere in this test on purpose: the no-input check must answer before + // d8 is ever loaded, so the tool paths are never opened. + DexTool(File(tempDir, "unopened-d8.jar"), File(tempDir, "unopened-android.jar"), minApi = 30).use { tool -> + val result = tool.dex(listOf(emptyDir), File(tempDir, "dex")) + + assertThat(result).isInstanceOf(DexTool.Result.Failed::class.java) + assertThat((result as DexTool.Result.Failed).message).contains("no .class files") + } + } + + @Test + fun `an unusable d8 jar fails with a message, not a throw`() { + val bogusJar = File(tempDir, "bogus.jar").apply { writeText("not a jar") } + val classesDir = compileTinyClass() + + // The r8 class lookup fails on the bogus jar before the platform jar is read, so + // this covers the wrong-build-tools-layout path on any host, SDK or not. + DexTool(bogusJar, File(tempDir, "unopened-android.jar"), minApi = 30).use { tool -> + val result = tool.dex(listOf(classesDir), File(tempDir, "dex")) + + assertThat(result).isInstanceOf(DexTool.Result.Failed::class.java) + } + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperInnerClassTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperInnerClassTest.kt new file mode 100644 index 0000000000..b71e2be210 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperInnerClassTest.kt @@ -0,0 +1,59 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.daemon.compile.JavaCompileStep +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.Opcodes +import java.io.File + +/** + * The InnerClasses attribute carries its own copy of each nested class's access flags; + * the dex verifier reads finality from there too, so stripping only the class-level + * ACC_FINAL would leave a final nested class the proxies cannot extend. + */ +class FinalStripperInnerClassTest { + @TempDir + lateinit var tempDir: File + + private fun innerAccessOf(classBytes: ByteArray): Int? { + var access: Int? = null + ClassReader(classBytes).accept( + object : ClassVisitor(Opcodes.ASM9) { + override fun visitInnerClass( + name: String?, + outerName: String?, + innerName: String?, + innerAccess: Int, + ) { + if (innerName == "Inner") access = innerAccess + } + }, + 0, + ) + return access + } + + @Test + fun `clears ACC_FINAL from the InnerClasses attribute entries`() { + val source = + File(tempDir, "Outer.java").apply { + writeText("package demo;\n\npublic class Outer {\n\tpublic final class Inner {}\n}\n") + } + val classesDir = File(tempDir, "classes").apply { mkdirs() } + val compiled = JavaCompileStep.compile(listOf(source), emptyList(), classesDir) + check(compiled.success) { "fixture compile failed: ${compiled.diagnostics}" } + val outerBytes = File(classesDir, "demo/Outer.class").readBytes() + // Guard against a vacuous fixture: the entry must start out final. + assertThat(innerAccessOf(outerBytes)!! and Opcodes.ACC_FINAL).isEqualTo(Opcodes.ACC_FINAL) + + val stripped = FinalStripper.strip(outerBytes) + + val strippedAccess = innerAccessOf(stripped)!! + assertThat(strippedAccess and Opcodes.ACC_FINAL).isEqualTo(0) + // Everything else about the entry survives (still a public member class). + assertThat(strippedAccess and Opcodes.ACC_PUBLIC).isEqualTo(Opcodes.ACC_PUBLIC) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperTest.kt new file mode 100644 index 0000000000..4e4fb2db69 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/dex/FinalStripperTest.kt @@ -0,0 +1,208 @@ +package org.appdevforall.cotg.quickbuild.daemon.dex + +import com.google.common.truth.Truth.assertThat +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Test +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.MethodVisitor +import org.objectweb.asm.Opcodes +import java.io.File +import java.lang.reflect.Modifier +import java.nio.file.Files +import javax.tools.ToolProvider + +class FinalStripperTest { + private fun compileToDir( + className: String, + source: String, + ): File { + val dir = Files.createTempDirectory("final-stripper").toFile() + val src = dir.resolve("$className.java").apply { writeText(source) } + val compiler = ToolProvider.getSystemJavaCompiler() + check(compiler.run(null, null, null, "-d", dir.absolutePath, src.absolutePath) == 0) { + "test fixture failed to compile" + } + return dir + } + + private fun compile( + className: String, + source: String, + ): ByteArray = compileToDir(className, source).resolve("$className.class").readBytes() + + private fun accessFlags(classBytes: ByteArray): Int = ClassReader(classBytes).access + + private fun methodAccessFlags( + classBytes: ByteArray, + methodName: String, + ): Int { + var access = 0 + ClassReader(classBytes).accept( + object : ClassVisitor(Opcodes.ASM9) { + override fun visitMethod( + methodAccess: Int, + name: String?, + descriptor: String?, + signature: String?, + exceptions: Array?, + ): MethodVisitor? { + if (name == methodName) access = methodAccess + return null + } + }, + 0, + ) + return access + } + + /** Defines exactly the bytes it is handed, so stripped output can be loaded and extended. */ + private class BytesClassLoader( + private val classes: Map, + ) : ClassLoader(BytesClassLoader::class.java.classLoader) { + override fun findClass(name: String): Class<*> { + val bytes = classes[name] ?: return super.findClass(name) + return defineClass(name, bytes, 0, bytes.size) + } + } + + /** + * Generates `public class extends ` with a default constructor - the shape + * of the proxy app's generated Proxy*Activity classes, which is what the strip exists to make + * loadable. Version 52 loads on any JDK these tests run on, and the JVM places no version + * relationship between a class and its superclass. + * + * @param superName internal name of the class to extend, e.g. `SealedFixture`. + * @param name internal name to give the generated subclass. + * @return a whole class file. + */ + private fun subclassBytes( + superName: String, + name: String, + ): ByteArray { + val writer = ClassWriter(0) + writer.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, name, null, superName, null) + val constructor = writer.visitMethod(Opcodes.ACC_PUBLIC, "", "()V", null, null) + constructor.visitCode() + constructor.visitVarInsn(Opcodes.ALOAD, 0) + constructor.visitMethodInsn(Opcodes.INVOKESPECIAL, superName, "", "()V", false) + constructor.visitInsn(Opcodes.RETURN) + constructor.visitMaxs(1, 1) + constructor.visitEnd() + writer.visitEnd() + return writer.toByteArray() + } + + @Test + fun `clears ACC_FINAL from a final class`() { + val bytes = compile("FinalFixture", "public final class FinalFixture {}") + assertThat(accessFlags(bytes) and Opcodes.ACC_FINAL).isNotEqualTo(0) + + val stripped = FinalStripper.strip(bytes) + + assertThat(accessFlags(stripped) and Opcodes.ACC_FINAL).isEqualTo(0) + // The class is otherwise intact: same name, still loadable by ASM, still public. + assertThat(ClassReader(stripped).className).isEqualTo("FinalFixture") + assertThat(accessFlags(stripped) and Opcodes.ACC_PUBLIC).isNotEqualTo(0) + } + + @Test + fun `leaves a non-final class byte-identical in behavior`() { + val bytes = compile("OpenFixture", "public class OpenFixture { public int f() { return 7; } }") + + val stripped = FinalStripper.strip(bytes) + + assertThat(accessFlags(stripped)).isEqualTo(accessFlags(bytes)) + assertThat(ClassReader(stripped).className).isEqualTo("OpenFixture") + } + + @Test + fun `stripped bytes load and a generated subclass of them instantiates`() { + // The contract is not "the flag is clear" but "a proxy can extend it": the JVM resolves + // the superclass while defining the subclass and rejects a final one, the same check the + // dex verifier makes on device. Asserting the flag alone would pass on bytes no verifier + // accepts (a broken constant pool, say). + val bytes = compile("SealedFixture", "public final class SealedFixture { public int v() { return 5; } }") + + val stripped = FinalStripper.strip(bytes) + + val loader = + BytesClassLoader( + mapOf( + "SealedFixture" to stripped, + "SubSealed" to subclassBytes("SealedFixture", "SubSealed"), + ), + ) + val opened = loader.loadClass("SealedFixture") + assertThat(Modifier.isFinal(opened.modifiers)).isFalse() + val instance = loader.loadClass("SubSealed").getDeclaredConstructor().newInstance() + assertThat(opened.isInstance(instance)).isTrue() + assertThat(opened.getMethod("v").invoke(instance)).isEqualTo(5) + } + + @Test + fun `the same subclass over UNSTRIPPED bytes is rejected by the JVM`() { + // Control for the test above: with the strip removed (or turned into a no-op) the JVM + // refuses the subclass, so that test cannot pass vacuously. A generator bug would fail + // both tests, never only this one. + val bytes = compile("ClosedFixture", "public final class ClosedFixture { public int v() { return 5; } }") + val loader = + BytesClassLoader( + mapOf( + "ClosedFixture" to bytes, + "SubClosed" to subclassBytes("ClosedFixture", "SubClosed"), + ), + ) + + // IncompatibleClassChangeError on HotSpot ("cannot inherit from final class"); the + // assertion names the LinkageError family so it does not pin one JVM's choice, and + // instantiates so a JVM that defers the check to initialization is covered too. + assertThrows(LinkageError::class.java) { + loader.loadClass("SubClosed").getDeclaredConstructor().newInstance() + } + } + + @Test + fun `a stripped nested class loads and can be extended, InnerClasses entry included`() { + // DexTool strips every .class file it walks, so a nested pair arrives here as two + // separate strips. HotSpot computes a member class's reflective modifiers from the + // InnerClasses attribute, so the modifier assertion also exercises the entry rewrite + // FinalStripperInnerClassTest checks at byte level - though only the subclass step below + // can fail on the class-level flag alone. + val dir = compileToDir("Nested", "public class Nested {\n\tpublic static final class Inner {}\n}\n") + val outer = FinalStripper.strip(dir.resolve("Nested.class").readBytes()) + val inner = FinalStripper.strip(dir.resolve("Nested\$Inner.class").readBytes()) + + val loader = + BytesClassLoader( + mapOf( + "Nested" to outer, + "Nested\$Inner" to inner, + "SubInner" to subclassBytes("Nested\$Inner", "SubInner"), + ), + ) + val openedInner = loader.loadClass("Nested\$Inner") + assertThat(Modifier.isFinal(openedInner.modifiers)).isFalse() + val instance = loader.loadClass("SubInner").getDeclaredConstructor().newInstance() + assertThat(openedInner.isInstance(instance)).isTrue() + } + + @Test + fun `a final METHOD keeps its flag - the strip opens classes, not members`() { + // Deliberate scope, matching the gradle-plugin's ClassOpener byte for byte: the payload + // dex must carry what the gen-0 baseline opened, no more. A final lifecycle method that + // a generated proxy overrides fails at gen-0, in the proxy's javac pass, not here. + val bytes = + compile( + "FinalMethodFixture", + "public final class FinalMethodFixture { public final int v() { return 3; } }", + ) + assertThat(methodAccessFlags(bytes, "v") and Opcodes.ACC_FINAL).isEqualTo(Opcodes.ACC_FINAL) + + val stripped = FinalStripper.strip(bytes) + + assertThat(accessFlags(stripped) and Opcodes.ACC_FINAL).isEqualTo(0) + assertThat(methodAccessFlags(stripped, "v") and Opcodes.ACC_FINAL).isEqualTo(Opcodes.ACC_FINAL) + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecEdgeTest.kt new file mode 100644 index 0000000000..d535ee04c4 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecEdgeTest.kt @@ -0,0 +1,109 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.ParseResult +import org.junit.jupiter.api.Test + +/** + * The codec's malformed-input taxonomy beyond ProtocolCodecTest: wrong TYPES (not just + * missing fields) for ids, ops, strings and arrays. Every one must come back as + * [ParseResult.Malformed] naming the offender - the daemon serves external callers, so an + * unexpected shape must produce an actionable reply, never a throw or a misparse. + */ +class ProtocolCodecEdgeTest { + private fun malformed(line: String): ParseResult.Malformed { + val parsed = ProtocolCodec.parse(line) + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + return parsed as ParseResult.Malformed + } + + @Test + fun `missing op is malformed but keeps the id for correlation`() { + val parsed = malformed("""{"id": 5}""") + + assertThat(parsed.id).isEqualTo(5) + assertThat(parsed.message).contains("op") + } + + @Test + fun `a non-string op is malformed, not misdispatched`() { + assertThat(malformed("""{"id": 5, "op": 42}""").message).contains("op") + assertThat(malformed("""{"id": 5, "op": {"nested": true}}""").message).contains("op") + } + + @Test + fun `a non-numeric id is malformed with the unknown id`() { + assertThat(malformed("""{"id": "seven", "op": "ping"}""").id).isEqualTo(ParseResult.Malformed.UNKNOWN_ID) + assertThat(malformed("""{"id": [7], "op": "ping"}""").id).isEqualTo(ParseResult.Malformed.UNKNOWN_ID) + assertThat(malformed("""{"id": true, "op": "ping"}""").id).isEqualTo(ParseResult.Malformed.UNKNOWN_ID) + } + + @Test + fun `a missing required string names the field`() { + val parsed = malformed("""{"id": 1, "op": "configure", "classpath": [], "outDir": "/out"}""") + + assertThat(parsed.id).isEqualTo(1) + assertThat(parsed.message).contains("projectRoot") + } + + @Test + fun `a required string of the wrong type names the field`() { + val parsed = + malformed("""{"id": 4, "op": "relink", "resDirs": ["/res"], "manifest": 7}""") + + assertThat(parsed.message).contains("manifest") + } + + @Test + fun `a required list that is not an array names the field`() { + val parsed = malformed("""{"id": 3, "op": "dex", "classesDirs": "/classes"}""") + + assertThat(parsed.id).isEqualTo(3) + assertThat(parsed.message).contains("classesDirs") + assertThat(parsed.message).contains("not an array") + } + + @Test + fun `a list containing a non-primitive element names the field`() { + val parsed = malformed("""{"id": 3, "op": "dex", "classesDirs": [{"path": "/x"}]}""") + + assertThat(parsed.message).contains("classesDirs") + assertThat(parsed.message).contains("non-string") + } + + @Test + fun `a missing required list names the field`() { + val parsed = malformed("""{"id": 2, "op": "compile", "changedFiles": []}""") + + assertThat(parsed.message).contains("allSources") + } + + @Test + fun `an op that hash-collides with a real one is unknown, never misdispatched`() { + // Each of these has the same String.hashCode() as a real op (the Java "Aa"/"BB" + // collision family) but different text. Dispatch must compare the actual value, + // not just the hash - a collision routed to a build op would run it with garbage. + val collisions = + listOf("dPnfigure", "dPmpile", "eFx", "sFlink", "qJng", "tIutdown") + + for (op in collisions) { + val parsed = malformed("""{"id": 8, "op": "$op"}""") + + assertThat(parsed.id).isEqualTo(8) + assertThat(parsed.message).contains("unknown op") + assertThat(parsed.message).contains(op) + } + } + + @Test + fun `encode writes boolean values as JSON booleans, not strings`() { + val encoded = ProtocolCodec.encode(DaemonResponse.ok(6, mapOf("incremental" to true))) + + val root = JsonParser.parseString(encoded).asJsonObject + assertThat(root.get("incremental").isJsonPrimitive).isTrue() + assertThat(root.get("incremental").asJsonPrimitive.isBoolean).isTrue() + assertThat(root.get("incremental").asBoolean).isTrue() + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecTest.kt new file mode 100644 index 0000000000..694e8284b1 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/ProtocolCodecTest.kt @@ -0,0 +1,330 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.CompileStats +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.DexStats +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.ParseResult +import org.appdevforall.cotg.quickbuild.protocol.PingRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.ShutdownRequest +import org.junit.jupiter.api.Test + +class ProtocolCodecTest { + @Test + fun `configure request round-trips every field`() { + val line = + """{"id": 1, "op": "configure", "projectRoot": "/p", "classpath": ["/a.jar", "/b.jar"], + "outDir": "/out", "aapt2": "/aapt2", "d8Jar": "/r8.jar", "androidJar": "/android.jar", + "minApi": 26, "compilerPlugins": ["/compose-compiler-plugin.jar"]}""".replace("\n", "") + + val parsed = ProtocolCodec.parse(line) + + assertThat(parsed).isInstanceOf(ParseResult.Parsed::class.java) + val request = (parsed as ParseResult.Parsed).request as ConfigureRequest + assertThat(request.id).isEqualTo(1) + assertThat(request.projectRoot).isEqualTo("/p") + assertThat(request.classpath).containsExactly("/a.jar", "/b.jar").inOrder() + assertThat(request.outDir).isEqualTo("/out") + assertThat(request.aapt2).isEqualTo("/aapt2") + assertThat(request.d8Jar).isEqualTo("/r8.jar") + assertThat(request.androidJar).isEqualTo("/android.jar") + assertThat(request.minApi).isEqualTo(26) + assertThat(request.compilerPlugins).containsExactly("/compose-compiler-plugin.jar") + } + + @Test + fun `configure without minApi defaults to the v1 floor`() { + val line = + """{"id": 1, "op": "configure", "projectRoot": "/p", "classpath": [], + "outDir": "/out", "aapt2": "/aapt2", "d8Jar": "/r8.jar", "androidJar": "/android.jar"}""".replace("\n", "") + + val request = ((ProtocolCodec.parse(line)) as ParseResult.Parsed).request as ConfigureRequest + + assertThat(request.minApi).isEqualTo(30) + assertThat(request.compilerPlugins).isEmpty() + } + + @Test + fun `configure without aapt2, d8Jar or androidJar parses to nulls so the daemon can self-discover them`() { + val line = """{"id": 1, "op": "configure", "projectRoot": "/p", "classpath": [], "outDir": "/out"}""" + + val request = ((ProtocolCodec.parse(line)) as ParseResult.Parsed).request as ConfigureRequest + + assertThat(request.aapt2).isNull() + assertThat(request.d8Jar).isNull() + assertThat(request.androidJar).isNull() + } + + @Test + fun `compile dex relink ping shutdown parse to their request types`() { + val compile = + ProtocolCodec.parse("""{"id": 2, "op": "compile", "allSources": ["/A.kt"], "changedFiles": []}""") + val dex = ProtocolCodec.parse("""{"id": 3, "op": "dex", "classesDirs": ["/classes"]}""") + val relink = ProtocolCodec.parse("""{"id": 4, "op": "relink", "resDirs": ["/res"], "manifest": "/M.xml"}""") + val ping = ProtocolCodec.parse("""{"id": 5, "op": "ping"}""") + val shutdown = ProtocolCodec.parse("""{"id": 6, "op": "shutdown"}""") + + assertThat((compile as ParseResult.Parsed).request) + .isEqualTo(CompileRequest(2, listOf("/A.kt"), emptyList())) + assertThat((dex as ParseResult.Parsed).request).isEqualTo(DexRequest(3, listOf("/classes"))) + assertThat((relink as ParseResult.Parsed).request) + .isEqualTo(RelinkRequest(4, listOf("/res"), "/M.xml")) + assertThat((ping as ParseResult.Parsed).request).isEqualTo(PingRequest(5)) + assertThat((shutdown as ParseResult.Parsed).request).isEqualTo(ShutdownRequest(6)) + } + + @Test + fun `compile request carries an optional removedFiles list when present, empty otherwise`() { + val withRemoved = + ProtocolCodec.parse( + """{"id": 2, "op": "compile", "allSources": ["/A.kt"], "changedFiles": [], "removedFiles": ["/Gone.kt"]}""", + ) + val withoutRemoved = + ProtocolCodec.parse("""{"id": 3, "op": "compile", "allSources": ["/A.kt"], "changedFiles": []}""") + + assertThat(((withRemoved as ParseResult.Parsed).request as CompileRequest).removedFiles) + .containsExactly("/Gone.kt") + assertThat(((withoutRemoved as ParseResult.Parsed).request as CompileRequest).removedFiles) + .isEmpty() + } + + @Test + fun `relink request carries an optional stableIds path when present, null otherwise`() { + val withStableIds = + ProtocolCodec.parse( + """{"id": 7, "op": "relink", "resDirs": ["/res"], "manifest": "/M.xml", + "stableIds": "/stableIds.txt"}""".replace("\n", ""), + ) + val withoutStableIds = + ProtocolCodec.parse("""{"id": 8, "op": "relink", "resDirs": ["/res"], "manifest": "/M.xml"}""") + + assertThat((withStableIds as ParseResult.Parsed).request) + .isEqualTo(RelinkRequest(7, listOf("/res"), "/M.xml", "/stableIds.txt")) + assertThat((withoutStableIds as ParseResult.Parsed).request) + .isEqualTo(RelinkRequest(8, listOf("/res"), "/M.xml", null)) + } + + @Test + fun `relink request carries an optional libraryResources list when present, empty otherwise`() { + val withLibraryResources = + ProtocolCodec.parse( + """{"id": 10, "op": "relink", "resDirs": ["/res"], "manifest": "/M.xml", + "libraryResources": ["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/merged_res/values_values.arsc.flat", "/lib/drawable_x.xml.flat"]}""".replace( + "\n", + "", + ), + ) + val withoutLibraryResources = + ProtocolCodec.parse("""{"id": 11, "op": "relink", "resDirs": ["/res"], "manifest": "/M.xml"}""") + + assertThat((withLibraryResources as ParseResult.Parsed).request) + .isEqualTo( + RelinkRequest( + 10, + listOf("/res"), + "/M.xml", + libraryResources = listOf("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/merged_res/values_values.arsc.flat", "/lib/drawable_x.xml.flat"), + ), + ) + assertThat((withoutLibraryResources as ParseResult.Parsed).request) + .isEqualTo(RelinkRequest(11, listOf("/res"), "/M.xml")) + } + + @Test + fun `invalid JSON is malformed with unknown id, never a throw`() { + val parsed = ProtocolCodec.parse("this is not json {") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + assertThat((parsed as ParseResult.Malformed).id).isEqualTo(ParseResult.Malformed.UNKNOWN_ID) + } + + @Test + fun `missing id is malformed`() { + val parsed = ProtocolCodec.parse("""{"op": "ping"}""") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + } + + @Test + fun `unknown op is malformed but keeps the id for correlation`() { + val parsed = ProtocolCodec.parse("""{"id": 9, "op": "transmogrify"}""") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + assertThat((parsed as ParseResult.Malformed).id).isEqualTo(9) + assertThat(parsed.message).contains("transmogrify") + } + + @Test + fun `missing required field is malformed with the field named`() { + val parsed = ProtocolCodec.parse("""{"id": 2, "op": "compile", "allSources": ["/A.kt"]}""") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + assertThat((parsed as ParseResult.Malformed).message).contains("changedFiles") + } + + @Test + fun `non-string element in a string list is malformed`() { + val parsed = ProtocolCodec.parse("""{"id": 3, "op": "dex", "classesDirs": ["/ok", 42]}""") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + } + + @Test + fun `array root is malformed`() { + val parsed = ProtocolCodec.parse("""[1, 2, 3]""") + + assertThat(parsed).isInstanceOf(ParseResult.Malformed::class.java) + } + + @Test + fun `ok response encodes flat values`() { + val encoded = + ProtocolCodec.encode( + DaemonResponse.ok(7, mapOf("classesDir" to "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/out/classes", "durationMillis" to 123L)), + ) + + val root = JsonParser.parseString(encoded).asJsonObject + assertThat(root.get("id").asLong).isEqualTo(7) + assertThat(root.get("ok").asBoolean).isTrue() + assertThat(root.get("classesDir").asString).isEqualTo("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/out/classes") + assertThat(root.get("durationMillis").asLong).isEqualTo(123) + assertThat(root.has("diagnostics")).isFalse() + } + + @Test + fun `ok response encodes list values as JSON arrays - the classesChanged shape`() { + val encoded = + ProtocolCodec.encode( + DaemonResponse.ok( + 8, + mapOf("classesChanged" to listOf("demo/Greeter.class", "demo/Outer\$Inner.class")), + ), + ) + + val root = JsonParser.parseString(encoded).asJsonObject + assertThat(root.get("classesChanged").isJsonArray).isTrue() + assertThat(root.getAsJsonArray("classesChanged").map { it.asString }) + .containsExactly("demo/Greeter.class", "demo/Outer\$Inner.class") + .inOrder() + } + + @Test + fun `failure response encodes diagnostics in the protocol shape`() { + val encoded = + ProtocolCodec.encode( + DaemonResponse.failure( + 8, + listOf( + Diagnostic(Diagnostic.Severity.ERROR, "expecting an expression", "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/p/B.kt", 7, 13), + Diagnostic(Diagnostic.Severity.WARNING, "no location"), + ), + ), + ) + + val root = JsonParser.parseString(encoded).asJsonObject + assertThat(root.get("ok").asBoolean).isFalse() + val diagnostics = root.getAsJsonArray("diagnostics") + assertThat(diagnostics.size()).isEqualTo(2) + val first = diagnostics[0].asJsonObject + assertThat(first.get("severity").asString).isEqualTo("ERROR") + assertThat(first.get("message").asString).isEqualTo("expecting an expression") + assertThat(first.get("file").asString).isEqualTo("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/p/B.kt") + assertThat(first.get("line").asInt).isEqualTo(7) + assertThat(first.get("column").asInt).isEqualTo(13) + val second = diagnostics[1].asJsonObject + assertThat(second.has("file")).isFalse() + assertThat(second.has("line")).isFalse() + } + + @Test + fun `compile stats survive the wire and read back identically`() { + val stats = + CompileStats( + preSnapMillis = 120, + postSnapMillis = 130, + javaAbiSnapMillis = 540, + allSources = 292, + kotlinToCompile = 74, + javaSources = 218, + changedClasses = 323, + compileOrdinal = 3, + ) + + val root = + JsonParser + .parseString(ProtocolCodec.encode(DaemonResponse.ok(1, stats.toValues()))) + .asJsonObject + + assertThat(CompileStats.fromValues { key -> root.get(key)?.asLong }).isEqualTo(stats) + } + + @Test + fun `dex stats survive the wire and read back identically`() { + val stats = DexStats(classFiles = 464, classBytes = 1_530_112) + + val root = + JsonParser + .parseString(ProtocolCodec.encode(DaemonResponse.ok(1, stats.toValues()))) + .asJsonObject + + assertThat(DexStats.fromValues { key -> root.get(key)?.asLong }).isEqualTo(stats) + } + + @Test + fun `stats read back as null from a daemon that predates them`() { + // The version-safety property: an OLDER daemon answering a NEWER client omits these + // keys entirely. That must read as "not measured", not as a zero-filled row claiming + // every phase was free. + val root = + JsonParser + .parseString(ProtocolCodec.encode(DaemonResponse.ok(1, mapOf("classesDir" to "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/out/classes")))) + .asJsonObject + + assertThat(CompileStats.fromValues { key -> root.get(key)?.asLong }).isNull() + assertThat(DexStats.fromValues { key -> root.get(key)?.asLong }).isNull() + } + + @Test + fun `a partially reported stats group fills the gaps rather than vanishing`() { + // The other direction: a FUTURE daemon that drops a key still reports what it has. + val partial = mapOf(CompileStats.KEY_COMPILE_ORDINAL to 5L) + + val stats = CompileStats.fromValues { key -> (partial[key] as? Long) } + + assertThat(stats).isNotNull() + assertThat(stats!!.compileOrdinal).isEqualTo(5) + assertThat(stats.preSnapMillis).isEqualTo(0) + } + + @Test + fun `adding response fields does not move the protocol version`() { + // Version is a hard session gate and a staged daemon jar can lag the client, so an + // additive optional field must NOT bump it - the additive shape is what lets the two + // sides drift safely. + assertThat(DaemonResponse.PROTOCOL_VERSION).isEqualTo(1) + } + + @Test + fun `encoded response is a single line even with newlines in messages`() { + val encoded = + ProtocolCodec.encode( + DaemonResponse.failure(9, listOf(Diagnostic(Diagnostic.Severity.ERROR, "line one\nline two"))), + ) + + assertThat(encoded).doesNotContain("\n") + val root = JsonParser.parseString(encoded).asJsonObject + val message = + root + .getAsJsonArray("diagnostics")[0] + .asJsonObject + .get("message") + .asString + assertThat(message).isEqualTo("line one\nline two") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterErrorTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterErrorTest.kt new file mode 100644 index 0000000000..d84948deee --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterErrorTest.kt @@ -0,0 +1,106 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +/** + * The [Error] half of the backstop, which the [Exception] cases in RequestRouterGuardTest do not + * cover: the compiler runs in the daemon's own JVM, so an out-of-memory or a parser stack + * overflow on the user's source would otherwise leave `route`, leave `main`, and exit the + * process. CoGo reads that as daemon death and restarts, so the same save would kill the same + * daemon forever with no diagnostic ever rendered. + */ +class RequestRouterErrorTest { + private class ThrowingHandlers( + private val boom: () -> Nothing, + ) : DaemonHandlers { + override fun configure(request: ConfigureRequest): DaemonResponse = boom() + + override fun compile(request: CompileRequest): DaemonResponse = boom() + + override fun dex(request: DexRequest): DaemonResponse = boom() + + override fun relink(request: RelinkRequest): DaemonResponse = boom() + } + + private fun everyBuildOp(): List = + listOf( + ConfigureRequest(31, "/p", emptyList(), "/out"), + CompileRequest(32, emptyList(), emptyList()), + DexRequest(33, emptyList()), + RelinkRequest(34, emptyList(), "/M.xml"), + ) + + @Test + fun `an out-of-memory from any build op becomes an ok-false reply naming the memory`() { + val router = RequestRouter(ThrowingHandlers { throw OutOfMemoryError("Java heap space") }) + + for (request in everyBuildOp()) { + val routed = router.route(request) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response.ok).isFalse() + assertThat(routed.response.id).isEqualTo(request.id) + val diagnostic = routed.response.diagnostics.single() + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).contains("ran out of memory") + // The point of naming the condition: "internal error" would tell the user nothing + // they could act on, and this is a build outcome they can. + assertThat(diagnostic.message).doesNotContain("internal") + } + } + + @Test + fun `a stack overflow from any build op becomes an ok-false reply naming the nesting`() { + val router = RequestRouter(ThrowingHandlers { throw StackOverflowError() }) + + for (request in everyBuildOp()) { + val routed = router.route(request) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response.ok).isFalse() + assertThat(routed.response.id).isEqualTo(request.id) + assertThat( + routed.response.diagnostics + .single() + .message, + ).contains("nests too") + } + } + + @Test + fun `a linkage error still escapes, because that one really is fatal`() { + val router = RequestRouter(ThrowingHandlers { throw NoClassDefFoundError("com/example/Gone") }) + + assertThrows { + router.route(CompileRequest(35, emptyList(), emptyList())) + } + } + + @Test + fun `the failure classifier splits request failures from fatal ones`() { + assertThat(RequestRouter.isRequestFailure(IllegalStateException("tool exploded"))).isTrue() + assertThat(RequestRouter.isRequestFailure(OutOfMemoryError("Java heap space"))).isTrue() + assertThat(RequestRouter.isRequestFailure(StackOverflowError())).isTrue() + + assertThat(RequestRouter.isRequestFailure(NoClassDefFoundError("com/example/Gone"))).isFalse() + assertThat(RequestRouter.isRequestFailure(UnsatisfiedLinkError("libd8"))).isFalse() + assertThat(RequestRouter.isRequestFailure(InternalError("vm"))).isFalse() + } + + @Test + fun `an ordinary exception keeps its class and message, which the two Errors replace`() { + assertThat(RequestRouter.describe(IllegalStateException("tool exploded"))) + .isEqualTo("internal: IllegalStateException: tool exploded") + assertThat(RequestRouter.describe(OutOfMemoryError("Java heap space"))) + .doesNotContain("Java heap space") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterGuardTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterGuardTest.kt new file mode 100644 index 0000000000..bf6d257747 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterGuardTest.kt @@ -0,0 +1,54 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.junit.jupiter.api.Test + +/** + * The exception backstop on EVERY build op, not just compile (RequestRouterTest covers + * that one): `guarded` is inline, so each op's call site carries its own copy of the + * catch - a throw escaping any one of them would kill the daemon process, breaking the + * README contract that the daemon only exits on shutdown, EOF, or a fatal internal error. + */ +class RequestRouterGuardTest { + private class ThrowingHandlers( + private val boom: Exception, + ) : DaemonHandlers { + override fun configure(request: ConfigureRequest): DaemonResponse = throw boom + + override fun compile(request: CompileRequest): DaemonResponse = throw boom + + override fun dex(request: DexRequest): DaemonResponse = throw boom + + override fun relink(request: RelinkRequest): DaemonResponse = throw boom + } + + @Test + fun `an exception from any build op becomes an ok-false reply carrying that op's id`() { + val router = RequestRouter(ThrowingHandlers(IllegalStateException("tool exploded"))) + val requests = + listOf( + ConfigureRequest(21, "/p", emptyList(), "/out"), + CompileRequest(22, emptyList(), emptyList()), + DexRequest(23, emptyList()), + RelinkRequest(24, emptyList(), "/M.xml"), + ) + + for (request in requests) { + val routed = router.route(request) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response.ok).isFalse() + assertThat(routed.response.id).isEqualTo(request.id) + val diagnostic = routed.response.diagnostics.single() + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).contains("IllegalStateException") + assertThat(diagnostic.message).contains("tool exploded") + } + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterTest.kt new file mode 100644 index 0000000000..d18751cc6e --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/protocol/RequestRouterTest.kt @@ -0,0 +1,92 @@ +package org.appdevforall.cotg.quickbuild.daemon.protocol + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.protocol.CompileRequest +import org.appdevforall.cotg.quickbuild.protocol.ConfigureRequest +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.DexRequest +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.appdevforall.cotg.quickbuild.protocol.PingRequest +import org.appdevforall.cotg.quickbuild.protocol.RelinkRequest +import org.appdevforall.cotg.quickbuild.protocol.ShutdownRequest +import org.junit.jupiter.api.Test + +class RequestRouterTest { + private class RecordingHandlers : DaemonHandlers { + val calls = mutableListOf() + var throwOnCompile: Exception? = null + + override fun configure(request: ConfigureRequest): DaemonResponse { + calls += "configure" + return DaemonResponse.ok(request.id) + } + + override fun compile(request: CompileRequest): DaemonResponse { + calls += "compile" + throwOnCompile?.let { throw it } + return DaemonResponse.ok(request.id, mapOf("classesDir" to "/out")) + } + + override fun dex(request: DexRequest): DaemonResponse { + calls += "dex" + return DaemonResponse.ok(request.id) + } + + override fun relink(request: RelinkRequest): DaemonResponse { + calls += "relink" + return DaemonResponse.ok(request.id) + } + } + + private val handlers = RecordingHandlers() + private val router = RequestRouter(handlers) + + private fun configureRequest(id: Long = 1) = ConfigureRequest(id, "/p", emptyList(), "/out", "/aapt2", "/r8.jar", "/android.jar") + + @Test + fun `build ops route to their handlers and reply`() { + val configure = router.route(configureRequest(1)) + val compile = router.route(CompileRequest(2, emptyList(), emptyList())) + val dex = router.route(DexRequest(3, emptyList())) + val relink = router.route(RelinkRequest(4, emptyList(), "/M.xml")) + + assertThat(handlers.calls).containsExactly("configure", "compile", "dex", "relink").inOrder() + for (routed in listOf(configure, compile, dex, relink)) { + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response.ok).isTrue() + } + assertThat(compile.response.values["classesDir"]).isEqualTo("/out") + } + + @Test + fun `ping replies ok with the protocol version, without touching handlers`() { + val routed = router.route(PingRequest(5)) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response) + .isEqualTo(DaemonResponse.ok(5, mapOf("protocolVersion" to DaemonResponse.PROTOCOL_VERSION))) + assertThat(handlers.calls).isEmpty() + } + + @Test + fun `shutdown replies ok and signals exit`() { + val routed = router.route(ShutdownRequest(6)) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.ReplyThenExit::class.java) + assertThat(routed.response).isEqualTo(DaemonResponse.ok(6)) + } + + @Test + fun `a handler exception becomes an ok-false response, never a throw`() { + handlers.throwOnCompile = IllegalStateException("compiler exploded") + + val routed = router.route(CompileRequest(7, emptyList(), emptyList())) + + assertThat(routed).isInstanceOf(RequestRouter.Routed.Reply::class.java) + assertThat(routed.response.ok).isFalse() + assertThat(routed.response.id).isEqualTo(7) + val diagnostic = routed.response.diagnostics.single() + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).contains("compiler exploded") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkEdgeTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkEdgeTest.kt new file mode 100644 index 0000000000..4b6d4648d1 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkEdgeTest.kt @@ -0,0 +1,165 @@ +package org.appdevforall.cotg.quickbuild.daemon.res + +import com.google.common.truth.Truth.assertThat +import com.google.gson.JsonParser +import org.appdevforall.cotg.quickbuild.daemon.protocol.ProtocolCodec +import org.appdevforall.cotg.quickbuild.protocol.DaemonResponse +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * Aapt2Link's output verification and diagnostic parsing, driven by scripted fake aapt2 + * binaries: what happens when aapt2 exits 0 but produced garbage, and how its stderr + * lines map to the protocol's diagnostics. No real toolchain needed - the fakes let these + * run (and pin behavior) on any POSIX host. + */ +class Aapt2LinkEdgeTest { + @TempDir + lateinit var tempDir: File + + private lateinit var resDir: File + private lateinit var manifest: File + private lateinit var workDir: File + + @BeforeEach + fun setUp() { + resDir = File(tempDir, "res/values").apply { mkdirs() }.parentFile + File(resDir, "values/strings.xml").writeText("") + workDir = File(tempDir, "work").apply { mkdirs() } + manifest = File(tempDir, "AndroidManifest.xml").apply { writeText("") } + } + + private fun fakeAapt2(script: String): File = + File(tempDir, "fake-aapt2").apply { + writeText("#!/bin/sh\n$script\n") + check(setExecutable(true)) { "could not mark fake aapt2 executable" } + } + + @Test + fun `link exiting 0 without producing an output fails instead of shipping nothing`() { + val link = Aapt2Link(fakeAapt2("exit 0"), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostics = (result as Aapt2Link.Result.Failed).diagnostics + assertThat(diagnostics.single().severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostics.single().message).contains("no resources.arsc") + } + + @Test + fun `a linked apk without a resource table fails instead of shipping a broken payload`() { + // The whole apk is the payload; an entry-less table means the runtime cannot load + // it, so exit-0-with-garbage must fail loudly (class KDoc: malformed despite 0). + val tableless = File(tempDir, "tableless.zip") + ZipOutputStream(tableless.outputStream()).use { zip -> + zip.putNextEntry(ZipEntry("res/dummy.txt")) + zip.write("no table here".toByteArray()) + zip.closeEntry() + } + // The fake link copies the prepared no-arsc zip to aapt2's -o argument ($3). + val script = "if [ \"\$1\" = \"link\" ]; then cp '${tableless.absolutePath}' \"\$3\"; fi\nexit 0" + val link = Aapt2Link(fakeAapt2(script), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + assertThat((result as Aapt2Link.Result.Failed).diagnostics.single().message).contains("no resources.arsc") + } + + @Test + fun `warning-only aapt2 output gains a fallback error so a failure is never silent`() { + val script = + "echo 'res/values/strings.xml:4: warning: dubious value'\n" + + "echo 'warning: general advice'\n" + + "exit 1" + val link = Aapt2Link(fakeAapt2(script), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostics = (result as Aapt2Link.Result.Failed).diagnostics + val located = diagnostics.single { it.severity == Diagnostic.Severity.WARNING && it.file != null } + assertThat(located.file).isEqualTo("res/values/strings.xml") + assertThat(located.line).isEqualTo(4) + assertThat(located.message).isEqualTo("dubious value") + val unlocated = diagnostics.single { it.severity == Diagnostic.Severity.WARNING && it.file == null } + assertThat(unlocated.message).isEqualTo("general advice") + // aapt2 failed but reported no ERROR line: the fallback must supply one, or the + // client would render a "failed" response containing only warnings. + val errors = diagnostics.filter { it.severity == Diagnostic.Severity.ERROR } + assertThat(errors).hasSize(1) + assertThat(errors.single().message).contains("aapt2 compile failed") + } + + @Test + fun `an empty compiled dir that cannot be deleted does not fail the reset`() { + // Only LEFTOVER ENTRIES can leak stale .flat files into the link. An empty + // res-compiled that survives deleteRecursively (read-only parent) is harmless and + // must fall through to the aapt2 run - whose own failure is then the result. + File(workDir, "res-compiled").mkdirs() + check(workDir.setWritable(false)) { "could not make work dir read-only" } + try { + val link = Aapt2Link(fakeAapt2("echo 'error: kaboom'\nexit 1"), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val messages = (result as Aapt2Link.Result.Failed).diagnostics.map { it.message } + assertThat(messages).containsExactly("kaboom") + } finally { + workDir.setWritable(true) + } + } + + @Test + fun `a wedged aapt2 is killed at the timeout instead of hanging the daemon loop`() { + // `exec`, so the sleeping process IS the child: a wrapping shell would leave a + // grandchild holding the stdout pipe open, and the output drain would outlive the kill. + val link = Aapt2Link(fakeAapt2("exec sleep 60"), File(tempDir, "android.jar"), timeoutMillis = 300) + + val startedAt = System.currentTimeMillis() + val result = link.relink(listOf(resDir), manifest, workDir) + val elapsedMillis = System.currentTimeMillis() - startedAt + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostic = (result as Aapt2Link.Result.Failed).diagnostics.single() + assertThat(diagnostic.severity).isEqualTo(Diagnostic.Severity.ERROR) + assertThat(diagnostic.message).contains("timed out") + // The whole point: relink RETURNS, rather than blocking the single-threaded daemon loop + // for the full sleep and leaving ping and shutdown unanswerable. + assertThat(elapsedMillis).isLessThan(30_000L) + } + + @Test + fun `a success without timings encodes as numeric zeros the client reads back as measured`() { + // "0 means unmeasured, never -1 and never a string" is a wire contract, so assert it on + // the wire: the same keys DaemonService.relink writes, through the real encoder, read + // back the way DaemonProcessClient reads them (JSON-number guard, else null). + val success = Aapt2Link.Result.Success(File("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/work/linked-res.apk")) + + val encoded = + ProtocolCodec.encode( + DaemonResponse.ok( + id = 7L, + values = + mapOf( + "resourcesArsc" to success.resourceApk.absolutePath, + "aapt2CompileMillis" to success.compileMillis, + "aapt2LinkMillis" to success.linkMillis, + ), + ), + ) + + val json = JsonParser.parseString(encoded).asJsonObject + val readLong = { key: String -> json.get(key)?.takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isNumber }?.asLong } + assertThat(readLong("aapt2CompileMillis")).isEqualTo(0L) + assertThat(readLong("aapt2LinkMillis")).isEqualTo(0L) + assertThat(json.get("resourcesArsc").asString).endsWith("linked-res.apk") + } +} diff --git a/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkTest.kt b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkTest.kt new file mode 100644 index 0000000000..5a6fff3f87 --- /dev/null +++ b/quickbuild/daemon/src/test/kotlin/org/appdevforall/cotg/quickbuild/daemon/res/Aapt2LinkTest.kt @@ -0,0 +1,483 @@ +package org.appdevforall.cotg.quickbuild.daemon.res + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.cotg.quickbuild.daemon.TestSdk +import org.appdevforall.cotg.quickbuild.protocol.Diagnostic +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIf +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.util.zip.ZipFile + +/** + * The guard is per-method, not per-class: only the tests that actually shell out to aapt2 need a + * real SDK. The argument-assembly and reset-guard tests below run fake or absent binaries, so a + * host without an Android SDK must still execute them. + */ +class Aapt2LinkTest { + @TempDir + lateinit var tempDir: File + + private lateinit var resDir: File + private lateinit var manifest: File + private lateinit var workDir: File + + @BeforeEach + fun setUp() { + resDir = File(tempDir, "res/values").apply { mkdirs() }.parentFile + workDir = File(tempDir, "work").apply { mkdirs() } + manifest = + File(tempDir, "AndroidManifest.xml").apply { + writeText( + """ + + + + + """.trimIndent(), + ) + } + } + + private fun writeStrings(content: String) { + File(resDir, "values/strings.xml").writeText(content) + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `relink produces a resources arsc from a valid res tree`() { + writeStrings( + """ + + + Quick Build Demo + + """.trimIndent(), + ) + val link = Aapt2Link(TestSdk.aapt2()!!, TestSdk.androidJar()!!) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Success::class.java) + val apk = (result as Aapt2Link.Result.Success).resourceApk + assertThat(apk.length()).isGreaterThan(0) + ZipFile(apk).use { zip -> assertThat(zip.getEntry("resources.arsc")).isNotNull() } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `relinked apk carries file-backed resources, not just the arsc table`() { + // A drawable XML has no useful value inside resources.arsc alone - the runtime needs the + // actual zip entry to resolve it. Ship only the table and ANY file-backed resource (even + // one the edit never touched, e.g. an adaptive-icon mipmap XML) fails to resolve on the + // next activity recreate. + File(resDir, "drawable").mkdirs() + File(resDir, "drawable/plain_shape.xml").writeText( + """ + + + """.trimIndent(), + ) + writeStrings( + """ + + + Quick Build Demo + + """.trimIndent(), + ) + val link = Aapt2Link(TestSdk.aapt2()!!, TestSdk.androidJar()!!) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Success::class.java) + val apk = (result as Aapt2Link.Result.Success).resourceApk + ZipFile(apk).use { zip -> + assertThat(zip.getEntry("resources.arsc")).isNotNull() + assertThat(zip.getEntry("res/drawable/plain_shape.xml")).isNotNull() + } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `relink twice in the same work dir succeeds (full recompile each time)`() { + writeStrings( + """ + + + First + + """.trimIndent(), + ) + val link = Aapt2Link(TestSdk.aapt2()!!, TestSdk.androidJar()!!) + assertThat(link.relink(listOf(resDir), manifest, workDir)) + .isInstanceOf(Aapt2Link.Result.Success::class.java) + + writeStrings( + """ + + + Second + + """.trimIndent(), + ) + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Success::class.java) + } + + @Test + fun `a compiled dir that cannot be cleared fails the relink instead of linking stale flat files`() { + // relink globs every .flat in res-compiled, so a leftover a failed deleteRecursively + // leaves behind would be swept into the link as a stale resource. POSIX: deleting a file + // needs write permission on its directory, so a read-only subdir makes the reset fail with + // entries still present. This fails before any aapt2 run, which both lets the binaries be + // fakes and pins the failure to the reset guard rather than a "failed to run" diagnostic. + val stuckDir = File(workDir, "res-compiled/stuck").apply { mkdirs() } + File(stuckDir, "leftover.arsc.flat").writeText("stale") + assertThat(stuckDir.setWritable(false)).isTrue() + try { + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostics = (result as Aapt2Link.Result.Failed).diagnostics + assertThat(diagnostics).isNotEmpty() + assertThat(diagnostics.any { it.severity == Diagnostic.Severity.ERROR }).isTrue() + assertThat(diagnostics.any { it.message.contains("failed to clear compiled-resource dir") }).isTrue() + assertThat(diagnostics.any { it.message.contains(File(workDir, "res-compiled").absolutePath) }).isTrue() + } finally { + stuckDir.setWritable(true) + } + } + + @Test + fun `an uncreatable compiled dir fails the relink with a message naming the dir`() { + // A read-only work dir: nothing to clear (deleteRecursively of a nonexistent path + // reports success), but mkdirs() cannot create res-compiled - so there is no usable + // dir for aapt2 compile to write into. Ignoring the mkdirs() return would let aapt2 + // fail later with a less actionable error. + val readOnlyWorkDir = File(tempDir, "ro-work").apply { mkdirs() } + assertThat(readOnlyWorkDir.setWritable(false)).isTrue() + try { + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, readOnlyWorkDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostics = (result as Aapt2Link.Result.Failed).diagnostics + assertThat(diagnostics.any { it.message.contains("failed to create compiled-resource dir") }).isTrue() + assertThat(diagnostics.any { it.message.contains(File(readOnlyWorkDir, "res-compiled").absolutePath) }).isTrue() + } finally { + readOnlyWorkDir.setWritable(true) + } + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `malformed resource xml fails with error diagnostics, not a throw`() { + writeStrings("unclosed") + val link = Aapt2Link(TestSdk.aapt2()!!, TestSdk.androidJar()!!) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + val diagnostics = (result as Aapt2Link.Result.Failed).diagnostics + assertThat(diagnostics).isNotEmpty() + assertThat(diagnostics.any { it.severity == Diagnostic.Severity.ERROR }).isTrue() + } + + @Test + fun `a missing aapt2 binary fails with a message, not a throw`() { + // Fails in the compile phase, before android.jar is ever named, so a fake jar path keeps + // this runnable on a host with no SDK. + writeStrings("") + val link = Aapt2Link(File(tempDir, "no-such-aapt2"), File(tempDir, "android.jar")) + + val result = link.relink(listOf(resDir), manifest, workDir) + + assertThat(result).isInstanceOf(Aapt2Link.Result.Failed::class.java) + } + + // aapt2's declaration-order type-index assignment shifts when a whole resource TYPE the real + // proxy app build produced (e.g. a library-injected `bool`) is absent from a relink's narrower + // res tree - the manifest, compiled once against the baseline table, then decodes its numeric + // ids against the WRONG type. `--stable-ids` pins ids to the baseline regardless. No real + // toolchain needed: `buildLinkArguments` is pure argument assembly, unlike `relink` itself. + + @Test + fun `link arguments carry --stable-ids when the file exists`() { + val stableIds = File(tempDir, "stableIds.txt").apply { writeText("mipmap:ic_launcher = 0x7f040000") } + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val arguments = + link.buildLinkArguments( + linkedApk = File(workDir, "linked-res.apk"), + manifest = manifest, + flatFiles = emptyList(), + stableIds = stableIds, + ) + + assertThat(arguments).containsAtLeast("--stable-ids", stableIds.absolutePath).inOrder() + } + + @Test + fun `link arguments omit --stable-ids when the file is null`() { + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val arguments = + link.buildLinkArguments( + linkedApk = File(workDir, "linked-res.apk"), + manifest = manifest, + flatFiles = emptyList(), + stableIds = null, + ) + + assertThat(arguments).doesNotContain("--stable-ids") + } + + @Test + fun `link arguments omit --stable-ids when the file does not exist`() { + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val arguments = + link.buildLinkArguments( + linkedApk = File(workDir, "linked-res.apk"), + manifest = manifest, + flatFiles = emptyList(), + stableIds = File(tempDir, "no-such-stableIds.txt"), + ) + + assertThat(arguments).doesNotContain("--stable-ids") + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `relink with a stable-ids mapping keeps a pinned resource at its baseline id`() { + writeStrings( + """ + + + Quick Build Demo + + """.trimIndent(), + ) + val link = Aapt2Link(TestSdk.aapt2()!!, TestSdk.androidJar()!!) + + // Baseline link (no stable-ids): discover the real id aapt2 assigns app_name so this + // test pins it to something ELSE, proving --stable-ids actually overrides the + // default assignment rather than merely matching it by coincidence. + val baselineResult = link.relink(listOf(resDir), manifest, File(workDir, "baseline").apply { mkdirs() }) + assertThat(baselineResult).isInstanceOf(Aapt2Link.Result.Success::class.java) + val baselineId = dumpResourceId((baselineResult as Aapt2Link.Result.Success).resourceApk, "string/app_name") + assertThat(baselineId).isNotNull() + + val pinnedId = "0x7f0199fe" + assertThat(pinnedId).isNotEqualTo(baselineId) + val stableIds = File(tempDir, "stableIds.txt").apply { writeText("demo.quickbuild:string/app_name = $pinnedId") } + + val pinnedWorkDir = File(workDir, "pinned").apply { mkdirs() } + val pinnedResult = link.relink(listOf(resDir), manifest, pinnedWorkDir, stableIds = stableIds) + + assertThat(pinnedResult).isInstanceOf(Aapt2Link.Result.Success::class.java) + val apk = (pinnedResult as Aapt2Link.Result.Success).resourceApk + assertThat(dumpResourceId(apk, "string/app_name")).isEqualTo(pinnedId) + } + + // A relink of the project's own res/ alone can't resolve a resource a dependency AAR provides + // (e.g. Material3's Theme.Material3.DayNight.NoActionBar), so the daemon feeds pre-compiled + // library-resource units back in as `-R` overlays. + + @Test + fun `link arguments carry library resources as -R overlays, ordered before the project's own compile`() { + val libraryResource = File(tempDir, "merged_res/values_values.arsc.flat") + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + val projectFlat = File(tempDir, "compiled/values_strings.arsc.flat") + + val arguments = + link.buildLinkArguments( + linkedApk = File(workDir, "linked-res.apk"), + manifest = manifest, + flatFiles = listOf(projectFlat), + stableIds = null, + libraryResources = listOf(libraryResource), + ) + + // Every resource input is `-R` (no bare positional) - see Aapt2Link's KDoc for why + // bare positional would silently lose to any `-R`, regardless of order. + val rIndices = arguments.withIndex().filter { it.value == "-R" }.map { it.index } + assertThat(rIndices).hasSize(2) + assertThat(arguments[rIndices[0] + 1]).isEqualTo(libraryResource.absolutePath) + assertThat(arguments[rIndices[1] + 1]).isEqualTo(projectFlat.absolutePath) + // The project's own fresh compile must be the LAST -R so it wins on conflict. + assertThat(rIndices[1]).isGreaterThan(rIndices[0]) + } + + @Test + fun `link arguments omit -R for an empty library resources list`() { + val link = Aapt2Link(File(tempDir, "aapt2"), File(tempDir, "android.jar")) + + val arguments = + link.buildLinkArguments( + linkedApk = File(workDir, "linked-res.apk"), + manifest = manifest, + flatFiles = emptyList(), + stableIds = null, + libraryResources = emptyList(), + ) + + assertThat(arguments).doesNotContain("-R") + } + + @Test + @EnabledIf("org.appdevforall.cotg.quickbuild.daemon.TestSdk#aapt2ToolchainAvailable") + fun `relink resolves a dependency-AAR-only style reference via libraryResources`() { + // The project's OWN theme extends a style that ONLY a "library" declares - the + // project's res/ never defines it, reproducing the exact BasicJ failure + // (`style/Theme.Material3.DayNight.NoActionBar ... not found`). + File(tempDir, "AndroidManifestTheme.xml").writeText( + """ + + + + + """.trimIndent(), + ) + writeStrings( + """ + + + Quick Build Demo +