diff --git a/.github/workflows/release-embedded.yml b/.github/workflows/release-embedded.yml index b9e2c06..063bfec 100644 --- a/.github/workflows/release-embedded.yml +++ b/.github/workflows/release-embedded.yml @@ -2,11 +2,21 @@ name: Release Embedded Build on: release: - types: [created] + types: [published] workflow_dispatch: + inputs: + cdn_release_tag: + description: 'Published release tag to upload to R2 (leave empty to test a build)' + required: false + type: string + +permissions: + contents: write jobs: build-and-release: + if: github.event_name == 'release' || !inputs.cdn_release_tag + timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -35,12 +45,17 @@ jobs: cd .. - name: Upload release asset - uses: softprops/action-gh-release@v1 if: github.event_name == 'release' - with: - files: dist-embedded.zip env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + count=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" --jq '[.assets[] | select(.name == "dist-embedded.zip")] | length') + if [ "$count" = 0 ]; then + gh release upload "$RELEASE_TAG" dist-embedded.zip + else + echo "Keeping the existing release archive for reproducible CDN publication." + fi - name: Upload artifact (for workflow_dispatch) if: github.event_name == 'workflow_dispatch' @@ -48,3 +63,47 @@ jobs: with: name: dist-embedded path: dist-embedded.zip + + publish-cdn: + needs: build-and-release + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.build-and-release.result == 'success') || + (github.event_name == 'workflow_dispatch' && inputs.cdn_release_tag && needs.build-and-release.result == 'skipped')) + runs-on: ubuntu-latest + timeout-minutes: 30 + concurrency: + group: cdn-sqlide-${{ github.event.release.tag_name || inputs.cdn_release_tag }} + cancel-in-progress: false + permissions: + contents: read + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_R2_CDN }} + CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.cdn_release_tag }} + WRANGLER_SEND_METRICS: 'false' + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.ref || github.event.repository.default_branch }} + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + - run: npm ci + - run: npm run test:cdn + - name: Install Wrangler + run: npm install --global wrangler@4.149.0 + - name: Check Cloudflare configuration + run: | + test -n "$CLOUDFLARE_API_TOKEN" || { echo 'Missing CLOUDFLARE_R2_CDN Actions secret'; exit 1; } + test -n "$CLOUDFLARE_ACCOUNT_ID" || { echo 'Missing CLOUDFLARE_ACCOUNT_ID Actions variable or secret'; exit 1; } + - name: Download and verify the published release + run: node scripts/fetch-cdn-release.mjs sqlide "$RELEASE_TAG" + - name: Allow browser access to runtime files + run: wrangler r2 bucket cors set cdn --file scripts/cdn-cors.json --force + - name: Publish immutable runtime files + run: node scripts/publish-cdn.mjs sqlide "$RELEASE_TAG" .cache/cdn-release + - name: Verify files through the public CDN + run: node scripts/verify-cdn.mjs sqlide "$RELEASE_TAG" .cache/cdn-release diff --git a/.github/workflows/test-cdn.yml b/.github/workflows/test-cdn.yml new file mode 100644 index 0000000..9d7b2aa --- /dev/null +++ b/.github/workflows/test-cdn.yml @@ -0,0 +1,18 @@ +name: Validate CDN publishing +on: + pull_request: + push: + branches: [master] +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + - run: npm ci + - run: npm run test:cdn diff --git a/.gitignore b/.gitignore index df97b93..b801ec8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ pnpm-debug.log* lerna-debug.log* node_modules +/.cache/cdn-release/ dist dist-embedded dist-ssr @@ -28,4 +29,4 @@ dist-ssr /upload_csg.cmd /package-lock.json -/intern/ \ No newline at end of file +/intern/ diff --git a/docs/cdn-releases.md b/docs/cdn-releases.md new file mode 100644 index 0000000..55996d0 --- /dev/null +++ b/docs/cdn-releases.md @@ -0,0 +1,48 @@ +# Embedded releases on Cloudflare R2 + +Publishing a GitHub release builds and attaches `dist-embedded.zip`, then uploads +that attached ZIP to the R2 bucket `cdn` under `sqlide//include/`. +The release workflow downloads the attached archive and verifies GitHub's SHA-256 +digest, keeping the CDN identical to the downloadable release. It excludes HTML +examples and source maps. An existing attached ZIP is retained on reruns. + +Configure organization Actions secret `CLOUDFLARE_R2_CDN` with a Cloudflare API +token with **Account → Workers R2 Storage → Edit** (R2 **Admin Read & Write**) +for the account holding `cdn`. Bucket-scoped **Object Read & Write** permissions +support only the S3 API, so Wrangler needs the account permission; see +[Cloudflare's documentation](https://developers.cloudflare.com/r2/api/tokens/#permissions). +Set `CLOUDFLARE_ACCOUNT_ID` as an organization Actions variable or secret. Make +both available to this repository. Wrangler uses the API token directly; S3 +credentials are unnecessary. + +Uploads set correct MIME types and immutable year-long caching. They preserve +older versions, reject different content at an existing key and safely resume +interrupted uploads. `manifest.json` is written last. The public CDN is checked +for every file's checksum, MIME type, caching and CORS before the job succeeds. +The CORS policy permits public GET/HEAD requests from any origin. + +To seed an existing release or retry it, use **Actions → Release Embedded Build → +Run workflow**, entering its published tag in `cdn_release_tag`, or: + +```sh +gh workflow run release-embedded.yml --repo openpatch/sql-ide -f cdn_release_tag=v2.0.0-hyperbook.4 +``` + +Leave that input empty to run the existing build/artifact validation without +publishing. Use this immutable base URL in Hyperbook: + +```json +{ + "elements": { + "sqlide": { "cdn": "https://cdn.openpatch.org/sqlide/v2.0.0-hyperbook.4/" } + } +} +``` + +The actual entry point is `include/sql-ide-embedded.js`; preserve the rest of +the `include/` directory for relative workers, chunks, fonts and WASM. +New releases publish themselves. Consumers adopt new versions by updating their +pinned tag. This process does not depend on the Hyperbook Markdown package. + +Run `npm run test:cdn` to test extraction, checksums, immutable publication, +resumable uploads and browser-facing verification locally. diff --git a/package.json b/package.json index b0fd7c4..0cbc660 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "build": "tsc && vite build && vite build --config vite.config-embedded.js", "build-ide": "tsc && vite build", "build-embedded": "tsc && vite build --config vite.config-embedded.js", + "test:cdn": "node --test scripts/publish-cdn.test.mjs scripts/fetch-cdn-release.test.mjs", "preview": "npm run build && vite preview --port 4000 --host", "build_tools": "tsc --project tsconfig_development-tools.json", "deploy": "node src/development/ZipAndUpload.js" diff --git a/scripts/cdn-cors.json b/scripts/cdn-cors.json new file mode 100644 index 0000000..1f31ff5 --- /dev/null +++ b/scripts/cdn-cors.json @@ -0,0 +1,13 @@ +{ + "rules": [ + { + "allowed": { + "origins": ["*"], + "methods": ["GET", "HEAD"], + "headers": ["*"] + }, + "exposeHeaders": ["ETag", "Content-Length"], + "maxAgeSeconds": 86400 + } + ] +} diff --git a/scripts/fetch-cdn-release.mjs b/scripts/fetch-cdn-release.mjs new file mode 100644 index 0000000..c8176af --- /dev/null +++ b/scripts/fetch-cdn-release.mjs @@ -0,0 +1,99 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import JSZip from "jszip"; +import { releasePrefix } from "./publish-cdn.mjs"; + +export async function extractRelease({ bytes, digest, directory, element }) { + if ( + !/^sha256:[a-f0-9]{64}$/.test(digest) || + `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== digest + ) { + throw new Error("GitHub release archive checksum mismatch"); + } + const zip = await JSZip.loadAsync(bytes); + // Check every filename before extracting; JSZip exposes names it sanitizes. + const files = Object.values(zip.files).filter((entry) => !entry.dir); + for (const entry of files) { + const original = entry.unsafeOriginalName || entry.name; + if ( + original !== entry.name || + original.startsWith("/") || + original.includes("\\") || + original.includes("\0") || + original + .split("/") + .some((part) => !part || part === "." || part === "..") || + (entry.unixPermissions & 0o170000) === 0o120000 + ) { + throw new Error(`Unsafe release archive entry: ${original}`); + } + } + const basename = element === "onlineide" ? "online-ide" : "sql-ide"; + for (const extension of ["js", "css"]) { + if (!zip.file(`${basename}-embedded.${extension}`)) + throw new Error(`Missing embedded ${extension} entry point`); + } + await fs.rm(directory, { recursive: true, force: true }); + for (const entry of files) { + if (/\.(map|html)$/i.test(entry.name)) continue; + const destination = path.join(directory, "include", entry.name); + await fs.mkdir(path.dirname(destination), { recursive: true }); + await fs.writeFile(destination, await entry.async("nodebuffer")); + } +} + +export async function fetchRelease({ + element, + version, + repository = process.env.GITHUB_REPOSITORY, + directory = ".cache/cdn-release", + fetchImpl = fetch, +}) { + releasePrefix(element, version); + if (!/^[\w.-]+\/[\w.-]+$/.test(repository)) + throw new Error("Invalid GitHub repository"); + const release = JSON.parse( + execFileSync( + "gh", + ["api", `repos/${repository}/releases/tags/${version}`], + { + encoding: "utf8", + env: { + ...process.env, + GH_TOKEN: process.env.GH_TOKEN || process.env.GITHUB_TOKEN, + }, + }, + ), + ); + if (release.draft) + throw new Error("Cannot publish a draft release to the CDN"); + const asset = release.assets.find( + (asset) => asset.name === "dist-embedded.zip", + ); + if (!asset) throw new Error(`No dist-embedded.zip attached to ${version}`); + const response = await fetchImpl(asset.browser_download_url, { + signal: AbortSignal.timeout(300_000), + }); + if (!response.ok) + throw new Error(`Cannot download release archive: HTTP ${response.status}`); + await extractRelease({ + bytes: Buffer.from(await response.arrayBuffer()), + digest: asset.digest, + directory, + element, + }); + console.log( + `Verified and extracted ${repository}@${version} into ${directory}`, + ); +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const [element, version] = process.argv.slice(2); + await fetchRelease({ element, version }); +} diff --git a/scripts/fetch-cdn-release.test.mjs b/scripts/fetch-cdn-release.test.mjs new file mode 100644 index 0000000..23c536b --- /dev/null +++ b/scripts/fetch-cdn-release.test.mjs @@ -0,0 +1,101 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import JSZip from "jszip"; +import { extractRelease } from "./fetch-cdn-release.mjs"; + +function archive(element = "onlineide") { + const zip = new JSZip(); + const name = element === "onlineide" ? "online-ide" : "sql-ide"; + zip.file(`${name}-embedded.js`, "export const ready = true;"); + zip.file(`${name}-embedded.css`, "body {}"); + zip.file("assets/worker.js", "self.onmessage = () => {};"); + zip.file("assets/runtime.wasm", Buffer.from([0, 97, 115, 109])); + zip.file("assets/worker.js.map", "source map"); + zip.file("embedded_java.html", "example"); + return zip; +} + +async function fixture(t, zip) { + const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "cdn-archive-")); + t.after(() => fs.rm(temporary, { recursive: true, force: true })); + const bytes = await zip.generateAsync({ + type: "nodebuffer", + platform: "UNIX", + }); + return { + bytes, + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + directory: path.join(temporary, "runtime"), + }; +} + +test("extracts both IDE entry points, workers and WASM under include, excluding examples and maps", async (t) => { + for (const element of ["onlineide", "sqlide"]) { + const f = await fixture(t, archive(element)); + await extractRelease({ ...f, element }); + assert.equal( + await fs.readFile( + path.join(f.directory, "include/assets/worker.js"), + "utf8", + ), + "self.onmessage = () => {};", + ); + assert.deepEqual( + await fs.readFile(path.join(f.directory, "include/assets/runtime.wasm")), + Buffer.from([0, 97, 115, 109]), + ); + await assert.rejects( + fs.access(path.join(f.directory, "include/assets/worker.js.map")), + ); + await assert.rejects( + fs.access(path.join(f.directory, "include/embedded_java.html")), + ); + } +}); + +test("rejects a checksum mismatch before extraction", async (t) => { + const f = await fixture(t, archive()); + await assert.rejects( + extractRelease({ + ...f, + digest: `sha256:${"0".repeat(64)}`, + element: "onlineide", + }), + /checksum/, + ); + await assert.rejects(fs.access(f.directory)); +}); + +test("rejects traversal, absolute paths and symlinks before extracting any files", async (t) => { + for (const name of ["../outside.js", "/outside.js", "assets\\outside.js"]) { + const zip = archive(); + zip.file(name, "unsafe"); + const f = await fixture(t, zip); + await assert.rejects( + extractRelease({ ...f, element: "onlineide" }), + /Unsafe/, + ); + await assert.rejects(fs.access(f.directory)); + } + const zip = archive(); + zip.file("link", "../outside", { unixPermissions: 0o120777 }); + const f = await fixture(t, zip); + await assert.rejects( + extractRelease({ ...f, element: "onlineide" }), + /Unsafe/, + ); +}); + +test("rejects an archive without its embedded entry point", async (t) => { + const zip = archive(); + zip.remove("online-ide-embedded.js"); + const f = await fixture(t, zip); + await assert.rejects( + extractRelease({ ...f, element: "onlineide" }), + /Missing embedded js/, + ); +}); diff --git a/scripts/publish-cdn.mjs b/scripts/publish-cdn.mjs new file mode 100644 index 0000000..7b15df3 --- /dev/null +++ b/scripts/publish-cdn.mjs @@ -0,0 +1,192 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +// Kept in each publishing repository so releases do not depend on another repo. +const run = promisify(execFile); +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const cacheControl = "public, max-age=31536000, immutable"; +const contentTypes = { + ".js": "text/javascript", + ".mjs": "text/javascript", + ".css": "text/css", + ".wasm": "application/wasm", + ".json": "application/json", + ".zip": "application/zip", + ".svg": "image/svg+xml", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".ico": "image/x-icon", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".ttf": "font/ttf", + ".otf": "font/otf", + ".mp3": "audio/mpeg", + ".ogg": "audio/ogg", + ".wav": "audio/wav", + ".txt": "text/plain", +}; + +export function releasePrefix(element, version) { + if ( + !["onlineide", "sqlide", "openscad"].includes(element) || + !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(version) || + version === "latest" + ) { + throw new Error( + "Expected a supported element and an immutable release version", + ); + } + return `${element}/${version}`; +} + +export async function runtimeManifest(directory) { + const files = {}; + async function visit(folder, prefix = "") { + const entries = await fs.readdir(folder, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name, "en")); + for (const entry of entries) { + const relative = `${prefix}${entry.name}`; + if (entry.isSymbolicLink()) + throw new Error(`Runtime contains a symlink: ${relative}`); + if (entry.isDirectory()) + await visit(path.join(folder, entry.name), `${relative}/`); + else if (entry.isFile() && !/\.(map|html)$/i.test(relative)) { + if (relative === "manifest.json") + throw new Error("manifest.json is reserved for publication metadata"); + const bytes = await fs.readFile(path.join(folder, entry.name)); + files[relative] = { + sha256: hash(bytes), + bytes: bytes.length, + contentType: + contentTypes[path.extname(relative).toLowerCase()] || + "application/octet-stream", + }; + } + } + } + await visit(directory); + if (!Object.keys(files).length) + throw new Error("No runtime files to publish"); + return { schema: 1, files }; +} + +export function wranglerStore(bucket = "cdn", runImpl = run) { + return { + async get(key) { + const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "cdn-read-")); + const file = path.join(temporary, "object"); + try { + await runImpl("wrangler", [ + "r2", + "object", + "get", + `${bucket}/${key}`, + "--remote", + "--file", + file, + ]); + return await fs.readFile(file); + } catch (error) { + if ( + /The specified key does not exist/.test( + `${error.stdout}\n${error.stderr}`, + ) + ) + return null; + throw error; + } finally { + await fs.rm(temporary, { recursive: true, force: true }); + } + }, + async put(key, file, contentType) { + await runImpl("wrangler", [ + "r2", + "object", + "put", + `${bucket}/${key}`, + "--remote", + "--file", + file, + "--content-type", + contentType, + "--cache-control", + cacheControl, + ]); + }, + }; +} + +export async function publishCdn({ + directory, + element, + version, + store = wranglerStore(), +}) { + const prefix = releasePrefix(element, version); + const manifest = await runtimeManifest(directory); + const bytes = Buffer.from(JSON.stringify(manifest, null, 2) + "\n"); + const manifestKey = `${prefix}/manifest.json`; + const previous = await store.get(manifestKey); + if (previous) { + if (!bytes.equals(previous)) + throw new Error(`Published runtime is immutable: ${prefix}`); + console.log(`${prefix} is already published and matches the release`); + return manifest; + } + // A failed upload can be resumed. Existing objects must match before reuse. + // The manifest is written last, after all payload objects have been uploaded. + const files = Object.entries(manifest.files); + let next = 0; + const uploads = await Promise.allSettled( + Array.from({ length: Math.min(4, files.length) }, async () => { + while (next < files.length) { + const [relative, metadata] = files[next++]; + const key = `${prefix}/${relative}`; + const previous = await store.get(key); + if (previous) { + if (hash(previous) !== metadata.sha256) + throw new Error(`Existing runtime object differs: ${key}`); + } else { + const file = path.join(directory, relative); + if (hash(await fs.readFile(file)) !== metadata.sha256) + throw new Error(`Runtime changed during upload: ${relative}`); + await store.put(key, file, metadata.contentType); + } + } + }), + ); + const failed = uploads.find((result) => result.status === "rejected"); + if (failed) throw failed.reason; + const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "cdn-manifest-")); + try { + const file = path.join(temporary, "manifest.json"); + await fs.writeFile(file, bytes); + await store.put(manifestKey, file, "application/json"); + } finally { + await fs.rm(temporary, { recursive: true, force: true }); + } + console.log( + `Published ${files.length} files at https://cdn.openpatch.org/${prefix}/`, + ); + return manifest; +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const [element, version, directory] = process.argv.slice(2); + if (!directory) + throw new Error( + "Usage: node scripts/publish-cdn.mjs ", + ); + await publishCdn({ directory, element, version }); +} diff --git a/scripts/publish-cdn.test.mjs b/scripts/publish-cdn.test.mjs new file mode 100644 index 0000000..2fa0d82 --- /dev/null +++ b/scripts/publish-cdn.test.mjs @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + publishCdn, + releasePrefix, + runtimeManifest, + wranglerStore, +} from "./publish-cdn.mjs"; +import { verifyCdn } from "./verify-cdn.mjs"; + +async function fixture(t) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "cdn-test-")); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + await fs.mkdir(path.join(directory, "include", "assets"), { + recursive: true, + }); + await fs.writeFile( + path.join(directory, "include", "online-ide-embedded.js"), + "export const version = 1;", + ); + await fs.writeFile( + path.join(directory, "include", "assets", "runtime.wasm"), + Buffer.from([0, 97, 115, 109]), + ); + await fs.writeFile( + path.join(directory, "include", "example.html"), + "example", + ); + await fs.writeFile( + path.join(directory, "include", "bundle.js.map"), + "source map", + ); + const objects = new Map([ + ["sqlide/v1/unchanged.js", Buffer.from("existing SQL release")], + ]); + const writes = []; + const store = { + get: async (key) => objects.get(key) || null, + async put(key, file, type) { + writes.push({ key, type }); + objects.set(key, await fs.readFile(file)); + }, + }; + return { + directory, + objects, + writes, + store, + element: "onlineide", + version: "v1", + }; +} + +test("publishes workers/WASM with correct MIME types, skips maps and HTML, and marks completion last", async (t) => { + const f = await fixture(t); + const manifest = await publishCdn(f); + assert.equal(f.writes.at(-1).key, "onlineide/v1/manifest.json"); + assert.equal( + f.writes.find((write) => write.key.endsWith(".js")).type, + "text/javascript", + ); + assert.equal( + f.writes.find((write) => write.key.endsWith(".wasm")).type, + "application/wasm", + ); + assert.equal(Object.keys(manifest.files).length, 2); + assert.equal( + manifest.files["include/assets/runtime.wasm"].sha256, + createHash("sha256") + .update(Buffer.from([0, 97, 115, 109])) + .digest("hex"), + ); + assert.equal( + f.objects.get("sqlide/v1/unchanged.js").toString(), + "existing SQL release", + ); +}); + +test("retrying a completed release performs no writes; changed releases cannot replace it", async (t) => { + const f = await fixture(t); + await publishCdn(f); + const writes = f.writes.length; + await publishCdn(f); + assert.equal(f.writes.length, writes); + await fs.writeFile( + path.join(f.directory, "include", "online-ide-embedded.js"), + "changed", + ); + await assert.rejects(publishCdn(f), /immutable/); + assert.equal(f.writes.length, writes); +}); + +test("failed publication leaves no completion marker and can resume existing payload files", async (t) => { + const f = await fixture(t); + const put = f.store.put; + f.store.put = async (key, ...args) => { + if (key.endsWith(".wasm")) throw new Error("Upload failed"); + await put(key, ...args); + }; + await assert.rejects(publishCdn(f), /Upload failed/); + assert.equal(f.objects.has("onlineide/v1/manifest.json"), false); + f.store.put = put; + await publishCdn(f); + assert.equal(f.objects.has("onlineide/v1/manifest.json"), true); +}); + +test("a conflicting object in a partial release is never overwritten", async (t) => { + const f = await fixture(t); + f.objects.set( + "onlineide/v1/include/assets/runtime.wasm", + Buffer.from("different"), + ); + await assert.rejects(publishCdn(f), /Existing runtime object differs/); + assert.equal( + f.objects.get("onlineide/v1/include/assets/runtime.wasm").toString(), + "different", + ); + assert.equal(f.objects.has("onlineide/v1/manifest.json"), false); +}); + +test("new versions retain all objects from prior versions", async (t) => { + const f = await fixture(t); + await publishCdn(f); + const previous = new Map(f.objects); + await publishCdn({ ...f, version: "v2" }); + for (const [key, value] of previous) + assert.deepEqual(f.objects.get(key), value); +}); + +test("unsafe namespaces, empty runtimes, reserved metadata and symlinks are rejected", async (t) => { + for (const version of ["../x", "/x", "..", "latest", "v1?x", "v1/x"]) { + assert.throws(() => releasePrefix("onlineide", version)); + } + assert.throws(() => releasePrefix("unknown", "v1")); + const f = await fixture(t); + const empty = path.join(f.directory, "empty"); + await fs.mkdir(empty); + await assert.rejects(runtimeManifest(empty), /No runtime files/); + await fs.writeFile(path.join(f.directory, "manifest.json"), "reserved"); + await assert.rejects(runtimeManifest(f.directory), /reserved/); + await fs.rm(path.join(f.directory, "manifest.json")); + await fs.symlink( + "include/assets/runtime.wasm", + path.join(f.directory, "link"), + ); + await assert.rejects(runtimeManifest(f.directory), /symlink/); +}); + +test("Wrangler uploads to remote R2 with immutable caching and treats only missing keys as absent", async () => { + const calls = []; + const store = wranglerStore("cdn", async (command, args) => { + calls.push({ command, args }); + }); + await store.put( + "onlineide/v1/include/worker.js", + "/tmp/worker.js", + "text/javascript", + ); + assert.equal(calls[0].command, "wrangler"); + assert.deepEqual(calls[0].args, [ + "r2", + "object", + "put", + "cdn/onlineide/v1/include/worker.js", + "--remote", + "--file", + "/tmp/worker.js", + "--content-type", + "text/javascript", + "--cache-control", + "public, max-age=31536000, immutable", + ]); + const missing = wranglerStore("cdn", async () => { + throw { stderr: "The specified key does not exist." }; + }); + assert.equal(await missing.get("absent"), null); + const denied = wranglerStore("cdn", async () => { + throw new Error("Authentication failed"); + }); + await assert.rejects(denied.get("absent"), /Authentication failed/); +}); + +test("public verification checks all payloads and fails for bad CORS, cache, MIME types and checksums", async (t) => { + const f = await fixture(t); + const manifest = await publishCdn(f); + let fault; + const fetchImpl = async (url, options) => { + assert.equal(options.headers.Origin, "https://hyperbook.openpatch.org"); + const key = new URL(url).pathname.slice(1); + const relative = key.slice("onlineide/v1/".length); + const headers = { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "public, max-age=31536000, immutable", + "Content-Type": + manifest.files[relative]?.contentType || "application/json", + }; + if (fault === "CORS") delete headers["Access-Control-Allow-Origin"]; + if (fault === "Cache-Control") delete headers["Cache-Control"]; + if (fault === "Content-Type" && relative !== "manifest.json") + headers["Content-Type"] = "text/plain"; + return new Response( + fault === "checksum" && relative !== "manifest.json" + ? "corrupt" + : f.objects.get(key), + { headers }, + ); + }; + await verifyCdn({ ...f, fetchImpl }); + for (fault of ["CORS", "Cache-Control", "Content-Type", "checksum"]) { + await assert.rejects( + verifyCdn({ ...f, fetchImpl }), + /CORS|Cache-Control|Content-Type|checksum/, + ); + } +}); diff --git a/scripts/verify-cdn.mjs b/scripts/verify-cdn.mjs new file mode 100644 index 0000000..6ede7d0 --- /dev/null +++ b/scripts/verify-cdn.mjs @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { releasePrefix, runtimeManifest } from "./publish-cdn.mjs"; + +// Read through the public hostname, exercising the path browsers actually use. +export async function verifyCdn({ + element, + version, + directory, + fetchImpl = fetch, +}) { + const base = `https://cdn.openpatch.org/${releasePrefix(element, version)}/`; + const expected = await runtimeManifest(directory); + async function get(relative) { + const response = await fetchImpl(new URL(relative, base), { + headers: { Origin: "https://hyperbook.openpatch.org" }, + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) throw new Error(`${relative}: HTTP ${response.status}`); + if (response.headers.get("access-control-allow-origin") !== "*") { + throw new Error(`${relative}: missing public CORS headers`); + } + if (!response.headers.get("cache-control")?.includes("immutable")) { + throw new Error(`${relative}: missing immutable Cache-Control`); + } + return response; + } + const manifest = await (await get("manifest.json")).json(); + if (JSON.stringify(manifest) !== JSON.stringify(expected)) + throw new Error("Public CDN manifest differs from the release"); + const files = Object.entries(expected.files); + let next = 0; + const results = await Promise.allSettled( + Array.from({ length: Math.min(4, files.length) }, async () => { + while (next < files.length) { + const [relative, metadata] = files[next++]; + const response = await get(relative); + if ( + response.headers.get("content-type")?.split(";")[0] !== + metadata.contentType + ) { + throw new Error(`${relative}: incorrect Content-Type`); + } + const bytes = Buffer.from(await response.arrayBuffer()); + if ( + bytes.length !== metadata.bytes || + createHash("sha256").update(bytes).digest("hex") !== metadata.sha256 + ) { + throw new Error(`${relative}: public CDN checksum mismatch`); + } + } + }), + ); + const failed = results.find((result) => result.status === "rejected"); + if (failed) throw failed.reason; + console.log( + `Verified ${files.length} public CDN files, CORS, MIME types and immutable caching at ${base}`, + ); +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const [element, version, directory] = process.argv.slice(2); + await verifyCdn({ element, version, directory }); +}