ci: add compile and runtime benchmark jobs

.github/workflows/main.yml

@@ -70,3 +70,19 @@ jobs:
     name: WebdriverIO Tests
     needs: [build_core]
     uses: ./.github/workflows/test-wdio.yml
+
+  perf_build_benchmark:
+    name: Perf Benchmark - Build
+    needs: [build_core]
+    uses: ./.github/workflows/test-perf-build.yml
+    permissions:
+      contents: read
+      id-token: write
+
+  perf_runtime_benchmark:
+    name: Perf Benchmark - Runtime
+    needs: [build_core]
+    uses: ./.github/workflows/test-perf-runtime.yml
+    permissions:
+      contents: read
+      pull-requests: write

.github/workflows/perf-nightly.yml

@@ -0,0 +1,47 @@
+name: Nightly Runtime Perf Baseline
+
+on:
+  schedule:
+    - cron: '0 6 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  record_baseline:
+    name: Record Runtime Benchmark Baseline
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Get Core Dependencies
+        uses: ./.github/workflows/actions/get-core-dependencies
+
+      - name: Build Stencil Core
+        run: npm run build -- --ci
+        shell: bash
+
+      - name: Install test/runtime-benchmark Dependencies
+        run: npm ci
+        working-directory: test/runtime-benchmark
+        shell: bash
+
+      - name: Run Runtime Benchmark
+        run: npm run benchmark
+        working-directory: test/runtime-benchmark
+        shell: bash
+
+      - name: Commit Updated Baseline
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add test/runtime-benchmark/benchmark-results.json test/runtime-benchmark/benchmark-results.md
+          if git diff --cached --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "chore(perf): update runtime benchmark baseline"
+            git push origin HEAD:main
+          fi
+        shell: bash

.github/workflows/test-perf-build.yml

@@ -0,0 +1,63 @@
+name: Perf Benchmark - Build
+
+on:
+  workflow_call:
+    # Make this a reusable workflow, no value needed
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows
+
+permissions:
+  contents: read
+
+jobs:
+  perf_build_warm:
+    name: Compile Time (warm)
+    runs-on: codspeed-macro
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Get Core Dependencies
+        uses: ./.github/workflows/actions/get-core-dependencies
+
+      - name: Download Build Archive
+        uses: ./.github/workflows/actions/download-archive
+        with:
+          name: stencil-core
+          path: .
+          filename: stencil-core-build.zip
+
+      - name: Install test/end-to-end Dependencies
+        run: npm ci
+        working-directory: test/end-to-end
+        shell: bash
+
+      - name: Warm Compile Cache
+        run: |
+          rm -rf .stencil
+          node ../../bin/stencil build --config ./stencil.build.config.ts
+        working-directory: test/end-to-end
+        shell: bash
+
+      - name: Write CodSpeed Benchmark Config
+        run: |
+          cat > codspeed.yml <<'EOF'
+          benchmarks:
+            - name: Compile Time (warm)
+              exec: node ${{ github.workspace }}/bin/stencil build --config ${{ github.workspace }}/test/end-to-end/stencil.build.config.ts
+              options:
+                warmup-time: 0s
+                min-rounds: 3
+                max-rounds: 5
+                max-time: 180s
+          EOF
+        working-directory: test/end-to-end
+        shell: bash
+
+      - name: Run Warm Build Benchmark
+        uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3
+        with:
+          mode: walltime
+          config: ${{ github.workspace }}/test/end-to-end/codspeed.yml

.github/workflows/test-perf-runtime.yml

@@ -0,0 +1,90 @@
+name: Perf Benchmark - Runtime
+
+on:
+  workflow_call:
+    # Make this a reusable workflow, no value needed
+    # https://docs.github.com/en/actions/using-workflows/reusing-workflows
+
+permissions:
+  contents: read
+
+jobs:
+  perf_runtime_diff:
+    name: Runtime Benchmark (PR vs base)
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Checkout PR Head
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Get Core Dependencies
+        uses: ./.github/workflows/actions/get-core-dependencies
+
+      - name: Download Build Archive
+        uses: ./.github/workflows/actions/download-archive
+        with:
+          name: stencil-core
+          path: .
+          filename: stencil-core-build.zip
+
+      - name: Install PR test/runtime-benchmark Dependencies
+        run: npm ci
+        working-directory: test/runtime-benchmark
+        shell: bash
+
+      - name: Run PR Runtime Benchmark
+        run: npm run benchmark
+        working-directory: test/runtime-benchmark
+        shell: bash
+
+      - name: Convert PR Results to Raw Format
+        run: node to-codspeed-raw.js "$GITHUB_WORKSPACE/pr-raw.json"
+        working-directory: test/runtime-benchmark
+        shell: bash
+
+      - name: Checkout Base Branch
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+
+      - name: Install Base Dependencies
+        run: |
+          npm ci
+          npm run install.jest
+        working-directory: base
+        shell: bash
+
+      - name: Build Base Stencil Core
+        run: npm run build -- --ci
+        working-directory: base
+        shell: bash
+
+      - name: Install Base test/runtime-benchmark Dependencies
+        run: npm ci
+        working-directory: base/test/runtime-benchmark
+        shell: bash
+
+      - name: Run Base Runtime Benchmark
+        run: npm run benchmark
+        working-directory: base/test/runtime-benchmark
+        shell: bash
+
+      - name: Convert Base Results to Raw Format
+        run: node to-codspeed-raw.js "$GITHUB_WORKSPACE/base-raw.json"
+        working-directory: base/test/runtime-benchmark
+        shell: bash
+
+      - name: Compare Runtime Benchmark (PR vs base)
+        uses: openpgpjs/github-action-pull-request-benchmark@11e6f9c49191a57127f53dddcbdcc30b18f7f0a3 # v1.4.0
+        with:
+          name: 'Runtime Benchmark'
+          tool: 'raw'
+          pr-benchmark-file-path: pr-raw.json
+          base-benchmark-file-path: base-raw.json
+          comment-on-alert: true
+          alert-threshold: '120%'
+          github-token: ${{ secrets.GITHUB_TOKEN }}

test/runtime-benchmark/benchmark-runtime.js

@@ -199,7 +199,7 @@ async function main() {
   const { server, port } = await startServer();
 
   // Launch browser
-  const browser = await puppeteer.launch({ headless: true });
+  const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });
   const page = await browser.newPage();
 
   try {

test/runtime-benchmark/to-codspeed-raw.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+const path = require('path');
+
+const RESULTS_FILE = path.join(__dirname, 'benchmark-results.json');
+const OUTPUT_FILE = process.argv[2] || path.join(__dirname, 'benchmark-raw.json');
+
+const BENCHMARK_LABELS = {
+  create1k: 'Create 1,000 rows',
+  replace1k: 'Replace 1,000 rows',
+  update: 'Update every 10th row',
+  swap: 'Swap rows',
+  select: 'Select row',
+  remove: 'Remove row',
+  create10k: 'Create 10,000 rows',
+  append: 'Append 1,000 rows',
+  clear: 'Clear rows',
+};
+
+const { latest } = JSON.parse(fs.readFileSync(RESULTS_FILE, 'utf-8'));
+
+const raw = Object.entries(latest.benchmarks).map(([name, stats]) => ({
+  name: BENCHMARK_LABELS[name] || name,
+  unit: 'ms',
+  value: stats.avg,
+  range: stats.stddev,
+  biggerIsBetter: false,
+}));
+
+fs.writeFileSync(OUTPUT_FILE, JSON.stringify(raw, null, 2));
+console.log(`Wrote raw benchmark results to ${OUTPUT_FILE}`);