QPY-83: compare QuickPython with CPython

benchmarks/talk_comparison/README.md

@@ -0,0 +1,26 @@
+# QuickPython vs CPython talk benchmark
+
+This is the one-off, process-level benchmark used for QPY-83. It is separate
+from the long-term suite proposed in `spec/benchmarks.md`.
+
+The harness builds QuickPython once in release mode, generates each workload
+from the committed templates and fixture, and then invokes the resulting
+binary directly. CPython and QuickPython receive the exact same generated
+source file. Non-cold workloads run three times and report the minimum
+wall-clock time divided by the fixed iteration count. Cold start uses twenty
+fresh processes and reports minimum and average wall-clock time.
+
+From the repository root, run:
+
+```sh
+python3 benchmarks/talk_comparison/run.py
+```
+
+The default output files are `raw-results.json` and `results.md` in this
+directory. A single workload process is limited to five minutes by default;
+use `--timeout SECONDS` to change that. A timeout or unsupported operation is
+reported as a failure, never as a timing result. The harness does not invoke a
+container engine.
+
+The checked-in result files are a snapshot from the machine described in
+`results.md`. Re-running replaces them with a new same-machine comparison.

benchmarks/talk_comparison/fixtures/medium_response.json

@@ -0,0 +1,93 @@
+{
+  "person": {
+    "id": "d50887ca-a6ce-4e59-b89f-14f0b5d03b03",
+    "name": {
+      "fullName": "Leonid Bugaev",
+      "givenName": "Leonid",
+      "familyName": "Bugaev"
+    },
+    "email": "leonsbox@gmail.com",
+    "gender": "male",
+    "location": "Saint Petersburg, Saint Petersburg, RU",
+    "geo": {
+      "city": "Saint Petersburg",
+      "state": "Saint Petersburg",
+      "country": "Russia",
+      "lat": 59.9342802,
+      "lng": 30.3350986
+    },
+    "bio": "Senior engineer at Granify.com",
+    "site": "http://flickfaver.com",
+    "avatar": "https://d1ts43dypk8bqh.cloudfront.net/v1/avatars/d50887ca-a6ce-4e59-b89f-14f0b5d03b03",
+    "employment": {
+      "name": "www.latera.ru",
+      "title": "Software Engineer",
+      "domain": "gmail.com"
+    },
+    "facebook": {
+      "handle": "leonid.bugaev"
+    },
+    "github": {
+      "handle": "buger",
+      "id": 14009,
+      "avatar": "https://avatars.githubusercontent.com/u/14009?v=3",
+      "company": "Granify",
+      "blog": "http://leonsbox.com",
+      "followers": 95,
+      "following": 10
+    },
+    "twitter": {
+      "handle": "flickfaver",
+      "id": 77004410,
+      "bio": null,
+      "followers": 2,
+      "following": 1,
+      "statuses": 5,
+      "favorites": 0,
+      "location": "",
+      "site": "http://flickfaver.com",
+      "avatar": null
+    },
+    "linkedin": {
+      "handle": "in/leonidbugaev"
+    },
+    "googleplus": {
+      "handle": null
+    },
+    "angellist": {
+      "handle": "leonid-bugaev",
+      "id": 61541,
+      "bio": "Senior engineer at Granify.com",
+      "blog": "http://buger.github.com",
+      "site": "http://buger.github.com",
+      "followers": 41,
+      "avatar": "https://d1qb2nb5cznatu.cloudfront.net/users/61541-medium_jpg?1405474390"
+    },
+    "klout": {
+      "handle": null,
+      "score": null
+    },
+    "foursquare": {
+      "handle": null
+    },
+    "aboutme": {
+      "handle": "leonid.bugaev",
+      "bio": null,
+      "avatar": null
+    },
+    "gravatar": {
+      "handle": "buger",
+      "urls": [
+      ],
+      "avatar": "http://1.gravatar.com/avatar/f7c8edd577d13b8930d5522f28123510",
+      "avatars": [
+        {
+          "url": "http://1.gravatar.com/avatar/f7c8edd577d13b8930d5522f28123510",
+          "type": "thumbnail"
+        }
+      ]
+    },
+    "fuzzy": false
+  },
+  "company": null
+}

benchmarks/talk_comparison/raw-results.json

@@ -0,0 +1,213 @@
+{
+  "schema_version": 1,
+  "environment": {
+    "os_arch": "Linux-7.0.11-orbstack-00354-gde2ab3f9e944-aarch64-with-glibc2.41",
+    "cpu": "unavailable",
+    "quickpython_commit": "a8168c3432111fd341c581ffacb798351054650f",
+    "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)",
+    "cargo": "cargo 1.98.1 (797e8a9bc 2026-08-05)",
+    "cpython": "Python 3.11.16"
+  },
+  "commands": [
+    "cargo build --release",
+    "target/release/quickpython run <generated-or-committed-workload.py>",
+    "python3 <generated-or-committed-workload.py>"
+  ],
+  "measurement": {
+    "non_cold_runs": 3,
+    "cold_runs": 20,
+    "timeout_seconds": 300.0,
+    "clock": "time.perf_counter_ns"
+  },
+  "results": {
+    "quickpython": {
+      "cold_start": {
+        "status": "success",
+        "samples_seconds": [
+          0.002484589,
+          0.000816751,
+          0.002415254,
+          0.000524543,
+          0.002312463,
+          0.002449422,
+          0.002501796,
+          0.000497917,
+          0.002567713,
+          0.00213592,
+          0.001992963,
+          0.002017962,
+          0.000503543,
+          0.002374129,
+          0.002101588,
+          0.000490793,
+          0.002319255,
+          0.002184629,
+          0.000510917,
+          0.002370921
+        ],
+        "min_ms": 0.490793,
+        "avg_ms": 1.7786534,
+        "checksum": "2"
+      },
+      "json_loads": {
+        "status": "success",
+        "iterations": 200000,
+        "samples_seconds": [
+          3.180023342,
+          3.335013011,
+          3.26884951
+        ],
+        "min_us_per_iter": 15.90011671,
+        "checksum": "d50887ca-a6ce-4e59-b89f-14f0b5d03b03"
+      },
+      "json_dumps": {
+        "status": "failed",
+        "iterations": 200000,
+        "stage": "runtime",
+        "category": "semantic difference",
+        "error": "expected stdout '1815', got '1685'",
+        "samples": [
+          {
+            "elapsed_seconds": 1.184112877,
+            "returncode": 0,
+            "stdout": "1685\n",
+            "stderr": "",
+            "timed_out": false
+          },
+          {
+            "elapsed_seconds": 1.216834148,
+            "returncode": 0,
+            "stdout": "1685\n",
+            "stderr": "",
+            "timed_out": false
+          },
+          {
+            "elapsed_seconds": 1.24416445,
+            "returncode": 0,
+            "stdout": "1685\n",
+            "stderr": "",
+            "timed_out": false
+          }
+        ]
+      },
+      "compute_loop": {
+        "status": "success",
+        "iterations": 20000,
+        "samples_seconds": [
+          1.311903371,
+          1.300991475,
+          1.202045287
+        ],
+        "min_us_per_iter": 60.10226435,
+        "checksum": "2646700"
+      },
+      "string_join": {
+        "status": "success",
+        "iterations": 200000,
+        "samples_seconds": [
+          0.104971033,
+          0.102914071,
+          0.103065405
+        ],
+        "min_us_per_iter": 0.514570355,
+        "checksum": "alpha,beta,gamma,delta"
+      },
+      "dict_ops": {
+        "status": "success",
+        "iterations": 50000,
+        "samples_seconds": [
+          0.840918225,
+          0.82496832,
+          0.883600098
+        ],
+        "min_us_per_iter": 16.4993664,
+        "checksum": "2401"
+      }
+    },
+    "cpython": {
+      "cold_start": {
+        "status": "success",
+        "samples_seconds": [
+          0.009372643,
+          0.007881515,
+          0.007195638,
+          0.006868554,
+          0.006970639,
+          0.006985347,
+          0.007123222,
+          0.006893013,
+          0.006987305,
+          0.006018803,
+          0.006011012,
+          0.008965017,
+          0.005422052,
+          0.006649429,
+          0.007071013,
+          0.006883305,
+          0.005946678,
+          0.007103806,
+          0.006899805,
+          0.006989389
+        ],
+        "min_ms": 5.422052,
+        "avg_ms": 7.011909250000001,
+        "checksum": "2"
+      },
+      "json_loads": {
+        "status": "success",
+        "iterations": 200000,
+        "samples_seconds": [
+          1.208114381,
+          1.169858476,
+          1.25310555
+        ],
+        "min_us_per_iter": 5.8492923800000005,
+        "checksum": "d50887ca-a6ce-4e59-b89f-14f0b5d03b03"
+      },
+      "json_dumps": {
+        "status": "success",
+        "iterations": 200000,
+        "samples_seconds": [
+          1.721984775,
+          1.671889139,
+          1.695675017
+        ],
+        "min_us_per_iter": 8.359445695,
+        "checksum": "1815"
+      },
+      "compute_loop": {
+        "status": "success",
+        "iterations": 20000,
+        "samples_seconds": [
+          0.152286289,
+          0.176022168,
+          0.21998496
+        ],
+        "min_us_per_iter": 7.614314449999999,
+        "checksum": "2646700"
+      },
+      "string_join": {
+        "status": "success",
+        "iterations": 200000,
+        "samples_seconds": [
+          0.021954375,
+          0.020200205,
+          0.02095654
+        ],
+        "min_us_per_iter": 0.101001025,
+        "checksum": "alpha,beta,gamma,delta"
+      },
+      "dict_ops": {
+        "status": "success",
+        "iterations": 50000,
+        "samples_seconds": [
+          0.125769906,
+          0.111035836,
+          0.110017335
+        ],
+        "min_us_per_iter": 2.2003467,
+        "checksum": "2401"
+      }
+    }
+  }
+}

benchmarks/talk_comparison/results.md

@@ -0,0 +1,51 @@
+# QuickPython vs CPython benchmark results
+
+## Environment
+
+- os_arch: `Linux-7.0.11-orbstack-00354-gde2ab3f9e944-aarch64-with-glibc2.41`
+- cpu: `unavailable`
+- quickpython_commit: `a8168c3432111fd341c581ffacb798351054650f`
+- rustc: `rustc 1.98.1 (48a229cea 2026-09-01)`
+- cargo: `cargo 1.98.1 (797e8a9bc 2026-08-05)`
+- cpython: `Python 3.11.16`
+
+## Commands
+
+- `cargo build --release`
+- `target/release/quickpython run <generated-or-committed-workload.py>`
+- `python3 <generated-or-committed-workload.py>`
+
+## Comparison
+
+| workload | N | QuickPython | CPython | QPY/CPython | status / notes |
+|---|---:|---:|---:|---:|---|
+| cold_start | 20 | 0.491/1.779 ms min/avg | 5.422/7.012 ms min/avg | 0.09x | success; checksum verified |
+| json_loads | 200000 | 15.900 µs/iter | 5.849 µs/iter | 2.72x | success; checksum verified |
+| json_dumps | 200000 | failed | success | — | runtime; semantic difference: expected stdout '1815', got '1685' |
+| compute_loop | 20000 | 60.102 µs/iter | 7.614 µs/iter | 7.89x | success; checksum verified |
+| string_join | 200000 | 0.515 µs/iter | 0.101 µs/iter | 5.09x | success; checksum verified |
+| dict_ops | 50000 | 16.499 µs/iter | 2.200 µs/iter | 7.50x | success; checksum verified |
+
+## Raw elapsed samples
+
+### quickpython
+
+- cold_start: `0.002, 0.001, 0.002, 0.001, 0.002, 0.002, 0.003, 0.000, 0.003, 0.002, 0.002, 0.002, 0.001, 0.002, 0.002, 0.000, 0.002, 0.002, 0.001, 0.002` seconds
+- json_loads: `3.180, 3.335, 3.269` seconds
+- json_dumps: failed at runtime (semantic difference): expected stdout '1815', got '1685'
+- compute_loop: `1.312, 1.301, 1.202` seconds
+- string_join: `0.105, 0.103, 0.103` seconds
+- dict_ops: `0.841, 0.825, 0.884` seconds
+
+### cpython
+
+- cold_start: `0.009, 0.008, 0.007, 0.007, 0.007, 0.007, 0.007, 0.007, 0.007, 0.006, 0.006, 0.009, 0.005, 0.007, 0.007, 0.007, 0.006, 0.007, 0.007, 0.007` seconds
+- json_loads: `1.208, 1.170, 1.253` seconds
+- json_dumps: `1.722, 1.672, 1.696` seconds
+- compute_loop: `0.152, 0.176, 0.220` seconds
+- string_join: `0.022, 0.020, 0.021` seconds
+- dict_ops: `0.126, 0.111, 0.110` seconds
+
+## Conclusion
+
+This same-machine snapshot supports comparisons only for the successful workloads (cold_start, json_loads, compute_loop, string_join, dict_ops). For short-lived agent calls, cold-start results include process startup plus parsing, compilation, and execution, while non-cold results amortize those costs over the fixed in-runtime loop. Unavailable workloads (json_dumps) are compatibility findings and provide no evidence about their relative performance.