Property tests and fuzz targets against the reference

.github/workflows/fuzz.yml

@@ -0,0 +1,148 @@
+name: Fuzz
+
+# The cargo-fuzz targets in runtime/fuzz, and the property tests at a high
+# case count. A pull request runs each target for a minute, seeded from the
+# conformance vectors. The nightly run gives each target ten minutes from a
+# corpus cached between runs, runs proptest with 100000 cases, and opens an
+# issue on a failure instead of blocking anyone's pull request.
+# See runtime/fuzz/README.md.
+
+on:
+  pull_request:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  # cargo-fuzz needs nightly; this overrides any rust-toolchain file.
+  RUSTUP_TOOLCHAIN: nightly
+  CARGO_FUZZ_VERSION: 0.13.2
+
+jobs:
+  fuzz:
+    name: fuzz (${{ matrix.target }})
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - decode_no_panic
+          - packed_canonical
+          - framing_decode
+          - seed_vs_derive
+    env:
+      NIGHTLY: ${{ github.event_name != 'pull_request' }}
+      SECONDS_PER_TARGET: ${{ github.event_name == 'pull_request' && 60 || 600 }}
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install nightly --profile minimal
+
+      - name: Cache cargo-fuzz
+        id: cargo-fuzz
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/bin/cargo-fuzz
+          key: cargo-fuzz-${{ env.CARGO_FUZZ_VERSION }}-${{ runner.os }}
+      - if: steps.cargo-fuzz.outputs.cache-hit != 'true'
+        run: cargo install cargo-fuzz --locked --version "$CARGO_FUZZ_VERSION"
+
+      - name: Cache the corpus
+        if: env.NIGHTLY == 'true'
+        uses: actions/cache@v4
+        with:
+          path: runtime/fuzz/corpus/${{ matrix.target }}
+          key: fuzz-corpus-${{ matrix.target }}-${{ github.run_id }}
+          restore-keys: fuzz-corpus-${{ matrix.target }}-
+
+      - name: Seed the corpus from the vectors
+        run: sh runtime/fuzz/seed-corpus.sh
+
+      - name: Fuzz for ${{ env.SECONDS_PER_TARGET }}s
+        run: >
+          cargo fuzz run --fuzz-dir runtime/fuzz ${{ matrix.target }} --
+          -max_total_time=${{ env.SECONDS_PER_TARGET }}
+          -rss_limit_mb=2048 -malloc_limit_mb=512
+
+      - name: Minimize the crash
+        if: failure()
+        run: |
+          for crash in runtime/fuzz/artifacts/${{ matrix.target }}/*; do
+            [ -f "$crash" ] || continue
+            cargo fuzz tmin --fuzz-dir runtime/fuzz ${{ matrix.target }} "$crash" -r 2000 || true
+          done
+      - name: Upload the crash
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: fuzz-${{ matrix.target }}
+          path: runtime/fuzz/artifacts/${{ matrix.target }}
+          if-no-files-found: ignore
+
+      - name: Open an issue
+        if: failure() && env.NIGHTLY == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TITLE: "Nightly fuzzing failed: ${{ matrix.target }}"
+          RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          open=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 200 \
+            --json title --jq "map(select(.title == env.TITLE)) | length")
+          if [ "$open" != "0" ]; then
+            echo "An issue titled '$TITLE' is already open."
+            exit 0
+          fi
+          {
+            echo "The nightly fuzz run failed: $RUN"
+            echo
+            echo "The crash and its minimized form are the run's \`fuzz-${{ matrix.target }}\` artifact. As hex:"
+            echo
+            for crash in runtime/fuzz/artifacts/${{ matrix.target }}/*; do
+              [ -f "$crash" ] || continue
+              echo "- \`$(basename "$crash")\`: \`$(od -An -v -tx1 "$crash" | tr -d ' \n' | cut -c1-2000)\`"
+            done
+            echo
+            echo "Reproduce with \`cargo +nightly fuzz run --fuzz-dir runtime/fuzz ${{ matrix.target }} <file>\`."
+            echo "A real bug becomes a \`found-*\` vector; see runtime/fuzz/README.md."
+          } > body.md
+          gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file body.md
+
+  properties:
+    name: proptest (100000 cases)
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install nightly --profile minimal
+      - name: Property tests
+        env:
+          PROPTEST_CASES: "100000"
+        run: cargo test --release -p comline-runtime-conformance --test properties
+      - name: Upload the failing seeds
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: proptest-regressions
+          path: runtime/conformance/tests/properties.proptest-regressions
+          if-no-files-found: ignore
+      - name: Open an issue
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TITLE: "Nightly property tests failed"
+          RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          open=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --limit 200 \
+            --json title --jq "map(select(.title == env.TITLE)) | length")
+          if [ "$open" != "0" ]; then
+            echo "An issue titled '$TITLE' is already open."
+            exit 0
+          fi
+          gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body \
+            "The nightly proptest run (PROPTEST_CASES=100000) failed: $RUN. The minimal failing input is in the job log; the seed is the run's \`proptest-regressions\` artifact."

Cargo.toml

@@ -15,6 +15,8 @@ members = [
     #"runtime-langs/luau"
     #"runtime-langs/python"
 ]
+# cargo-fuzz targets: nightly only, built with `cargo +nightly fuzz`.
+exclude = ["runtime/fuzz"]
 # `runtime/core_no-std` removed — `comline-runtime` is now `no_std`-first with a
 # `std` feature, not a parallel fork. See ComlineProject/docs → Design.
 

runtime/conformance/Cargo.toml

@@ -18,3 +18,5 @@ serde_json = "1.0.108"
 [dev-dependencies]
 # Generated Rust uses it for fixed arrays longer than 32.
 serde-big-array = "0.5"
+# Property tests (`tests/properties.rs`); `PROPTEST_CASES` raises the count.
+proptest = "1.5"

runtime/conformance/README.md

@@ -76,6 +76,11 @@ generated code, not a hand-written decoder.
 [`tests/seed_vs_derive.rs`](tests/seed_vs_derive.rs) holds it to real
 derives, on valid and mutated bytes in every format.
 
+[`tests/properties.rs`](tests/properties.rs) checks the reference's properties
+over random types and values with `proptest`, and
+[`runtime/fuzz`](../fuzz/README.md) fuzzes it with `cargo-fuzz`. Cases they
+find become `found-*` vectors; the fuzz README says how.
+
 A value is typed, so a runtime knows which width and shape to encode:
 `{"u8": 255}`, `{"u64": "18446744073709551615"}` (and `u128`, `i128`, also as
 decimal strings), `{"string": "x"}`, `{"tuple": [...]}` for `T[N]`,

runtime/conformance/reports/rust.json

@@ -61,7 +61,8 @@
         "format-json/decode-fixed-array-33",
         "format-json/decode-fixed-array-short",
         "format-json/decode-fixed-array-long",
-        "format-json/decode-array-of-options"
+        "format-json/decode-array-of-options",
+        "format-json/found-f64-exact-digits"
       ]
     },
     "format-msgpack": {
@@ -160,6 +161,7 @@
         "format-packed/reject-overlong-length",
         "format-packed/reject-length-past-input",
         "format-packed/reject-invalid-utf8",
+        "format-packed/reject-count-past-input",
         "format-packed/decode-fields-reordered",
         "format-packed/decode-unknown-field",
         "format-packed/decode-missing-optional",

runtime/conformance/src/lib.rs

@@ -1139,7 +1139,10 @@ pub fn reference_cases() -> Vec<Vectors> {
     for v in &mut vectors {
         v.cases.extend(rejections(&v.feature));
         match v.feature.as_str() {
-            "format-json" => v.cases.extend(decode_cases("format-json", FORMAT_JSON)),
+            "format-json" => {
+                v.cases.extend(decode_cases("format-json", FORMAT_JSON));
+                v.cases.extend(found_cases("format-json", FORMAT_JSON));
+            }
             "format-msgpack" => v
                 .cases
                 .extend(decode_cases("format-msgpack", FORMAT_MSGPACK)),
@@ -1552,6 +1555,33 @@ fn decode_cases(feature_id: &str, format: &str) -> Vec<Case> {
     ]
 }
 
+/// Inputs the property tests and fuzzers found (`tests/properties.rs`,
+/// `runtime/fuzz`), minimized, as `found-*` cases. See `runtime/fuzz/README.md`.
+fn found_cases(feature_id: &str, format: &str) -> Vec<Case> {
+    let p = |id: &str| format!("{feature_id}/found-{id}");
+    // A value that must read back as itself, not only as whatever the
+    // reference happened to decode.
+    let exact = |id: &str, written: Value, ty: Type| {
+        let case = decode_case(p(id), format, &written, ty);
+        let decoded = serde_json::from_value::<DecodeIn>(case.input.clone())
+            .ok()
+            .and_then(|i| i.value);
+        assert_eq!(decoded, Some(written), "`{}` doesn't read back", case.id);
+        case
+    };
+    match format {
+        // serde_json's default float parser can be one ulp off; the
+        // runtime enables `float_roundtrip`, which reads every `f64` that
+        // was written back exactly.
+        FORMAT_JSON => vec![exact(
+            "f64-exact-digits",
+            Value::F64(4.520618856987477e-279),
+            Type::F64,
+        )],
+        _ => vec![],
+    }
+}
+
 /// Bytes a runtime must refuse, per feature.
 fn rejections(feature: &str) -> Vec<Case> {
     let none = serde_json::Value::Null;
@@ -1656,6 +1686,19 @@ fn rejections(feature: &str) -> Vec<Case> {
                 r("reject-overlong-length", Type::String, &[0x80, 0x00]),
                 r("reject-length-past-input", Type::String, &[5, b'a', b'b']),
                 r("reject-invalid-utf8", Type::String, &[2, 0xc3, 0x28]),
+                // Found by tests/properties.rs (as a `found-*` case would be):
+                // a count is never more than the bytes left, even for
+                // elements that take none, so `[Empty {}]`'s `01` is refused.
+                // Schemas can't declare such an array: core refuses arrays
+                // of data-less types (ComlineProject/core#141).
+                r(
+                    "reject-count-past-input",
+                    Type::Array(Box::new(Type::Struct {
+                        name: "Empty".into(),
+                        fields: vec![],
+                    })),
+                    &[1],
+                ),
             ]
         }
         "framing-jsonrpc" => vec![