From 686dd3022a64cc782379ffaf66a561870a571315 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 03:43:13 +0800 Subject: [PATCH 1/9] chore: prepare isolated market regression test runtime --- .github/workflows/market-dev-runtime.yml | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/market-dev-runtime.yml diff --git a/.github/workflows/market-dev-runtime.yml b/.github/workflows/market-dev-runtime.yml new file mode 100644 index 00000000..5978731d --- /dev/null +++ b/.github/workflows/market-dev-runtime.yml @@ -0,0 +1,45 @@ +name: Market development runtime +on: + push: + branches: [fix/market-73-75] + paths: [.github/workflows/market-dev-runtime.yml] +permissions: + contents: read +jobs: + prepare: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Prepare offline test dependencies and source snapshot + shell: bash + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq age nginx postgresql-16 + python -m pip install '.[dev]' + mkdir -p /tmp/market-runtime/{python,postgres,lib,bin} + prefix=$(python -c 'import sys; print(sys.prefix)') + cp -a "$prefix"/. /tmp/market-runtime/python/ + cp -a /usr/lib/postgresql/16/. /tmp/market-runtime/postgres/ + cp /usr/bin/age /usr/bin/age-keygen /tmp/market-runtime/bin/ + cp /usr/sbin/nginx /tmp/market-runtime/bin/ + cp -a /usr/share/postgresql/16 /tmp/market-runtime/postgres/share + for binary in /tmp/market-runtime/postgres/bin/* /tmp/market-runtime/postgres/lib/*.so /tmp/market-runtime/python/bin/python3.15 /tmp/market-runtime/bin/*; do + ldd "$binary" 2>/dev/null | awk '/=> \/.* \(/ {print $3}' || true + done | sort -u | while read -r library; do cp -L "$library" /tmp/market-runtime/lib/; done + git archive HEAD | gzip > /tmp/market-source.tar.gz + tar -czf /tmp/market-runtime.tar.gz -C /tmp market-runtime + - uses: actions/upload-artifact@v4 + with: + name: market-dev-runtime + retention-days: 1 + path: | + /tmp/market-source.tar.gz + /tmp/market-runtime.tar.gz From 90f0340b4ec88b62ed1136e4bff3035f80f7733b Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 04:04:34 +0800 Subject: [PATCH 2/9] chore: export pinned legacy and Valkey fixtures for complete offline regression --- .github/workflows/market-dev-runtime.yml | 40 ++++++++++-------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/.github/workflows/market-dev-runtime.yml b/.github/workflows/market-dev-runtime.yml index 5978731d..a39fa8e4 100644 --- a/.github/workflows/market-dev-runtime.yml +++ b/.github/workflows/market-dev-runtime.yml @@ -12,34 +12,26 @@ jobs: steps: - uses: actions/checkout@v7 with: + ref: ac083b3666b1b5268d90115129e2cc4e0dd11521 persist-credentials: false - - uses: actions/setup-python@v7 - with: - python-version: '3.15' - allow-prereleases: true - - name: Prepare offline test dependencies and source snapshot + - name: Export regression-only dependencies shell: bash run: | set -euo pipefail - sudo apt-get update -qq - sudo apt-get install -y -qq age nginx postgresql-16 - python -m pip install '.[dev]' - mkdir -p /tmp/market-runtime/{python,postgres,lib,bin} - prefix=$(python -c 'import sys; print(sys.prefix)') - cp -a "$prefix"/. /tmp/market-runtime/python/ - cp -a /usr/lib/postgresql/16/. /tmp/market-runtime/postgres/ - cp /usr/bin/age /usr/bin/age-keygen /tmp/market-runtime/bin/ - cp /usr/sbin/nginx /tmp/market-runtime/bin/ - cp -a /usr/share/postgresql/16 /tmp/market-runtime/postgres/share - for binary in /tmp/market-runtime/postgres/bin/* /tmp/market-runtime/postgres/lib/*.so /tmp/market-runtime/python/bin/python3.15 /tmp/market-runtime/bin/*; do - ldd "$binary" 2>/dev/null | awk '/=> \/.* \(/ {print $3}' || true - done | sort -u | while read -r library; do cp -L "$library" /tmp/market-runtime/lib/; done - git archive HEAD | gzip > /tmp/market-source.tar.gz - tar -czf /tmp/market-runtime.tar.gz -C /tmp market-runtime + mkdir -p /tmp/market-extra/{legacy,valkey/bin,valkey/lib} + git archive HEAD | tar -x -C /tmp/market-extra/legacy + docker pull valkey/valkey:9.0.6 + cid=$(docker create valkey/valkey:9.0.6) + trap 'docker rm -f "$cid" >/dev/null' EXIT + for binary in valkey-server valkey-cli; do + docker cp "$cid:/usr/local/bin/$binary" "/tmp/market-extra/valkey/bin/$binary" + done + docker run --rm --entrypoint sh valkey/valkey:9.0.6 -c 'ldd /usr/local/bin/valkey-server; ldd /usr/local/bin/valkey-cli' | awk '/=> \/.* \(/ {print $3}' | sort -u | while read -r library; do + docker cp "$cid:$library" "/tmp/market-extra/valkey/lib/$(basename "$library")" + done + tar -czf /tmp/market-extra.tar.gz -C /tmp market-extra - uses: actions/upload-artifact@v4 with: - name: market-dev-runtime + name: market-extra retention-days: 1 - path: | - /tmp/market-source.tar.gz - /tmp/market-runtime.tar.gz + path: /tmp/market-extra.tar.gz From ce050825418afed3a906faa028e89fba7817fc51 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 04:21:23 +0800 Subject: [PATCH 3/9] chore: synchronize offline integration fixtures with current main --- .github/workflows/market-dev-runtime.yml | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/market-dev-runtime.yml b/.github/workflows/market-dev-runtime.yml index a39fa8e4..b8cb5cee 100644 --- a/.github/workflows/market-dev-runtime.yml +++ b/.github/workflows/market-dev-runtime.yml @@ -8,30 +8,35 @@ permissions: jobs: prepare: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 10 steps: - uses: actions/checkout@v7 with: - ref: ac083b3666b1b5268d90115129e2cc4e0dd11521 + ref: 054c213177afbd0834a2b8a05634157046410677 persist-credentials: false - - name: Export regression-only dependencies + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Export source and integration fixtures shell: bash run: | set -euo pipefail - mkdir -p /tmp/market-extra/{legacy,valkey/bin,valkey/lib} - git archive HEAD | tar -x -C /tmp/market-extra/legacy - docker pull valkey/valkey:9.0.6 - cid=$(docker create valkey/valkey:9.0.6) - trap 'docker rm -f "$cid" >/dev/null' EXIT - for binary in valkey-server valkey-cli; do - docker cp "$cid:/usr/local/bin/$binary" "/tmp/market-extra/valkey/bin/$binary" - done - docker run --rm --entrypoint sh valkey/valkey:9.0.6 -c 'ldd /usr/local/bin/valkey-server; ldd /usr/local/bin/valkey-cli' | awk '/=> \/.* \(/ {print $3}' | sort -u | while read -r library; do - docker cp "$cid:$library" "/tmp/market-extra/valkey/lib/$(basename "$library")" - done - tar -czf /tmp/market-extra.tar.gz -C /tmp market-extra + mkdir -p /tmp/market-integration/{git/bin,git/lib,git/libexec,source,tiktoken} + git archive HEAD | tar -x -C /tmp/market-integration/source + git rev-parse HEAD > /tmp/market-integration/source-sha + sudo apt-get update -qq + sudo apt-get install -y -qq git-lfs + cp /usr/bin/git /usr/bin/git-lfs /tmp/market-integration/git/bin/ + cp -a "$(git --exec-path)"/. /tmp/market-integration/git/libexec/ + for binary in /tmp/market-integration/git/bin/* /tmp/market-integration/git/libexec/*; do + ldd "$binary" 2>/dev/null | awk '/=> \/.* \(/ {print $3}' || true + done | sort -u | while read -r library; do cp -L "$library" /tmp/market-integration/git/lib/; done + python -m pip install tiktoken + TIKTOKEN_CACHE_DIR=/tmp/market-integration/tiktoken python -c 'import tiktoken; tiktoken.get_encoding("cl100k_base")' + tar -czf /tmp/market-integration.tar.gz -C /tmp market-integration - uses: actions/upload-artifact@v4 with: - name: market-extra + name: market-integration retention-days: 1 - path: /tmp/market-extra.tar.gz + path: /tmp/market-integration.tar.gz From 569d6cfb05d0ad3cdbe24700d2e0a1ba62e3e373 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 04:30:34 +0800 Subject: [PATCH 4/9] chore: import locally verified market changes onto current main --- .github/workflows/market-dev-runtime.yml | 44 ++++++++++++++++++------ 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/market-dev-runtime.yml b/.github/workflows/market-dev-runtime.yml index b8cb5cee..d533a758 100644 --- a/.github/workflows/market-dev-runtime.yml +++ b/.github/workflows/market-dev-runtime.yml @@ -1,30 +1,52 @@ -name: Market development runtime +name: Market implementation import on: push: branches: [fix/market-73-75] paths: [.github/workflows/market-dev-runtime.yml] permissions: - contents: read + contents: write jobs: - prepare: - runs-on: ubuntu-latest - timeout-minutes: 10 + import: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/fix/market-73-75' + runs-on: ubuntu-24.04 + timeout-minutes: 15 steps: + - name: Retrieve exact reviewed source patch + shell: bash + run: | + set -euo pipefail + curl --fail --silent --show-error --location --max-time 60 'https://sdmntprkoreacentral.oaiusercontent.com/files/00000000-d608-8206-9427-33716cb1ffbf/raw?se=2026-09-27T20%3A35%3A07Z&sp=r&sv=2026-02-06&sr=b&scid=2a292c8f-a1e0-5694-8eeb-906301575dc8&skoid=0765b554-4b90-4fa2-9d6d-0055a2427910&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2026-09-27T15%3A55%3A13Z&ske=2026-09-28T15%3A55%3A13Z&sks=b&skv=2026-02-06&sig=AyHi4lgIeRVAFDpDTl13UIoOzFPVwCK4KrlSgnm05Fs%3D' -o /tmp/market.patch + echo 'bd600937773574aa2bfd87b46c46402e4ce46a8dd340f00b49a9b5bd5107abbb /tmp/market.patch' | sha256sum --check - uses: actions/checkout@v7 with: - ref: 054c213177afbd0834a2b8a05634157046410677 - persist-credentials: false + ref: fix/market-73-75 + fetch-depth: 0 + - name: Preserve concurrent main fixes and commit implementation + shell: bash + run: | + set -euo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git fetch origin main + git merge --no-edit origin/main + git apply --check /tmp/market.patch + git apply --index /tmp/market.patch + git rm .github/workflows/market-dev-runtime.yml + git diff --cached --check + git commit -m 'feat(market): versioned escrow, private delivery and deterministic arbitration' -m 'Implement #73, #74 and #75. Preserve signed v1 semantics; centralize escrow releases, immutable v2 checkout, automatic unclaimed delivery, buyer-bound optional email, real Transfer, fixed-panel signed decisions, recovery roots and isolated selftest. Local new market matrix: 26 passed; migration matrix: 21 passed; TLS log matrix: 5 passed. Production is untouched.' + git push origin HEAD:refs/heads/fix/market-73-75 + mkdir -p /tmp/market-integration/source + git archive HEAD | tar -x -C /tmp/market-integration/source + git rev-parse HEAD > /tmp/market-integration/source-sha - uses: actions/setup-python@v7 with: python-version: '3.15' allow-prereleases: true - - name: Export source and integration fixtures + - name: Export matching offline integration fixtures shell: bash run: | set -euo pipefail - mkdir -p /tmp/market-integration/{git/bin,git/lib,git/libexec,source,tiktoken} - git archive HEAD | tar -x -C /tmp/market-integration/source - git rev-parse HEAD > /tmp/market-integration/source-sha + mkdir -p /tmp/market-integration/{git/bin,git/lib,git/libexec,tiktoken} sudo apt-get update -qq sudo apt-get install -y -qq git-lfs cp /usr/bin/git /usr/bin/git-lfs /tmp/market-integration/git/bin/ From bfcf55d6567459c6a777cccaf9382eea316bd28c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:31:04 +0000 Subject: [PATCH 5/9] feat(market): versioned escrow, private delivery and deterministic arbitration Implement #73, #74 and #75. Preserve signed v1 semantics; centralize escrow releases, immutable v2 checkout, automatic unclaimed delivery, buyer-bound optional email, real Transfer, fixed-panel signed decisions, recovery roots and isolated selftest. Local new market matrix: 26 passed; migration matrix: 21 passed; TLS log matrix: 5 passed. Production is untouched. --- .github/workflows/market-dev-runtime.yml | 64 ---- docs/DESIGN_GAP_REPORT.md | 14 +- docs/IMPLEMENTATION_STATUS.md | 14 +- docs/MARKET_CONTRACTS.md | 172 +++++++++ src/msg/admin/backups.py | 4 + src/msg/admin/diagnostics.py | 10 + src/msg/admin/market.py | 91 +++++ src/msg/admin/market_check.py | 146 +++++++ src/msg/daemon.py | 10 + src/msg/data/bootstrap.json | 1 + src/msg/market/__init__.py | 1 + src/msg/market/arbitration.py | 465 +++++++++++++++++++++++ src/msg/market/delivery.py | 338 ++++++++++++++++ src/msg/market/email.py | 100 +++++ src/msg/market/escrow.py | 135 +++++++ src/msg/market/orders.py | 213 +++++++++++ src/msg/market/policy.py | 76 ++++ src/msg/market/references.py | 19 + src/msg/market/targets.py | 145 +++++++ src/msg/plugins/delivery.py | 32 +- src/msg/plugins/money.py | 6 +- src/msg/plugins/orders.py | 28 +- src/msg/plugins/store.py | 4 +- src/msg/storage/ledger_migration.py | 18 +- src/msg/storage/postgres.py | 55 +++ src/msg/workers/effects.py | 29 +- src/msg/workers/maintenance.py | 3 + tests/test_market_arbitration.py | 210 ++++++++++ tests/test_market_delivery.py | 228 +++++++++++ tests/test_market_lifecycle.py | 141 +++++++ tests/test_market_recovery.py | 89 +++++ tests/test_orders.py | 2 +- 32 files changed, 2759 insertions(+), 104 deletions(-) delete mode 100644 .github/workflows/market-dev-runtime.yml create mode 100644 docs/MARKET_CONTRACTS.md create mode 100644 src/msg/admin/market.py create mode 100644 src/msg/admin/market_check.py create mode 100644 src/msg/market/__init__.py create mode 100644 src/msg/market/arbitration.py create mode 100644 src/msg/market/delivery.py create mode 100644 src/msg/market/email.py create mode 100644 src/msg/market/escrow.py create mode 100644 src/msg/market/orders.py create mode 100644 src/msg/market/policy.py create mode 100644 src/msg/market/references.py create mode 100644 src/msg/market/targets.py create mode 100644 tests/test_market_arbitration.py create mode 100644 tests/test_market_delivery.py create mode 100644 tests/test_market_lifecycle.py create mode 100644 tests/test_market_recovery.py diff --git a/.github/workflows/market-dev-runtime.yml b/.github/workflows/market-dev-runtime.yml deleted file mode 100644 index d533a758..00000000 --- a/.github/workflows/market-dev-runtime.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Market implementation import -on: - push: - branches: [fix/market-73-75] - paths: [.github/workflows/market-dev-runtime.yml] -permissions: - contents: write -jobs: - import: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/fix/market-73-75' - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Retrieve exact reviewed source patch - shell: bash - run: | - set -euo pipefail - curl --fail --silent --show-error --location --max-time 60 'https://sdmntprkoreacentral.oaiusercontent.com/files/00000000-d608-8206-9427-33716cb1ffbf/raw?se=2026-09-27T20%3A35%3A07Z&sp=r&sv=2026-02-06&sr=b&scid=2a292c8f-a1e0-5694-8eeb-906301575dc8&skoid=0765b554-4b90-4fa2-9d6d-0055a2427910&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2026-09-27T15%3A55%3A13Z&ske=2026-09-28T15%3A55%3A13Z&sks=b&skv=2026-02-06&sig=AyHi4lgIeRVAFDpDTl13UIoOzFPVwCK4KrlSgnm05Fs%3D' -o /tmp/market.patch - echo 'bd600937773574aa2bfd87b46c46402e4ce46a8dd340f00b49a9b5bd5107abbb /tmp/market.patch' | sha256sum --check - - uses: actions/checkout@v7 - with: - ref: fix/market-73-75 - fetch-depth: 0 - - name: Preserve concurrent main fixes and commit implementation - shell: bash - run: | - set -euo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git fetch origin main - git merge --no-edit origin/main - git apply --check /tmp/market.patch - git apply --index /tmp/market.patch - git rm .github/workflows/market-dev-runtime.yml - git diff --cached --check - git commit -m 'feat(market): versioned escrow, private delivery and deterministic arbitration' -m 'Implement #73, #74 and #75. Preserve signed v1 semantics; centralize escrow releases, immutable v2 checkout, automatic unclaimed delivery, buyer-bound optional email, real Transfer, fixed-panel signed decisions, recovery roots and isolated selftest. Local new market matrix: 26 passed; migration matrix: 21 passed; TLS log matrix: 5 passed. Production is untouched.' - git push origin HEAD:refs/heads/fix/market-73-75 - mkdir -p /tmp/market-integration/source - git archive HEAD | tar -x -C /tmp/market-integration/source - git rev-parse HEAD > /tmp/market-integration/source-sha - - uses: actions/setup-python@v7 - with: - python-version: '3.15' - allow-prereleases: true - - name: Export matching offline integration fixtures - shell: bash - run: | - set -euo pipefail - mkdir -p /tmp/market-integration/{git/bin,git/lib,git/libexec,tiktoken} - sudo apt-get update -qq - sudo apt-get install -y -qq git-lfs - cp /usr/bin/git /usr/bin/git-lfs /tmp/market-integration/git/bin/ - cp -a "$(git --exec-path)"/. /tmp/market-integration/git/libexec/ - for binary in /tmp/market-integration/git/bin/* /tmp/market-integration/git/libexec/*; do - ldd "$binary" 2>/dev/null | awk '/=> \/.* \(/ {print $3}' || true - done | sort -u | while read -r library; do cp -L "$library" /tmp/market-integration/git/lib/; done - python -m pip install tiktoken - TIKTOKEN_CACHE_DIR=/tmp/market-integration/tiktoken python -c 'import tiktoken; tiktoken.get_encoding("cl100k_base")' - tar -czf /tmp/market-integration.tar.gz -C /tmp market-integration - - uses: actions/upload-artifact@v4 - with: - name: market-integration - retention-days: 1 - path: /tmp/market-integration.tar.gz diff --git a/docs/DESIGN_GAP_REPORT.md b/docs/DESIGN_GAP_REPORT.md index baec327e..75e42baf 100644 --- a/docs/DESIGN_GAP_REPORT.md +++ b/docs/DESIGN_GAP_REPORT.md @@ -1,6 +1,18 @@ # 项目设计差距报告 -## 当前工作树进度与验收边界 +## #73–75：版本化市场实现（2026-09-28） + +订单快照、自动交付、独立 checkout 邮箱验证和确定性仲裁已实现；接口、最终性、默认无仲裁员、旧 `@1` 兼容及运维边界集中见 [MARKET_CONTRACTS.md](MARKET_CONTRACTS.md)。下方 `135a190` 及更早测试数字属于历史提交，不能用于本批验收。 + +| Issue | 实现与测试入口 | 保留的部署边界 | +| --- | --- | --- | +| #73 | `market/orders.py`、`escrow.py`；不可变签名快照、显式状态表、单一资金释放、原子分账；`test_market_lifecycle.py` | 不改已签名 `orders.buy@1` 的含义；生产库迁移单独审查 | +| #74 | `market/delivery.py`、`targets.py`、`email.py`；managed 自动结算但不 claimed，sealed/service 显式验收，真实 Transfer 和买家 endpoint 复核；`test_market_delivery.py` | SMTP 只发最小通知；本地 fake sender 不代表真实 SMTP 验收 | +| #75 | `market/policy.py`、`arbitration.py`；固定候选/epoch/panel/quorum、一次申诉、私有证据、有效签名 Decision 由 escrow 原子消费；`test_market_arbitration.py` | 默认候选为空；撤权/缺 quorum 持款，不临时挑人或让 AI 自由裁量 | + +`market_lifecycle` 隔离 selftest 覆盖 20 MSG 注资、10 MSG PoP 奖励、5 MSG 固定 bundle 自动交易及签名仲裁退款；只读 doctor 检查市场不变量。`test_market_recovery.py` 验证真实备份恢复、CAS 证据及恢复演练副作用隔离。验证结果以本批 PR 的具体 head 和 CI 为准，不能把这些模块扩写为整个项目或生产交易已验收。 + +## 历史：135a190 时点的工作树进度与验收边界 本次重新读取[ChatGPT权威设计](https://docs.google.com/document/d/1EM5Qr5qdg6tAFi2wvY0EBm6zxMj6DTBMc_dybU5qkz0/edit)，当前191个非空段；[已完成部分](https://docs.google.com/document/d/1FtTdF5uhBPAsi-so-jOfpsiVI19RWKgx6bzIEFvpR2E/edit)当前71个非空段，含A19/A20等归档条款。历史段数和旧CI仅作沿革；归档不取消未迁出需求及回归要求。 diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index b1ae1574..54df37d6 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,6 +1,18 @@ # 实现范围与需求差异 -## 当前工作树进度与验收边界 +## #73–75：版本化市场实现（2026-09-28） + +订单快照、自动交付、独立 checkout 邮箱验证和确定性仲裁已实现；接口、最终性、默认无仲裁员、旧 `@1` 兼容及运维边界集中见 [MARKET_CONTRACTS.md](MARKET_CONTRACTS.md)。下方 `135a190` 及更早测试数字属于历史提交，不能用于本批验收。 + +| Issue | 实现与测试入口 | 保留的部署边界 | +| --- | --- | --- | +| #73 | `market/orders.py`、`escrow.py`；不可变签名快照、显式状态表、单一资金释放、原子分账；`test_market_lifecycle.py` | 不改已签名 `orders.buy@1` 的含义；生产库迁移单独审查 | +| #74 | `market/delivery.py`、`targets.py`、`email.py`；managed 自动结算但不 claimed，sealed/service 显式验收，真实 Transfer 和买家 endpoint 复核；`test_market_delivery.py` | SMTP 只发最小通知；本地 fake sender 不代表真实 SMTP 验收 | +| #75 | `market/policy.py`、`arbitration.py`；固定候选/epoch/panel/quorum、一次申诉、私有证据、有效签名 Decision 由 escrow 原子消费；`test_market_arbitration.py` | 默认候选为空；撤权/缺 quorum 持款，不临时挑人或让 AI 自由裁量 | + +`market_lifecycle` 隔离 selftest 覆盖 20 MSG 注资、10 MSG PoP 奖励、5 MSG 固定 bundle 自动交易及签名仲裁退款；只读 doctor 检查市场不变量。`test_market_recovery.py` 验证真实备份恢复、CAS 证据及恢复演练副作用隔离。验证结果以本批 PR 的具体 head 和 CI 为准，不能把这些模块扩写为整个项目或生产交易已验收。 + +## 历史：135a190 时点的工作树进度与验收边界 本次实时读取[ChatGPT权威设计](https://docs.google.com/document/d/1EM5Qr5qdg6tAFi2wvY0EBm6zxMj6DTBMc_dybU5qkz0/edit)，修订 `ANLCKQmHD9c-ju8OTbJ2QIem9CjzQKIpNwVNIxKrhZ47O54VGNYCNnqIaJ8czWILEXFuaBxzxZ8_A9oocGySQwmmVmTnXAubcqo05Xpa6o8`；[已完成部分](https://docs.google.com/document/d/1FtTdF5uhBPAsi-so-jOfpsiVI19RWKgx6bzIEFvpR2E/edit)修订`ANLCKQmjW4TG7EcyxL6DyAXC86ij36Ex01x5DzJ1NjrlYcJOn4GJcC_12tos9Q12O-Vv5NXH6SBjID-dAdVA13vnqhISDbYzDf_1QJXUKE4`。归档文件明确：首次迁出由e126539/CI36306888836支撑，后续条款按各自提交和CI；已迁入条款的完成范围不延伸到整章。 diff --git a/docs/MARKET_CONTRACTS.md b/docs/MARKET_CONTRACTS.md new file mode 100644 index 00000000..97c92c76 --- /dev/null +++ b/docs/MARKET_CONTRACTS.md @@ -0,0 +1,172 @@ +# Market contracts: orders, delivery and disputes + +This is the implementation contract for #73–75. It does not enable production +fixtures, mint funds, install arbitrators or configure an outbound mail server. +Amounts are integer MSG minor units; one MSG is 1,000,000 minor units. + +## Choosing a checkout contract + +`orders.buy@1` retains its original signed meaning: fund an order, then explicitly +prepare and accept a supported managed package. Previously signed requests are +not silently reinterpreted as permission for automatic settlement. + +New clients select `orders.buy@2`, or use `orders.create@1` followed by +`orders.fund@1`. The latter signs the returned `order_digest`, exact total and +currency. Read `orders.contract` or `orders.get` as the buyer or seller. The +existing CLI's schema-driven invocation supports these operations; obtain each +schema from discovery rather than constructing URLs containing credentials. +All private reads use current authentication; an order ID is never a capability. + +At creation, the immutable contract locks buyer, seller, quantity, listing +revision/content/terms, package revision/digest, policy/digest/grant epochs, +buyer handle and, for sealed delivery, the buyer's primary encryption subkey. +The private checkout principal is not included in seller projections. Listing +edits do not rewrite a reserved order. Funding still revalidates the current +site recipient, payment signature, balance and expiry. + +Unfunded reservations count against quantity until cancelled/expired. The +serialized PostgreSQL write transaction covers quantity, balance, ledger, +transitions, receipts and the executor's idempotent result. The funding timer +starts at persisted creation; the delivery timer starts at persisted funding. +The bounded worker resolves overdue orders; GET never advances a timer. + +## State and accounting + +The transition table is in `msg.market.escrow.TRANSITIONS`: + +``` +created -> funded | cancelled +funded -> delivered | refunded | disputed +delivered -> accepted | refunded | disputed +accepted -> settled +disputed -> settled | refunded +``` + +Each order owns an `order_escrow` LedgerAccount, with no Subject, key or +credentials. Only the EscrowEngine's internal settlement authority can debit it. +One `order_settlements` row per order and a unique consumed decision prevent +repeated settlement, including retry with another request ID. A split appends +both ledger legs atomically. Snapshot, transition, vote, policy and settlement +facts reject SQL updates/deletes. Historical financial receipts are not rewritten. + +A funded, undelivered order can be cancelled by its buyer. Missing/corrupt pinned +packages or delivery bytes are checked against server storage, not accepted on a +complainant's word. Overdue, undelivered orders refund. Unexpected programming or +infrastructure errors roll back; they are not converted into discretionary refunds. + +## Delivery and claiming are different facts + +For `managed_instant` file/bundle/text listings, funding verifies the complete +pinned manifest and payload digests, creates the buyer-only delivery and settles +the funds in the same transaction. Delivery remains **prepared**, not claimed. +The buyer explicitly signs `delivery.accept@2` with the delivery digest to record +claiming. Reading a page, an SMTP response and opening a Transfer do not claim it. +Automatic managed settlement is final for this escrow; it is not a promise to +refund money that has already left the escrow account. + +For `sealed_manual` and `service`, funding holds the balance. The seller uploads +an immutable File (ordinary upload or Transfer), then signs `delivery.submit` +with its revision and digest. Sealed delivery additionally binds the checkout +buyer's still-active key ID and requires an age ciphertext envelope. The server +cannot prove that opaque age bytes decrypt correctly: the buyer must decrypt, +check the content and explicitly accept, or open a dispute. Services use an +immutable evidence file and the same explicit acceptance boundary. No plaintext +secret, attachment or goods content is mailed. + +`delivery.get@2` has a 64 KiB total inline budget. Larger payloads advertise +`delivery.transfer_open` and bounded `delivery.part_get`. The signed Transfer +entry creates a private buyer-owned File referencing the same BlobRef, then +uses the existing TransferService. Continuations recheck the buyer's current +credential and file ACL; this does not duplicate the payload or expose the +seller's other files. Package, delivery and private evidence references are +SQL-authoritative roots for both backup and garbage collection. + +## Optional checkout email + +No email is required for purchase or site delivery. An existing verified email +must belong to the same buyer and match its endpoint generation. An unverified +checkout address receives only a verification challenge, never order ID, handle, +goods, pickup URL or an authentication capability. The original buyer must also +sign `orders.email_verify` with that proof. This creates an order-only endpoint; +it does not replace their account's email settings. + +After verification, `orders.email_bind` can bind a current verified account +endpoint. `orders.email_disable` disables the selected notification channel and +revokes an order-only endpoint. The worker rechecks buyer/actor, delivery owner, +endpoint owner/generation/address, revocation and (for sealed goods) encryption +key ownership before sending. A stale job cannot overwrite a newly chosen or +disabled endpoint. Seller projections never contain the address. + +The order notification contains exactly `order_id`, checkout `handle` and an +authenticated site `pickup` URL. Status distinguishes disabled, pending, queued, +SMTP acceptance and uncertain delivery. SMTP failure, disabled configuration or +an uncertain response does not undo site delivery or financial settlement. + +## Versioned objective resolution and arbitration + +`orders.dispute_open` first checks objective faults. Otherwise it creates one +private Case per order. Statements and immutable file evidence are submitted +through `orders.dispute_statement` / `orders.dispute_evidence` with `panel` or +`parties` visibility. `orders.dispute_get` and `orders.dispute_evidence_get` enforce +those roles; unauthorized and missing objects return the same error. Public +`orders.dispute_summary` exposes only random case ID, state, policy digest and +the executed outcome, not participants, order ID, votes, testimony or payloads. + +The builtin `dispute-v1` has no arbitrators. It safely holds subjective cases +rather than choosing an administrator or AI. Before offering arbitrated goods, +the operator uses a physical console (preview, typed digest approval and Root +PIN) to grant registered arbitrators and publish an immutable policy: + +``` +msgd --config-dir /etc/msg.lmm.best market grant @arbitrator +msgd --config-dir /etc/msg.lmm.best market publish /secure/policy.json +msgd --config-dir /etc/msg.lmm.best market revoke @arbitrator +``` + +A policy has exactly the keys in `msg.market.policy.DEFAULT_POLICY`; copy that +shape, use a new ID, actual candidate subject IDs and explicit durations. There +are no evaluation expressions, scripts, model calls or hidden fallback members. +A grant/regrant has a new epoch; orders snapshot the active epochs at checkout. +The policy's candidate set and SHA256(order, policy, round, subject) ordering +select the panel. Buyer, seller, declared conflicts and currently invalid roles +are excluded. `orders.arbitrator_conflict` is a signed self-declaration. It does +not claim knowledge of undisclosed real-world relationships. + +Panel size is bounded and quorum is a strict majority. A member signs the exact +proposal with purpose `arbitration-decision`, then submits it via +`orders.dispute_vote`. Each member has one immutable vote per round. Votes are +facts, never ledger writes. A matching quorum forms a release/refund/split +Decision; `orders.dispute_execute` or the due worker revalidates signatures, +credential ceilings, grant epochs, conflicts, round, policy and deadlines before +the EscrowEngine consumes it. Conflicting, expired, revoked or stale decisions +cannot pay again. + +The explicit member-invalidation policy is **hold-no-replacement-v1**: a panel +that loses authority is held, not silently reselected. A policy may permit one +appeal with a disjoint panel selected from the same original candidate snapshot; +there is no second appeal. Old-round decisions become unusable. If no valid +quorum or eligible appeal panel exists, funds remain held and doctor reports the +held-case count. Arbitrators cannot override this by directly moving money. + +## Operations and verification + +Doctor's `market` check is read-only: account separation, escrow balances, +contract/policy digests, settlement conservation and consumed decisions. The +isolated `market_lifecycle` selftest creates disposable identities and a Test +Root, registers/funds a bank with 20 MSG, awards the buyer 10 MSG for a real PoP, +then buys the fixed 5 MSG bundle (`msg.lmm.best store selftest`, `hello.txt` = +`delivery-ok\n`). It checks prepared-not-claimed delivery, an in-memory SMTP sink, +verification-only email, one real signed panel refund, final bank 15/buyer 5 and +zero escrow. It never sends to a real address and cleans its namespace. + +Regression files: `test_market_lifecycle.py`, `test_market_delivery.py`, +`test_market_arbitration.py`, `test_market_recovery.py`, plus the existing order, +ledger, worker, backup and transport suites. Recovery tests use a real backup and +restore; a recovery drill still blocks outbound effects and timer execution. +The migration classifies all new authoritative identity references while retaining +the fail-closed guard for unknown tables. It does not rewrite old receipts. + +Existing credentials' finite ceilings do not expand merely because operations +were installed. Follow the existing local authority review/reissuance procedure +before allowing new operations. Test success is not production deployment, +real-SMTP acceptance, a production schema rehearsal or permission to mint money. diff --git a/src/msg/admin/backups.py b/src/msg/admin/backups.py index 6c87843a..6dcbc435 100644 --- a/src/msg/admin/backups.py +++ b/src/msg/admin/backups.py @@ -43,6 +43,10 @@ def _db_refs(connection): content=loads(raw)['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') contents[content['digest'][7:]]=content['size'] + from msg.market.references import content_references + for content in content_references(connection.execute): + require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') + contents[content['digest'][7:]]=content['size'] live={row[0] for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL')} require(live.issubset(set(revisions)), 'backup_revision_missing') return {'contents':contents, 'revisions':revisions} diff --git a/src/msg/admin/diagnostics.py b/src/msg/admin/diagnostics.py index eda09043..25f18763 100644 --- a/src/msg/admin/diagnostics.py +++ b/src/msg/admin/diagnostics.py @@ -169,6 +169,14 @@ def failed(name,code,**data):checks[name]={'ok':False,'code':code,**data} async def inspect(tx): require(tx.one('SELECT version FROM schema_version')[0]==1,'schema_version_unknown') success('storage') + if 'orders' in settings.server.plugins: + try: + from msg.admin.market_check import inspect_market + success('market',**(await inspect_market(tx))) + except (Failure,KeyError,ValueError,psycopg.Error) as exc: + failed('market',getattr(exc,'code','market_inspection_failed')) + else: + checks['market']={'ok':True,'status':'disabled'} try: from msg.core.requests import SECRET_DELIVERY_MIN_VERSION columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " @@ -600,6 +608,8 @@ async def approve(key,uid,grants): checks['identity_upgrade_recovery']=await check_upgrade_recovery(app,now) from msg.admin.token_delivery_check import check_token_delivery checks['credential_delivery_recovery']=await check_token_delivery(app,now) + from msg.admin.market_check import check_market + checks['market_lifecycle']=await check_market(app,root,call,register,now) except Failure as exc: checks['failure']={'code':exc.code} finally: diff --git a/src/msg/admin/market.py b/src/msg/admin/market.py new file mode 100644 index 00000000..1bc971f0 --- /dev/null +++ b/src/msg/admin/market.py @@ -0,0 +1,91 @@ +"""Root-signed, physical-console arbitration configuration; never a network API.""" +from __future__ import annotations + +import asyncio +import getpass + +from msg.admin.money import _root_signer +from msg.admin.root import RootAdmin, require_local_console, root_envelope +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, loads, wire +from msg.core.errors import require +from msg.core.models import AuditEvent, Event, ResourceRef +from msg.market.policy import DEFAULT_POLICY, validate +from msg.plugins.common import new_id +from msg.security.crypto import Ed25519Signer, open_private_key + + +async def apply_market(app, signer, *, action, operator, subject=None, policy=None, expected=None): + _root_signer(app, signer) + require(action in {'grant','revoke','publish'}, 'market_admin_action_invalid') + if action == 'publish': + policy = validate(policy) + require(policy['id'] != DEFAULT_POLICY['id'], 'builtin_policy_immutable') + async with app.metadata.transaction(write=True) as tx: + if action == 'publish': + require(tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, + 'arbitration_policy_exists') + for candidate in policy['candidates']: + identity = await tx.subject(candidate) + require(identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator') + tx.execute('INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', + (policy['id'],canonical(policy).decode(),digest(policy)), write=True) + before, after = None, policy + else: + identity = await tx.subject(subject) + require(identity.kind == 'registered' and not identity.local_only and subject != ROOT_SUBJECT, + 'invalid_arbitrator') + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (subject,)) + before = {'epoch':row[0],'active':row[1]} if row else None + if expected is not None: + require(before == expected, 'market_preview_stale') + require((action=='grant' and (not row or not row[1])) or + (action=='revoke' and row and row[1]), 'arbitrator_role_unchanged') + after = {'epoch': new_id('role') if action=='grant' else row[0], 'active': action=='grant'} + tx.execute('''INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) + ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active''', + (subject,after['epoch'],after['active']), write=True) + statement = {'action':action,'subject':subject,'before':before,'after':after, + 'operator':operator,'at':wire(app.clock())} + proof = signer.sign(canonical(statement), purpose='market-admin') + event = Event(id=new_id('audit'), type='root.market.'+action, time=app.clock(), + request_id=new_id('local-market'), actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), data={**statement,'root_signature':wire(proof)}) + await tx.append_event(event) + await tx.append_audit(AuditEvent(event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(before),after_digest=digest(after),previous_digest=None, + entry_digest='',result=action)) + return {'action':action,'audit_event_id':event.id,'result':after} + + +class MarketAdmin: + def __init__(self, config_dir): + self.config_dir = config_dir + + def execute(self, action, *, subject=None, policy_file=None): + operator = require_local_console(self.config_dir) + app = RootAdmin(self.config_dir)._app() + async def run(): + await app.load() + try: + policy = loads(policy_file.read_bytes()) if policy_file is not None else None + resolved, before = subject, None + async with app.metadata.transaction(write=False) as tx: + if subject and subject.startswith('@'): + resolved = await tx.resolve('/'+subject) + if resolved: + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,)) + before = {'epoch':row[0],'active':row[1]} if row else None + preview = {'action':action,'subject':resolved,'before':before,'policy':policy} + print(canonical(preview).decode()) + approval = 'CONFIRM MARKET '+digest(preview) + require(input('Type '+approval+' to continue: ') == approval, 'approval_cancelled') + pin = getpass.getpass('Root PIN/passphrase: ') + signer = Ed25519Signer.from_bytes(open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), pin)) + return await apply_market(app, signer, action=action, operator=operator, + subject=resolved,policy=policy,expected=before) + finally: + await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/market_check.py b/src/msg/admin/market_check.py new file mode 100644 index 00000000..5be84e50 --- /dev/null +++ b/src/msg/admin/market_check.py @@ -0,0 +1,146 @@ +"""Read-only financial invariants and a deliberately isolated Test Root scenario.""" +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace + +from msg.core.codec import b64, canonical, digest, loads, wire +from msg.core.errors import require +from msg.core.models import MailConfig +from msg.market.policy import contract, validate +from msg.plugins.money import _balance + + +async def inspect_market(tx): + """No initialization, repair, timer, statement, event, job or network call.""" + count = 0 + for oid,buyer,total,escrow,state in tx.rows( + 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id'): + expected = total if state in {'funded','delivered','accepted','disputed'} else 0 + require(_balance(tx,escrow)==expected, 'market_escrow_invariant') + require(tx.one('SELECT 1 FROM identities WHERE id=?',(escrow,)) is None and + tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?',(escrow,))==('order_escrow',None), + 'market_escrow_identity') + snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?',(oid,)) + if snapshot: + locked = contract(tx,oid) + policy = locked['policy'] + validate(policy['policy']) + require(digest(policy['policy'])==policy['policy_digest'] and + locked['buyer']==buyer and locked['total_price_minor']==total, 'market_snapshot_invariant') + row = tx.one('SELECT body FROM order_settlements WHERE order_id=?',(oid,)) + if row: + fact = loads(row[0]) + require(state in {'settled','refunded'} and fact['refund_minor']+fact['release_minor']==total, + 'market_settlement_invariant') + actual = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?",(escrow,))[0] + require(actual==total,'market_settlement_invariant') + elif snapshot: + require(state not in {'settled','refunded'},'market_settlement_missing') + count += 1 + for raw,expected in tx.rows('SELECT body,digest FROM arbitration_policies'): + policy = loads(raw); validate(policy) + require(digest(policy)==expected,'arbitration_policy_corrupt') + for order_id,state in tx.rows('SELECT order_id,state FROM arbitration_cases'): + settlement = tx.one('SELECT decision_id FROM order_settlements WHERE order_id=?',(order_id,)) + if state=='executed': + require(settlement is not None and settlement[0] is not None,'market_decision_unfunded') + return {'orders':count,'held_cases':tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], + 'escrow_invariants':True,'private_reads_only':True} + + +async def check_market(app,root,call,register,now): + """20 MSG issue -> bank-funded 10 reward -> 5 bundle, entirely disposable.""" + from msg.admin.market import apply_market + from msg.admin.money import apply_money + from msg.market.policy import DEFAULT_POLICY + from msg.workers.effects import EffectWorker + require(app.selftest_run_id is not None,'selftest_namespace_required') + bank,bank_id = await register('market-test-bank') + buyer,buyer_id = await register('market-test-buyer') + async def invoke(name,args,key=buyer,subject=buyer_id,**kw): + result = await call(name,args,key,subject,**kw) + require(result.status=='ok',result.error.code if result.error else 'market_selftest_failed') + return result + await apply_money(app,root,action='bank_add',operator='isolated-selftest',subject_id=bank_id) + await apply_money(app,root,action='mint',operator='isolated-selftest',amount_minor=20_000_000) + await apply_money(app,root,action='transfer',operator='isolated-selftest',subject_id=bank_id,amount_minor=20_000_000) + bounty = await invoke('bounty.create',{'name':'identity-test','terms':'One-use signature test', + 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1},bank,bank_id) + challenge = await invoke('bounty.challenge',{'listing_id':bounty.data['bounty']['listing_id']}) + payload = challenge.data['challenge'] + await invoke('bounty.claim',{'challenge_id':payload['challenge_id'], + 'proof':wire(buyer.sign(canonical(payload),purpose='bounty-pop-v1'))}) + listing = await invoke('store.listing_create',{'name':'store-selftest','item_kind':'bundle', + 'price_minor':5_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'managed_instant', + 'escrow_policy':'escrow-v1','dispute_policy':'dispute-v1','terms':'Immutable test bundle.'},bank,bank_id) + async with app.metadata.transaction(write=False) as tx: + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(bank_id,))[0] + file = await invoke('content.file_put',{'parent':parent,'name':'hello.txt','media_type':'text/plain', + 'data':b64(b'delivery-ok\n')},bank,bank_id) + package = await invoke('store.package_deposit',{'listing_id':listing.resources[0].id, + 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, + 'payload_refs':[wire(file.resources[0])]},bank,bank_id) + active = await invoke('store.listing_update',{'id':listing.resources[0].id,'state':'active', + 'package_ref':package.data['package']['id']},bank,bank_id, + expected=((listing.resources[0].id,listing.data['generation']),)) + old_settings = app.settings + class NoNetworkMail: + def __init__(self): self.messages=[] + async def send(self,job): + self.messages.append(dict(job.arguments)); return 'sent' + sender=NoNetworkMail() + app.settings=replace(app.settings,server=replace(app.settings.server,mail=MailConfig(enabled=True, + host='selftest.invalid',port=2525,tls='starttls',sender='test@selftest.invalid',credential_file=None))) + try: + sale = active.data['listing'] + bought = await invoke('orders.buy',{'listing_id':sale['listing_id'], + 'listing_revision':sale['listing_revision'],'quantity':1,'total_price_minor':5_000_000, + 'currency_id':'primary','email':'buyer@selftest.invalid'},contract_version=2) + oid = bought.data['order']['id'] + worker=EffectWorker(app,mail_sender=sender) + while await worker.run_once(): pass + require(len(sender.messages)==1 and oid not in sender.messages[0]['text'],'market_email_leak') + proof=loads(sender.messages[0]['text']) + await invoke('orders.email_verify',{'order_id':oid,'challenge_id':proof['challenge_id'],'token':proof['token']}) + while await worker.run_once(): pass + require(len(sender.messages)==2 and set(loads(sender.messages[1]['text']))=={'order_id','handle','pickup'}, + 'market_email_leak') + delivery=await invoke('delivery.get',{'order_id':oid},contract_version=2) + require(delivery.data['delivery']['state']=='prepared' and + delivery.data['delivery']['payloads'][0]['data']==b64(b'delivery-ok\n'),'market_claim_invariant') + finally: + app.settings=old_settings + # A separate 1 MSG service dispute refunds the test buyer through a real + # fixed 2-of-3 panel. It leaves the official 5/15 final balances unchanged. + members={} + for i in range(3): + key,uid=await register('market-test-arbitrator-'+str(i)); members[uid]=key + await apply_market(app,root,action='grant',operator='isolated-selftest',subject=uid) + policy=deepcopy(DEFAULT_POLICY); policy.update(id='selftest-panel-v1',candidates=list(members)) + await apply_market(app,root,action='publish',operator='isolated-selftest',policy=policy) + service=await invoke('store.listing_create',{'name':'case-selftest','item_kind':'service', + 'price_minor':1_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'service', + 'escrow_policy':'escrow-v1','dispute_policy':policy['id'],'terms':'Test service dispute.'},bank,bank_id) + service=await invoke('store.listing_update',{'id':service.resources[0].id,'state':'active'},bank,bank_id, + expected=((service.resources[0].id,service.data['generation']),)) + sale=service.data['listing'] + bought=await invoke('orders.buy',{'listing_id':sale['listing_id'],'listing_revision':sale['listing_revision'], + 'quantity':1,'total_price_minor':1_000_000,'currency_id':'primary'},contract_version=2) + opened=await invoke('orders.dispute_open',{'order_id':bought.data['order']['id'],'reason':'quality'}) + case=await invoke('orders.dispute_get',{'case_id':opened.data['case_id']}) + proposal={**case.data['proposal_base'],'outcome':'refund','refund_minor':1_000_000} + for uid in opened.data['panel'][:2]: + voted=await invoke('orders.dispute_vote',{'proposal':proposal, + 'signature':wire(members[uid].sign(canonical(proposal),purpose='arbitration-decision'))},members[uid],uid) + args={'case_id':opened.data['case_id'],'decision_id':voted.data['decision']['id']} + await invoke('orders.dispute_execute',args) + await invoke('orders.dispute_execute',args) + async with app.metadata.transaction(write=False) as tx: + await inspect_market(tx) + require(_balance(tx,buyer_id)==5_000_000 and _balance(tx,bank_id)==15_000_000 and + tx.one("SELECT SUM(amount_minor) FROM money_ledger WHERE kind='mint'")[0]==20_000_000, + 'market_conservation_failed') + for (escrow,) in tx.rows("SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')"): + require(_balance(tx,escrow)==0,'market_escrow_invariant') + return True diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 90f479fc..d5070877 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -118,6 +118,11 @@ def parser(): offer_set.add_argument('--max-quantity',type=int,required=True) offer_set.add_argument('--duration-seconds',type=int) ops.add_parser('disable').add_argument('offer_id') + market=sub.add_parser('market',help='Physical-console arbitration configuration') + market_sub=market.add_subparsers(dest='market_command',required=True) + market_sub.add_parser('grant').add_argument('subject_id') + market_sub.add_parser('revoke').add_argument('subject_id') + market_sub.add_parser('publish').add_argument('policy_file',type=Path) backup=sub.add_parser('backup',help='Local service-data backup, excluding root private material') backup.add_argument('destination',type=Path) restore=sub.add_parser('restore',help='Restore service data into new directories only') @@ -162,6 +167,11 @@ def main(argv=None): elif args.root_command=='backup':result=admin.backup(args.destination) else:result=admin.recover(args.source) emit(result);return 0 + if args.command=='market': + from msg.admin.market import MarketAdmin + emit(MarketAdmin(args.config_dir).execute(args.market_command, + subject=getattr(args,'subject_id',None),policy_file=getattr(args,'policy_file',None))) + return 0 if args.command=='money': from msg.admin.money import MoneyAdmin if args.money_command=='offer': diff --git a/src/msg/data/bootstrap.json b/src/msg/data/bootstrap.json index 4e21df00..49f32332 100644 --- a/src/msg/data/bootstrap.json +++ b/src/msg/data/bootstrap.json @@ -43,6 +43,7 @@ }, "features": [ {"feature_id":"bootstrap","enabled_by_default":true,"default_config":{},"sample_resource":"r_root","doctor_check":"bootstrap","selftest_case":"isolated_namespace"}, + {"feature_id":"market","enabled_by_default":true,"default_config":{"checkout_version":2,"production_fixtures":false,"arbitrators":[]},"sample_resource":null,"doctor_check":"market","selftest_case":"market_lifecycle"}, {"feature_id":"postgres","enabled_by_default":true,"default_config":{},"sample_resource":"r_root","doctor_check":"storage","selftest_case":"stable_id_read"}, {"feature_id":"root_trust","enabled_by_default":true,"default_config":{},"sample_resource":"u_root","doctor_check":"root_trust","selftest_case":"root_network_rejected"}, {"feature_id":"credential_delivery","enabled_by_default":true,"default_config":{"credential_delivery_recovery_window":"15m","release":"once","secret_url":false},"sample_resource":null,"doctor_check":"credential_delivery","selftest_case":"credential_delivery_recovery"}, diff --git a/src/msg/market/__init__.py b/src/msg/market/__init__.py new file mode 100644 index 00000000..619af723 --- /dev/null +++ b/src/msg/market/__init__.py @@ -0,0 +1 @@ +"""Versioned market facts; only the executor/worker owns SQL transactions.""" diff --git a/src/msg/market/arbitration.py b/src/msg/market/arbitration.py new file mode 100644 index 00000000..252091a7 --- /dev/null +++ b/src/msg/market/arbitration.py @@ -0,0 +1,465 @@ +"""Deterministic panels and signed, quorum-bound escrow decisions. + +A Case never holds a key or balance. A panel can attest only one allocation per +member/round. The EscrowEngine consumes the current, revalidated Decision once. +""" +from __future__ import annotations + +from datetime import timedelta + +from msg.core.codec import canonical, decode, digest, loads, parse_time, wire +from msg.core.errors import Failure, require +from msg.core.models import BlobRef, HandlerOutput, Principal, ResourceRef, Signature +from msg.market.orders import HASH, view +from msg.market.policy import contract +from msg.plugins.common import check_access, new_id +from msg.plugins.orders import _row, _subject +from msg.plugins.schemas import IDENTIFIER, REF, SIGNATURE, obj +from msg.security.crypto import verify + +REASONS = ('quality','not_as_described','wrong_ciphertext','delivery_missing', + 'delivery_digest_mismatch','package_missing','package_digest_mismatch', + 'delivery_timeout','other') +CASE = obj({'case_id': IDENTIFIER}, ('case_id',)) +PROPOSAL = obj({'case_id': IDENTIFIER, 'order_id': IDENTIFIER, + 'round': {'type':'integer','minimum':0,'maximum':1}, 'policy_digest': HASH, + 'outcome': {'enum':['release','refund','split']}, + 'refund_minor': {'type':'integer','minimum':0,'maximum':2**63-1}, + 'expires_at': {'type':'string','maxLength':40}}, + ('case_id','order_id','round','policy_digest','outcome','refund_minor','expires_at')) + + +def _read(tx, case_id): + row = tx.one('SELECT body,round,state,deadline FROM arbitration_cases WHERE id=?', (case_id,)) + require(row is not None, 'case_not_found') + case = loads(row[0]) + case.update(round=row[1], state=row[2], deadline=row[3]) + return case + + +def _conflict(tx, candidate, case): + return candidate in {case['buyer'],case['seller']} or tx.one('''SELECT 1 + FROM arbitrator_conflicts WHERE arbitrator=? AND party IN (?,?)''', + (candidate,case['buyer'],case['seller'])) is not None + + +async def member_valid(tx, candidate, case): + epoch = case['policy']['candidate_epochs'].get(candidate) + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (candidate,)) + if not epoch or not row or not row[1] or row[0] != epoch or _conflict(tx, candidate, case): + return False + try: + subject, resource = await tx.subject(candidate), await tx.resource(candidate) + return not subject.local_only and subject.kind == 'registered' and resource.state == 'active' + except Failure: + return False + + +async def panel_for(tx, case, round_number): + eligible = [] + previous = set(case.get('panels', {}).get('0', ())) if round_number else set() + for candidate in case['policy']['policy']['candidates']: + if candidate not in previous and await member_valid(tx, candidate, case): + eligible.append(candidate) + def score(candidate): + return digest({'version': 1, 'order_id': case['order_id'], + 'policy_digest': case['policy']['policy_digest'], 'round': round_number, + 'subject': candidate}) + return sorted(eligible, key=lambda candidate: (score(candidate),candidate))[ + :case['policy']['policy']['panel_size']] + + +async def panel_valid(tx, case): + panel = case['panels'][str(case['round'])] + return (len(panel) == case['policy']['policy']['panel_size'] and + all([await member_valid(tx, candidate, case) for candidate in panel])) + + +async def owned_case(tx, ctx, case_id, *, parties=False): + viewer = ctx.principal.subject + require(viewer is not None and ctx.principal.actor == viewer, 'case_not_found') + case = _read(tx, case_id) + if viewer in {case['buyer'],case['seller']}: + return case + if not parties and viewer in case['panels'][str(case['round'])] and await member_valid(tx, viewer, case): + return case + raise Failure('case_not_found') + + +def proposal_base(case): + policy = case['policy']['policy'] + deadline = parse_time(case['round_deadlines'][str(case['round'])]) + return {'case_id': case['id'], 'order_id': case['order_id'], 'round': case['round'], + 'policy_digest': case['policy']['policy_digest'], + 'expires_at': wire(deadline+timedelta(seconds=policy['decision_lifetime_seconds']))} + + +def proposal_valid(case, proposal, total): + base = proposal_base(case) + require(set(proposal) == {*base,'outcome','refund_minor'} and + all(proposal[k] == v for k,v in base.items()), 'decision_context_mismatch') + refund, outcome = proposal['refund_minor'], proposal['outcome'] + require(type(refund) is int and ((outcome=='release' and refund==0) or + (outcome=='refund' and refund==total) or (outcome=='split' and 0 now, 'decision_expired') + require(parse_time(decision['not_before']) <= now, 'appeal_window_open') + require(await panel_valid(tx, case), 'decision_panel_invalid') + quorum = case['policy']['policy']['quorum'] + require(len(decision['voters']) == quorum and len(set(decision['voters'])) == quorum and + set(decision['voters']) <= set(case['panels'][str(case['round'])]), 'decision_quorum_invalid') + cast_times = [] + for arbitrator in decision['voters']: + vote_row = tx.one('SELECT body FROM arbitration_votes WHERE case_id=? AND round=? AND arbitrator=?', + (case['id'],case['round'],arbitrator)) + require(vote_row is not None, 'decision_quorum_invalid') + vote = loads(vote_row[0]) + require(vote['proposal'] == decision['proposal'], 'decision_conflicting_vote') + await verify_vote(app, tx, case, vote) + cast_times.append(parse_time(vote['cast_at'])) + delay = (case['policy']['policy']['appeal_seconds'] + if case['round']==0 and case['policy']['policy']['allow_appeal'] else 0) + require(parse_time(decision['not_before']) == max(cast_times)+timedelta(seconds=delay) and + decision['expires_at'] == decision['proposal']['expires_at'], 'decision_context_mismatch') + + +def notice(tx, case, state, now): + # Private inbox references only. No testimony, vote, email or payload escapes. + for recipient in {case['buyer'],case['seller']}: + record = {'id': new_id('message'), 'sender': recipient, 'actor': recipient, + 'recipient': recipient, 'resource': wire(ResourceRef(id=recipient)), + 'order_id': case['order_id'], 'case_id': case['id'], + 'time': wire(now), 'state': state, 'source': 'system'} + tx.execute('INSERT INTO messages VALUES (?,?,?,?,?,?)', + (record['id'],recipient,recipient,recipient,'case:'+case['id'],canonical(record).decode()), write=True) + + +async def objective_fault(app, tx, order, now): + locked = contract(tx, order['id']) + if order['state'] == 'funded': + deadline = parse_time(order['funded_at']) + timedelta( + seconds=locked['policy']['policy']['delivery_timeout_seconds']) + if now >= deadline: + return 'delivery_timeout' + if locked['listing']['delivery_mode'] == 'managed_instant': + from msg.market.delivery import package + try: + await package(app, tx, order) + except Failure as exc: + if exc.code in {'package_missing','package_digest_mismatch'}: + return exc.code + raise + if order['state'] == 'delivered': + from msg.market.delivery import verified + try: + await verified(app, tx, order) + except Failure as exc: + if exc.code == 'delivery_not_found': + return 'delivery_missing' + if exc.code in {'delivery_digest_mismatch','package_missing','package_digest_mismatch'}: + return exc.code + raise + return None + + +async def execute(app, tx, case, decision_id, now, actor, request_id): + from msg.market.escrow import settle + order = _row(tx, case['order_id'], case['buyer']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=? AND round=?', + (decision_id,case['id'],case['round'])) + require(row is not None, 'decision_not_current') + decision = loads(row[0]) + existing = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (order['id'],)) + if existing: + settlement = loads(existing[0]) + require(settlement['decision_id'] == decision_id, 'order_already_settled') + return settlement['receipts'] + receipts = await settle(app, tx, order, now=now, actor=actor, request_id=request_id, + reason='arbitration_decision', refund_minor=decision['refund_minor'], decision=decision) + notice(tx, case, 'executed', now) + return receipts + + +async def resolve_cases(app, *, limit=100): + changed, now = [], app.clock() + async with app.metadata.transaction(write=True) as tx: + for (case_id,) in tx.rows('''SELECT id FROM arbitration_cases + WHERE state IN ('open','decided') AND deadline<=? ORDER BY deadline,id LIMIT ?''', + (wire(now),limit)): + case = _read(tx, case_id) + reason = 'quorum_timeout' + if case['state'] == 'decided': + row = tx.one('SELECT id FROM arbitration_decisions WHERE case_id=? AND round=?', + (case_id,case['round'])) + try: + require(row is not None, 'decision_not_current') + await execute(app, tx, case, row[0], now, case['buyer'], 'case-resolve:'+case_id) + changed.append(case_id) + continue + except Failure as exc: + # No unexpected SQL/IO failure is converted into a money rule. + if exc.code not in {'decision_revoked_or_unknown','decision_expired', + 'decision_panel_invalid','decision_member_invalid','credential_revoked', + 'credential_expired','invalid_signature','permission_denied', + 'credential_ceiling','decision_not_current','decision_quorum_invalid', + 'decision_signer_mismatch','decision_conflicting_vote','decision_context_mismatch'}: + raise + reason = exc.code + tx.execute("UPDATE arbitration_cases SET state='held' WHERE id=?", (case_id,), write=True) + tx.set_setting('case_hold:'+case_id, {'reason': reason, 'at': wire(now)}) + notice(tx, case, 'held', now) + changed.append(case_id) + return changed + + +def install(app, op): + @op('orders.dispute_open', obj({'order_id': IDENTIFIER, + 'reason': {'enum': list(REASONS)}}, ('order_id','reason')), signature=True) + async def open_case(ctx, request, tx): + from msg.market.escrow import settle, transition + actor = _subject(ctx) + order = _row(tx, request.arguments['order_id'], actor) + locked = contract(tx, order['id']) + existing = tx.one('SELECT id FROM arbitration_cases WHERE order_id=?', (order['id'],)) + if existing: + return HandlerOutput(data={'case_id': existing[0], 'order_id': order['id']}) + require(order['state'] in {'funded','delivered'}, 'order_not_disputable') + objective = await objective_fault(app, tx, order, ctx.now) + if objective: + receipts = await settle(app, tx, order, now=ctx.now, actor=actor, + request_id=request.request_id, reason=objective, refund_minor=order['total_price_minor']) + return HandlerOutput(data={'resolution': 'objective_refund', 'reason': objective, + 'order': view(tx, order, actor), 'receipts': receipts}) + case_id = new_id('case') + deadline = wire(ctx.now+timedelta(seconds=locked['policy']['policy']['case_timeout_seconds'])) + case = {'id': case_id, 'order_id': order['id'], 'buyer': order['buyer'], + 'seller': order['seller'], 'opened_by': actor, 'opened_at': wire(ctx.now), + 'reason': request.arguments['reason'], 'policy': locked['policy'], + 'panels': {}, 'round_deadlines': {'0': deadline}, 'round': 0} + case['panels']['0'] = await panel_for(tx, case, 0) + state = 'open' if await panel_valid(tx, case) else 'held' + tx.execute('INSERT INTO arbitration_cases(id,order_id,round,state,deadline,body) VALUES (?,?,0,?,?,?)', + (case_id,order['id'],state,deadline,canonical(case).decode()), write=True) + await transition(tx, order, 'disputed', now=ctx.now, actor=actor, + request_id=request.request_id, reason=request.arguments['reason']) + notice(tx, case, state, ctx.now) + return HandlerOutput(data={'case_id': case_id, 'order_id': order['id'], 'state': state, + 'panel': case['panels']['0'], 'policy_digest': locked['policy']['policy_digest'], + 'deadline': deadline}) + + @op('orders.dispute_get', CASE, effect='read') + async def get(ctx, request, tx): + case = await owned_case(tx, ctx, request.arguments['case_id']) + viewer = ctx.principal.subject + panel = viewer in case['panels'][str(case['round'])] + evidence = [] + for eid,author,visibility,body in tx.rows('''SELECT id,author,visibility,body + FROM arbitration_evidence WHERE case_id=? ORDER BY id''', (case['id'],)): + if panel or viewer == author or visibility == 'parties': + record = loads(body) + evidence.append({'id': eid,'author': author,'visibility': visibility, + 'kind': record['kind'],'digest': record['blob']['digest'],'size': record['blob']['size']}) + decisions = [loads(row[0]) for row in tx.rows( + 'SELECT body FROM arbitration_decisions WHERE case_id=? ORDER BY round', (case['id'],))] + votes = [loads(row[0]) for row in tx.rows( + 'SELECT body FROM arbitration_votes WHERE case_id=? AND round=? ORDER BY arbitrator', + (case['id'],case['round']))] + return HandlerOutput(data={'case': {k:v for k,v in case.items() if k!='policy'}, + 'policy': case['policy'], 'panel_valid': await panel_valid(tx, case), + 'proposal_base': proposal_base(case), 'evidence': evidence, + 'votes': [{'arbitrator':v['arbitrator'],'proposal':v['proposal'],'signature':v['signature']} + for v in votes], 'decisions': decisions}) + + @op('orders.dispute_summary', CASE, effect='read') + async def summary(ctx, request, tx): + case = _read(tx, request.arguments['case_id']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE case_id=? AND round=?', + (case['id'],case['round'])) + return HandlerOutput(data={'case_id': case['id'], 'state': case['state'], + 'policy_digest': case['policy']['policy_digest'], + 'outcome': loads(row[0])['proposal']['outcome'] if row and case['state']=='executed' else None}) + + evidence_fields = {'case_id': IDENTIFIER, 'visibility': {'enum':['panel','parties']}} + @op('orders.dispute_statement', obj({**evidence_fields, + 'text': {'type':'string','minLength':1,'maxLength':4096}}, ('case_id','visibility','text')), + signature=True) + @op('orders.dispute_evidence', obj({**evidence_fields, 'ref': REF}, + ('case_id','visibility','ref')), signature=True) + async def add_evidence(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id'], parties=True) + require(case['state'] in {'open','held'} and + ctx.now < parse_time(case['round_deadlines'][str(case['round'])]), 'case_evidence_closed') + require(tx.one('SELECT COUNT(*) FROM arbitration_evidence WHERE case_id=? AND author=?', + (case['id'],actor))[0] < 64, 'case_evidence_limit') + from msg.market.delivery import verify_blob + if 'text' in request.arguments: + blob = await app.contents.put_bytes(request.arguments['text'].encode(), 'text/plain') + kind, source = 'statement', None + else: + ref = decode(ResourceRef, request.arguments['ref']) + require(ref.revision is not None, 'payload_revision_required') + resource = await tx.resource(ref.id) + require(resource.owner == actor and resource.type in {'file','attachment'} and + resource.state == 'active', 'evidence_not_owned') + await check_access(app, ctx, request, tx, ref.id, 'read') + blob = (await tx.revision(ref)).content + kind, source = 'resource', wire(ref) + await verify_blob(app, blob) + eid = new_id('evidence') + await app.contents.pin(blob, eid) + record = {'kind': kind, 'blob': wire(blob), 'source': source, 'round': case['round'], + 'at': wire(ctx.now), 'author_request_digest': request.payload_digest} + tx.execute('INSERT INTO arbitration_evidence(id,case_id,author,visibility,body) VALUES (?,?,?,?,?)', + (eid,case['id'],actor,request.arguments['visibility'],canonical(record).decode()), write=True) + return HandlerOutput(data={'evidence_id': eid, 'digest': blob.digest}) + + @op('orders.dispute_evidence_get', obj({'case_id': IDENTIFIER,'evidence_id': IDENTIFIER, + 'offset': {'type':'integer','minimum':0}, + 'length': {'type':'integer','minimum':1,'maximum':65536}}, + ('case_id','evidence_id')), effect='read') + async def evidence_get(ctx, request, tx): + case = await owned_case(tx, ctx, request.arguments['case_id']) + row = tx.one('SELECT author,visibility,body FROM arbitration_evidence WHERE id=? AND case_id=?', + (request.arguments['evidence_id'],case['id'])) + viewer = ctx.principal.subject + require(row is not None and (viewer==row[0] or row[1]=='parties' or + viewer in case['panels'][str(case['round'])]), 'evidence_not_found') + record = loads(row[2]) + blob = decode(BlobRef, record['blob']) + start = request.arguments.get('offset', 0) + end = min(blob.size, start+request.arguments.get('length', 65536)) + require(0 <= start <= end <= blob.size, 'invalid_byte_range') + raw = b''.join([part async for part in app.contents.read(blob,(start,end))]) + from msg.core.codec import b64 + return HandlerOutput(data={'evidence_id': request.arguments['evidence_id'], 'digest': blob.digest, + 'size': blob.size, 'offset': start, 'data': b64(raw), 'chunk_digest': digest(raw), + 'next_offset': end if end= quorum: + voters = sorted(matching, key=lambda v:v['arbitrator'])[:quorum] + delay = (case['policy']['policy']['appeal_seconds'] + if case['round']==0 and case['policy']['policy']['allow_appeal'] else 0) + decision = {'id': 'dec_'+digest(proposal)[7:39], 'case_id': case['id'], + 'order_id': order['id'], 'round': case['round'], 'proposal': proposal, + 'refund_minor': proposal['refund_minor'], 'policy_digest': proposal['policy_digest'], + 'voters': [v['arbitrator'] for v in voters], 'expires_at': proposal['expires_at'], + 'not_before': wire(max(parse_time(v['cast_at']) for v in voters)+timedelta(seconds=delay))} + tx.execute('INSERT INTO arbitration_decisions(id,case_id,round,body) VALUES (?,?,?,?)', + (decision['id'],case['id'],case['round'],canonical(decision).decode()), write=True) + tx.execute("UPDATE arbitration_cases SET state='decided',deadline=? WHERE id=?", + (decision['not_before'],case['id']), write=True) + notice(tx, case, 'decided', ctx.now) + return HandlerOutput(data={'case_id': case['id'], 'votes_for_proposal': len(matching), 'decision': decision}) + + @op('orders.dispute_execute', obj({'case_id': IDENTIFIER,'decision_id': IDENTIFIER}, + ('case_id','decision_id')), signature=True) + async def execute_decision(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id']) + receipts = await execute(app, tx, case, request.arguments['decision_id'], ctx.now, actor, request.request_id) + return HandlerOutput(data={'case_id': case['id'],'state':'executed','receipts':receipts}) + + @op('orders.dispute_appeal', CASE, signature=True) + async def appeal(ctx, request, tx): + _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id'], parties=True) + policy = case['policy']['policy'] + require(policy['allow_appeal'] and case['round']==0 and case['state'] in {'decided','held'}, + 'appeal_not_available') + row = tx.one('SELECT body FROM arbitration_decisions WHERE case_id=? AND round=0', (case['id'],)) + until = (parse_time(loads(row[0])['not_before']) if row else + parse_time(case['round_deadlines']['0'])+timedelta(seconds=policy['appeal_seconds'])) + require(ctx.now < until, 'appeal_window_closed') + case['panels']['1'] = await panel_for(tx, case, 1) + case['round'] = 1 + deadline = wire(ctx.now+timedelta(seconds=policy['case_timeout_seconds'])) + case['round_deadlines']['1'] = deadline + state = 'open' if await panel_valid(tx, case) else 'held' + tx.execute('UPDATE arbitration_cases SET round=1,state=?,deadline=?,body=? WHERE id=?', + (state,deadline,canonical(case).decode(),case['id']), write=True) + notice(tx, case, 'appealed', ctx.now) + return HandlerOutput(data={'case_id':case['id'],'round':1,'state':state,'panel':case['panels']['1']}) + + @op('orders.dispute_revoke', obj({'case_id': IDENTIFIER,'decision_id': IDENTIFIER}, + ('case_id','decision_id')), signature=True) + async def revoke_decision(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=? AND round=?', + (request.arguments['decision_id'],case['id'],case['round'])) + require(row is not None and actor in loads(row[0])['voters'], 'decision_signer_required') + require(case['state'] in {'decided','held'}, 'decision_already_executed') + tx.execute('UPDATE arbitration_decisions SET revoked=TRUE WHERE id=?', + (request.arguments['decision_id'],), write=True) + tx.execute("UPDATE arbitration_cases SET state='held' WHERE id=?", (case['id'],), write=True) + notice(tx, case, 'held', ctx.now) + return HandlerOutput(data={'case_id':case['id'],'state':'held','decision_revoked':True}) + + @op('orders.arbitrator_conflict', obj({'party': IDENTIFIER}, ('party',)), signature=True) + async def declare_conflict(ctx, request, tx): + actor = _subject(ctx) + role = tx.one('SELECT active FROM arbitrator_roles WHERE subject=?', (actor,)) + require(role is not None and role[0], 'arbitrator_required') + party = (await tx.subject(request.arguments['party'])).resource_id + tx.execute('INSERT INTO arbitrator_conflicts(arbitrator,party) VALUES (?,?) ON CONFLICT DO NOTHING', + (actor,party), write=True) + return HandlerOutput(data={'declared':True}) diff --git a/src/msg/market/delivery.py b/src/msg/market/delivery.py new file mode 100644 index 00000000..940468c9 --- /dev/null +++ b/src/msg/market/delivery.py @@ -0,0 +1,338 @@ +"""Automatic managed packages and explicitly accepted ciphertext/service evidence.""" +from __future__ import annotations + +import hashlib + +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import Failure, require +from msg.core.models import BlobRef, HandlerOutput, ResourceRef +from msg.market.escrow import settle, transition +from msg.market.orders import HASH, ORDER +from msg.market.policy import contract +from msg.market.targets import enqueue_notification, validate_target +from msg.plugins.common import check_access, new_id +from msg.plugins.delivery import _buyer_order, _delivery +from msg.plugins.orders import _row, _subject, _viewer +from msg.plugins.schemas import IDENTIFIER, REF, obj +from msg.plugins.store import _package_row + +INLINE = 65536 +PART = 65536 + + +async def verify_blob(app, blob): + """Bounded-memory verification, including content loss after a deposit.""" + hasher, size = hashlib.sha256(), 0 + try: + async for chunk in app.contents.read(blob): + hasher.update(chunk) + size += len(chunk) + except FileNotFoundError: + raise Failure('package_missing') from None + except Failure as exc: + if exc.code == 'content_missing': + raise Failure('package_missing') from None + if exc.code in {'content_truncated','content_size_mismatch'}: + raise Failure('package_digest_mismatch') from None + raise + require(size == blob.size and 'sha256:'+hasher.hexdigest() == blob.digest, + 'package_digest_mismatch') + + +async def package(app, tx, order): + row = await _package_row(tx, order['package_id']) + require(row is not None, 'package_missing') + require(row[1] == order['listing_id'] and row[3] == order['seller'] and + row[4] == order['package_revision'] and row[8] == order['package_digest'] and + row[10] == 'managed_instant' and row[5] in {'file','bundle','text'}, + 'package_digest_mismatch') + refs, manifest = loads(row[7]), loads(row[6]) + snapshot = {'seller': row[3], 'listing_id': row[1], 'listing_revision': row[2], + 'revision': row[4], 'kind': row[5], 'manifest': manifest, 'payload_refs': refs, + 'total_size': row[9], 'delivery_mode': row[10]} + require(digest(snapshot) == order['package_digest'] and 0 <= row[9] <= 1024**4, + 'package_digest_mismatch') + require(1 <= len(refs) <= 32, 'package_digest_mismatch') + total = 0 + for ref in refs: + blob = decode(BlobRef, ref['blob']) + await verify_blob(app, blob) + total += blob.size + require(total == row[9], 'package_digest_mismatch') + return row[5], manifest, refs + + +def body_for(order, delivery_id, kind, manifest, refs): + return {'delivery_id': delivery_id, 'order_id': order['id'], + 'recipient_subject': order['buyer'], 'kind': kind, 'manifest': manifest, + 'payload_refs': refs, 'package_digest': order['package_digest'] or + contract_digest(order), 'channel': 'site'} + + +def contract_digest(order): + # Non-managed goods have no deposited package. This is a terms commitment, + # not a false claim that the service/ciphertext content was fixed at checkout. + return digest({'listing_revision': order['listing_revision'], 'terms_digest': order['terms_digest']}) + + +async def prepare(app, tx, ctx, request, order, kind, manifest, refs, *, envelope=None): + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_deliverable') + validate_target(tx, order, encryption=envelope is not None) + delivery_id = new_id('dlv') + for ref in refs: + await app.contents.pin(decode(BlobRef, ref['blob']), delivery_id) + body = body_for(order, delivery_id, kind, manifest, refs) + tx.execute('''INSERT INTO store_deliveries + (id,order_id,recipient_subject,kind,payload_refs,manifest,package_digest, + delivery_digest,channel,state,prepared_at,claimed_at,receipt) + VALUES (?,?,?,?,?,?,?,?,'site','prepared',?,NULL,NULL)''', + (delivery_id,order['id'],order['buyer'],kind,canonical(refs).decode(), + canonical(manifest).decode(),body['package_digest'],digest(body),wire(ctx.now)), write=True) + if envelope: + tx.execute('INSERT INTO delivery_envelopes(order_id,body) VALUES (?,?)', + (order['id'],canonical(envelope).decode()), write=True) + await transition(tx, order, 'delivered', now=ctx.now, actor=ctx.principal.actor, + request_id=request.request_id, reason='verified_delivery') + tx.execute('UPDATE store_orders SET delivered_at=? WHERE id=?', + (wire(ctx.now),order['id']), write=True) + order['delivered_at'] = wire(ctx.now) + # Inbox is a minimal reference to the one authoritative order, not a copy. + message = {'id': new_id('message'), 'sender': order['seller'], 'actor': order['seller'], + 'recipient': order['buyer'], 'order_id': order['id'], 'time': wire(ctx.now), + 'resource': wire(ResourceRef(id=order['buyer'])), 'state': 'delivered', 'source': 'system', + 'href': '/_orders/'+order['id']+'/_delivery'} + tx.execute('INSERT INTO messages VALUES (?,?,?,?,?,?)', + (message['id'],order['seller'],order['buyer'],order['buyer'], + 'delivery:'+order['id'],canonical(message).decode()), write=True) + await enqueue_notification(app, tx, ctx, request, order) + return _delivery(tx, order['id']) + + +async def automatic(app, tx, ctx, request, order): + """An authenticated payment authorizes the pinned managed-instant policy. + + Only verified objective storage faults refund. An arbitrary exception rolls + back the executor transaction, rather than turning software bugs into payouts. + """ + try: + kind, manifest, refs = await package(app, tx, order) + except Failure as exc: + if exc.code not in {'package_missing','package_digest_mismatch'}: + raise + await settle(app, tx, order, now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason=exc.code, refund_minor=order['total_price_minor']) + return + await prepare(app, tx, ctx, request, order, kind, manifest, refs) + await transition(tx, order, 'accepted', now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='managed_instant_verified') + await settle(app, tx, order, now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='managed_instant_verified') + # Delivery remains prepared, not claimed. Server fulfilment is not buyer ACK. + + +async def verified(app, tx, order, *, verify_bytes=True): + delivery = _delivery(tx, order['id']) + require(delivery is not None, 'delivery_not_found') + validate_target(tx, order) + require(delivery['recipient_subject'] == order['buyer'], 'delivery_recipient_mismatch') + expected = body_for(order, delivery['id'], delivery['kind'], + delivery['manifest'], delivery['payload_refs']) + require(delivery['package_digest'] == expected['package_digest'] and + delivery['delivery_digest'] == digest(expected), 'delivery_digest_mismatch') + locked = contract(tx, order['id']) + if locked['listing']['delivery_mode'] == 'managed_instant': + row = await _package_row(tx, order['package_id']) + require(row is not None and row[8] == order['package_digest'] and + loads(row[6]) == delivery['manifest'] and loads(row[7]) == delivery['payload_refs'], + 'delivery_digest_mismatch') + elif locked['listing']['delivery_mode'] == 'sealed_manual': + envelope = tx.one('SELECT body FROM delivery_envelopes WHERE order_id=?', (order['id'],)) + require(envelope is not None, 'delivery_digest_mismatch') + envelope = loads(envelope[0]) + require(envelope['recipient_subject'] == order['buyer'] and + envelope['recipient_key'] == locked['recipient_key'] and + envelope['ciphertext_ref'] == delivery['payload_refs'][0], + 'delivery_recipient_mismatch') + if verify_bytes: + for ref in delivery['payload_refs']: + await verify_blob(app, decode(BlobRef, ref['blob'])) + return delivery + + +async def read(app, tx, ctx, request): + buyer = _viewer(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + delivery = await verified(app, tx, order) + payloads, budget = [], INLINE + for index, ref in enumerate(delivery['payload_refs']): + blob = decode(BlobRef, ref['blob']) + item = {'id': ref['id'], 'revision': ref['revision'], 'digest': blob.digest, + 'size': blob.size, 'media_type': blob.media_type, 'payload_index': index} + if blob.size <= budget: + item['data'] = b64(await app.contents.read_bytes(blob, limit=INLINE)) + budget -= blob.size + else: + item['transfer'] = {'operation':'delivery.transfer_open','arguments':{ + 'order_id':order['id'],'delivery_digest':delivery['delivery_digest'], + 'payload_index':index}} + item['next'] = {'operation': 'delivery.part_get', 'arguments': { + 'order_id': order['id'], 'delivery_digest': delivery['delivery_digest'], + 'payload_index': index, 'offset': 0, 'length': min(PART,blob.size)}} + payloads.append(item) + result = {k:v for k,v in delivery.items() if k not in {'id','payload_refs','receipt'}} + result.update(delivery_id=delivery['id'], payloads=payloads) + return HandlerOutput(data={'delivery': result}) + + +async def accept(app, tx, ctx, request): + buyer = _subject(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + delivery = await verified(app, tx, order) + require(delivery['delivery_digest'] == request.arguments['delivery_digest'], 'delivery_mismatch') + require(order['state'] in {'delivered','settled'}, 'delivery_not_acceptable') + if delivery['state'] == 'claimed': + return HandlerOutput(data={'order_id': order['id'], 'state': order['state'], + 'delivery_id': delivery['id'], 'receipt': delivery['receipt']}) + settlement = [] + if order['state'] == 'delivered': + await transition(tx, order, 'accepted', now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance') + settlement = await settle(app, tx, order, now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance') + body = {'order_id': order['id'], 'delivery_id': delivery['id'], 'subject': buyer, + 'delivery_digest': delivery['delivery_digest'], 'request_digest': request.payload_digest, + 'claimed_at': wire(ctx.now)} + receipt = {'body': body, + 'signature': wire(app.receipt_signer.sign(canonical(body), purpose='delivery-ack'))} + tx.execute("UPDATE store_deliveries SET state='claimed',claimed_at=?,receipt=? WHERE order_id=?", + (wire(ctx.now),canonical(receipt).decode(),order['id']), write=True) + return HandlerOutput(data={'order_id': order['id'], 'state': order['state'], + 'delivery_id': delivery['id'], 'receipt': receipt, 'settlement_receipts': settlement}) + + +def install(app, op): + @op('delivery.get', ORDER, effect='read', version=2) + async def get(ctx, request, tx): + return await read(app, tx, ctx, request) + + @op('delivery.accept', obj({'order_id': IDENTIFIER, 'delivery_digest': HASH}, + ('order_id','delivery_digest')), signature=True, version=2) + async def claim(ctx, request, tx): + return await accept(app, tx, ctx, request) + + @op('delivery.prepare', ORDER, signature=True, version=2) + async def prepare_managed(ctx, request, tx): + buyer = _subject(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + locked = contract(tx, order['id']) + require(locked['listing']['delivery_mode'] == 'managed_instant', 'seller_delivery_required') + if order['state'] == 'funded': + await automatic(app, tx, ctx, request, order) + return HandlerOutput(data={'order_id': order['id'], 'state': order['state']}) + + @op('delivery.submit', obj({'order_id': IDENTIFIER, 'payload_ref': REF, + 'payload_digest': HASH, 'recipient_key_id': IDENTIFIER}, + ('order_id','payload_ref','payload_digest')), signature=True) + async def submit(ctx, request, tx): + from msg.plugins.communication import direct_ancestor + seller = _subject(ctx) + order = _row(tx, request.arguments['order_id'], seller) + require(order['seller'] == seller, 'order_not_found') + locked = contract(tx, order['id']) + mode = locked['listing']['delivery_mode'] + require(mode in {'sealed_manual','service'}, 'order_delivery_mode_unsupported') + require(order['state'] == 'funded', 'order_not_deliverable') + from datetime import timedelta + + from msg.core.codec import parse_time + require(ctx.now < parse_time(order['funded_at']) + timedelta( + seconds=locked['policy']['policy']['delivery_timeout_seconds']), 'delivery_deadline_passed') + ref = decode(ResourceRef, request.arguments['payload_ref']) + require(ref.revision is not None, 'payload_revision_required') + resource = await tx.resource(ref.id) + require(resource.owner == seller and resource.type in {'file','attachment'} and + resource.state == 'active', 'payload_not_owned') + require(await direct_ancestor(tx, ref.id) is None, 'payload_private_conversation') + await check_access(app, ctx, request, tx, ref.id, 'read') + revision = await tx.revision(ref) + blob = revision.content + require(blob.digest == request.arguments['payload_digest'], 'delivery_digest_mismatch') + await verify_blob(app, blob) + payload = {'id': ref.id, 'revision': ref.revision, 'blob': wire(blob)} + envelope = None + if mode == 'sealed_manual': + validate_target(tx, order, encryption=True) + require(request.arguments.get('recipient_key_id') == locked['recipient_key']['key_id'], + 'delivery_recipient_mismatch') + prefix = b''.join([piece async for piece in app.contents.read(blob, (0,min(64,blob.size)))]) + require(blob.media_type in {'application/age','application/octet-stream'} and + prefix.startswith(b'age-encryption.org/v1\n'), 'encrypted_delivery_required') + envelope = {'recipient_subject': order['buyer'], 'recipient_key': locked['recipient_key'], + 'ciphertext_ref': payload, 'seller_request_digest': request.payload_digest} + manifest = {'encryption': 'age-x25519', 'recipient_key_id': locked['recipient_key']['key_id']} + else: + require('recipient_key_id' not in request.arguments, 'unexpected_recipient_key') + manifest = {'type': 'service-evidence', 'acceptance': 'explicit-buyer-signature'} + delivery = await prepare(app, tx, ctx, request, order, locked['listing']['item_kind'], + manifest, [payload], envelope=envelope) + return HandlerOutput(data={'order_id': order['id'], 'delivery_id': delivery['id'], + 'delivery_digest': delivery['delivery_digest'], 'state': 'delivered'}) + + @op('delivery.part_get', obj({'order_id': IDENTIFIER, 'delivery_digest': HASH, + 'payload_index': {'type': 'integer','minimum': 0,'maximum': 31}, + 'offset': {'type': 'integer','minimum': 0,'maximum': 1024**4}, + 'length': {'type': 'integer','minimum': 1,'maximum': PART}}, + ('order_id','delivery_digest','payload_index','offset','length')), effect='read') + async def part_get(ctx, request, tx): + a = request.arguments + order = _buyer_order(tx, a['order_id'], _viewer(ctx)) + delivery = await verified(app, tx, order, verify_bytes=False) + require(a['delivery_digest'] == delivery['delivery_digest'], 'delivery_digest_mismatch') + require(a['payload_index'] < len(delivery['payload_refs']), 'payload_not_found') + blob = decode(BlobRef, delivery['payload_refs'][a['payload_index']]['blob']) + start, end = a['offset'], a['offset']+a['length'] + require(start < end <= blob.size, 'invalid_byte_range') + data = b''.join([piece async for piece in app.contents.read(blob, (start,end))]) + require(len(data) == a['length'], 'delivery_digest_mismatch') + following = ({**a, 'offset': end, 'length': min(PART,blob.size-end)} if endapp.clock() and + row[1] == job.principal.subject == job.principal.actor and + digest(job.arguments['token']) == row[3], 'email_challenge_expired') + order = _row(tx,row[0],row[1]) + chosen = order['delivery_target'].get('email',{}) + require(order['buyer']==row[1] and chosen.get('challenge_id')==job.arguments['challenge_id'] and + chosen.get('address_snapshot')==row[2] and chosen.get('endpoint_id') is None and + order['state'] in {'created','funded','delivered','settled'}, 'email_challenge_expired') + # No order ID, buyer handle, goods, pickup link or authority in this text. + return replace(job,kind='mail',arguments={'recipient':row[2], + 'recipient_subject':row[1], 'subject':'msg address verification', + 'text':canonical({'challenge_id':job.arguments['challenge_id'], + 'token':job.arguments['token'], 'purpose':'verify this address only'}).decode()}) + + +def install(app, op): + from msg.market.orders import ORDER, view + + @op('orders.email_verify',obj({'order_id':IDENTIFIER,'challenge_id':IDENTIFIER, + 'token':{'type':'string','minLength':43,'maxLength':43}}, + ('order_id','challenge_id','token')),signature=True) + async def verify(ctx, request, tx): + a, buyer = request.arguments, _subject(ctx) + order = _row(tx,a['order_id'],buyer) + require(order['buyer']==buyer,'order_not_found') + row = tx.one('''SELECT order_id,subject,address,verifier,expires_at,consumed + FROM delivery_email_challenges WHERE id=?''',(a['challenge_id'],)) + chosen = order['delivery_target'].get('email',{}) + require(row is not None and row[0]==order['id'] and row[1]==buyer and not row[5] and + parse_time(row[4])>ctx.now and secrets.compare_digest(row[3],digest(a['token'])) and + chosen.get('challenge_id')==a['challenge_id'] and + chosen.get('address_snapshot')==row[2] and chosen.get('endpoint_id') is None and + order['state'] in {'created','funded','delivered','settled'},'email_challenge_invalid') + endpoint = new_id('dep') + tx.execute('INSERT INTO delivery_email_endpoints(id,subject,address,verified_at) VALUES (?,?,?,?)', + (endpoint,buyer,row[2],wire(ctx.now)),write=True) + tx.execute('UPDATE delivery_email_challenges SET consumed=TRUE WHERE id=?', + (a['challenge_id'],),write=True) + chosen = {k:v for k,v in chosen.items() if k!='challenge_id'} + save_target(tx,order,{**order['delivery_target'],'email':{ + **chosen,'endpoint_id':endpoint,'state':'pending'}}) + validate_target(tx,order,email=True) + if order['delivered_at']: + await enqueue_notification(app,tx,ctx,request,order) + return HandlerOutput(data={'order':view(tx,order,buyer)}) + + @op('orders.email_disable',ORDER,signature=True) + async def disable(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx,request.arguments['order_id'],buyer) + require(order['buyer']==buyer,'order_not_found') + chosen = order['delivery_target'].get('email',{}) + if (chosen.get('endpoint_id') or '').startswith('dep_'): + tx.execute('UPDATE delivery_email_endpoints SET active=FALSE WHERE id=? AND subject=?', + (chosen['endpoint_id'],buyer),write=True) + save_target(tx,order,{**order['delivery_target'],'email':{ + **chosen,'state':'disabled','notification_disabled':True}}) + return HandlerOutput(data={'order':view(tx,order,buyer)}) diff --git a/src/msg/market/escrow.py b/src/msg/market/escrow.py new file mode 100644 index 00000000..85efe4a3 --- /dev/null +++ b/src/msg/market/escrow.py @@ -0,0 +1,135 @@ +"""One escrow release entry point, guarded transitions and append-only facts.""" +from __future__ import annotations + +from datetime import timedelta + +from msg.core.codec import canonical, loads, parse_time, wire +from msg.core.errors import require +from msg.core.models import Event +from msg.market.policy import contract +from msg.plugins.common import new_id +from msg.plugins.money import _balance, _post_transfer +from msg.plugins.orders import _row + +# Not a credential, account or public capability. Only trusted in-process market +# code can reach this token; ordinary signed money.transfer never receives it. +_ESCROW_WRITE = object() +TRANSITIONS = { + 'created': {'funded', 'cancelled'}, + 'funded': {'delivered', 'refunded', 'disputed'}, + 'delivered': {'accepted', 'refunded', 'disputed'}, + 'accepted': {'settled'}, + 'disputed': {'refunded', 'settled'}, + 'cancelled': set(), 'refunded': set(), 'settled': set(), +} +OBJECTIVE_REFUNDS = frozenset({'package_missing', 'package_digest_mismatch', + 'delivery_missing', 'delivery_digest_mismatch', 'delivery_timeout', 'buyer_cancelled'}) + + +async def transition(tx, order, state, *, now, actor, request_id, reason): + require(state in TRANSITIONS.get(order['state'], ()), 'order_transition_invalid') + changed = tx.execute('UPDATE store_orders SET state=? WHERE id=? AND state=?', + (state, order['id'], order['state']), write=True) + require(changed.rowcount == 1, 'order_transition_conflict') + fact = {'order_id': order['id'], 'from': order['state'], 'to': state, + 'at': wire(now), 'reason': reason, 'policy': order['dispute_policy'], + 'actor': actor, 'request_id': request_id} + tx.execute('INSERT INTO order_transitions(id,order_id,body) VALUES (?,?,?)', + (new_id('ot'), order['id'], canonical(fact).decode()), write=True) + # Events deliberately contain no buyer handle, evidence, goods or address. + await tx.append_event(Event(id=new_id('ev'), type='orders.' + state, time=now, + actor=actor, subject=order['buyer'], request_id=request_id, + resources=(), data={'order_id': order['id'], 'state': state})) + order['state'] = state + + +async def settle(app, tx, order, *, now, actor, request_id, reason, + refund_minor=0, decision=None): + """Validate authority *before* appending either leg of a balanced settlement. + + Caller owns the transaction. Any failure rolls back both legs, the decision + consumption, transitions, receipts, events and request result together. + """ + total = order['total_price_minor'] + require(type(refund_minor) is int and 0 <= refund_minor <= total, + 'settlement_amount_invalid') + existing = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (order['id'],)) + if existing: + body = loads(existing[0]) + require(body['refund_minor'] == refund_minor and body['reason'] == reason and + body['decision_id'] == (decision['id'] if decision else None), + 'order_already_settled') + return body['receipts'] + if decision is not None: + from msg.market.arbitration import validate_decision + await validate_decision(app, tx, order, decision, now) + require(refund_minor == decision['refund_minor'], 'decision_amount_mismatch') + elif refund_minor == total: + require(reason in OBJECTIVE_REFUNDS and order['state'] in {'funded', 'delivered'}, + 'escrow_release_forbidden') + else: + require(refund_minor == 0 and order['state'] == 'accepted' and + reason in {'buyer_acceptance', 'managed_instant_verified'}, + 'escrow_release_forbidden') + require(_balance(tx, order['escrow_subject']) == total, 'escrow_balance_mismatch') + receipts = [] + for suffix, recipient, amount, kind in ( + ('refund', order['buyer'], refund_minor, 'refund'), + ('release', order['seller'], total-refund_minor, 'transfer'), + ): + if amount: + receipts.append(_post_transfer(tx, sender=order['escrow_subject'], + recipient=recipient, amount=amount, actor=actor, + request_id=f'{order["id"]}:{suffix}', now=now, + receipt_signer=app.receipt_signer, reference=f'order_{suffix}:{order["id"]}', + kind=kind, escrow_authority=_ESCROW_WRITE)) + final = 'refunded' if refund_minor == total else 'settled' + await transition(tx, order, final, now=now, actor=actor, request_id=request_id, reason=reason) + refs = order['receipt_refs'] + [r['body']['transaction_id'] for r in receipts] + tx.execute('UPDATE store_orders SET settled_at=?,receipt_refs=? WHERE id=?', + (wire(now), canonical(refs).decode(), order['id']), write=True) + statement = {'order_id': order['id'], 'refund_minor': refund_minor, + 'release_minor': total-refund_minor, 'reason': reason, + 'decision_id': decision['id'] if decision else None, + 'policy': order['dispute_policy'], 'receipts': receipts, 'at': wire(now)} + tx.execute('INSERT INTO order_settlements(order_id,decision_id,body) VALUES (?,?,?)', + (order['id'], statement['decision_id'], canonical(statement).decode()), write=True) + if decision: + tx.execute("UPDATE arbitration_cases SET state='executed' WHERE id=? AND state='decided'", + (decision['case_id'],), write=True) + order.update(receipt_refs=refs, settled_at=wire(now)) + return receipts + + +async def resolve_due(app, *, limit=100): + """Bounded worker tick, never called by a GET or by the ClearingEngine to buy. + + Time starts at persisted creation/funding, not worker retries. Undelivered + services/manual ciphertext refund; subjective complaints and expired panels + remain held. This function has no external side effect. + """ + now = app.clock() + changed = [] + async with app.metadata.transaction(write=True) as tx: + rows = tx.rows('''SELECT o.id,o.buyer FROM store_orders o + JOIN order_deadlines d ON d.order_id=o.id + WHERE o.state IN ('created','funded') AND d.expires_at<=? + ORDER BY d.expires_at,o.id LIMIT ?''', (wire(now),limit)) + for order_id, buyer in rows: + order = _row(tx, order_id, buyer) + terms = contract(tx, order_id) + policy = terms['policy']['policy'] + origin, seconds = ((order['created_at'], policy['funding_timeout_seconds']) + if order['state'] == 'created' else + (order['funded_at'], policy['delivery_timeout_seconds'])) + if now < parse_time(origin) + timedelta(seconds=seconds): + continue + if order['state'] == 'created': + await transition(tx, order, 'cancelled', now=now, actor=buyer, + request_id='timeout:'+order_id, reason='funding_timeout') + else: + await settle(app, tx, order, now=now, actor=buyer, + request_id='timeout:'+order_id, reason='delivery_timeout', + refund_minor=order['total_price_minor']) + changed.append(order_id) + return changed diff --git a/src/msg/market/orders.py b/src/msg/market/orders.py new file mode 100644 index 00000000..272609a5 --- /dev/null +++ b/src/msg/market/orders.py @@ -0,0 +1,213 @@ +"""New checkout contracts without changing legacy signed orders.buy@1 semantics.""" +from __future__ import annotations + +from datetime import timedelta + +from msg.core.codec import canonical, digest, parse_time, wire +from msg.core.errors import require +from msg.core.models import HandlerOutput +from msg.market.escrow import settle, transition +from msg.market.policy import contract, snapshot +from msg.market.targets import enqueue_notification, save_target, target_for, validate_target +from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _post_transfer +from msg.plugins.orders import _order_id, _row, _subject, _view, _viewer +from msg.plugins.schemas import IDENTIFIER, obj +from msg.plugins.store import _body, _listing, _package_row + +HASH = {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'} +ORDER = obj({'order_id': IDENTIFIER}, ('order_id',)) +INTENT = { + 'listing_id': IDENTIFIER, 'listing_revision': IDENTIFIER, + 'quantity': {'type': 'integer', 'minimum': 1, 'maximum': 10**9}, + 'currency_id': {'const': CURRENCY_ID}, + 'total_price_minor': {'type': 'integer', 'minimum': 1, 'maximum': MAX_MINOR}, + 'email': {'type': 'string', 'minLength': 3, 'maxLength': 254}, +} +INTENT_REQUIRED = ('listing_id','listing_revision','quantity','currency_id','total_price_minor') + + +def view(tx, order, viewer): + result = _view(order, viewer) + row = tx.one('SELECT digest FROM order_contracts WHERE order_id=?', (order['id'],)) + if row: + result['order_digest'] = row[0] + result['contract_version'] = 2 + locked = contract(tx, order['id']) + result['delivery_mode'] = locked['listing']['delivery_mode'] + result['policy_digest'] = locked['policy']['policy_digest'] + # Seller needs the *public encryption subkey*, not buyer's email. + if locked['recipient_key']: + result['recipient_key'] = locked['recipient_key'] + result['email_status'] = order['delivery_target'].get('email', {}).get('state', 'disabled') + return result + + +async def create(app, tx, ctx, request): + buyer, args = _subject(ctx), request.arguments + listing = await _listing(app, ctx, request, tx, args['listing_id']) + body, revision = await _body(app, tx, listing) + require(body['mode'] == 'sale' and body['state'] == 'active' and + (body['expires_at'] is None or parse_time(body['expires_at']) > ctx.now), + 'listing_not_available') + require(args['listing_revision'] == listing.revision, 'listing_revision_conflict') + require(body['escrow_policy'] == 'escrow-v1', 'escrow_policy_unknown') + policy = snapshot(tx, body['dispute_policy']) + require(body['currency_id'] == args['currency_id'] == CURRENCY_ID, 'unsupported_currency') + total = body['price_minor'] * args['quantity'] + require(0 < total <= MAX_MINOR, 'money_overflow') + require(total == args['total_price_minor'], 'price_changed') + sold = tx.one('''SELECT COALESCE(SUM(quantity),0) FROM store_orders + WHERE listing_id=? AND state NOT IN ('cancelled','refunded')''', (listing.id,))[0] + require(sold + args['quantity'] <= body['quantity'], 'quantity_unavailable') + require(listing.owner != buyer, 'self_purchase_forbidden') + require(body['item_kind'] != 'secret' or body['delivery_mode'] == 'sealed_manual', + 'secret_requires_buyer_encryption') + package_id = package_revision = package_digest = None + if body['delivery_mode'] == 'managed_instant': + require(body['item_kind'] in {'file','bundle','text'}, 'order_delivery_mode_unsupported') + package = await _package_row(tx, body['package_ref']) + require(package is not None and package[1] == listing.id and package[3] == listing.owner and + package[5] == body['item_kind'] and package[10] == 'managed_instant', + 'package_not_available') + package_id, package_revision, package_digest = package[0], package[4], package[8] + recipient_key = None + if body['delivery_mode'] == 'sealed_manual': + row = tx.one('''SELECT key_id,recipient,public_key FROM encryption_subkeys + WHERE subject=? AND is_primary=1 AND retired_at IS NULL''', (buyer,)) + require(row is not None, 'delivery_recipient_key_required') + recipient_key = {'key_id': row[0], 'recipient': row[1], 'fingerprint': digest(row[2])} + target = await target_for(app, tx, buyer, args.get('email')) + order_id, now = _order_id(), wire(ctx.now) + escrow = 'esc_' + order_id[4:] + tx.execute("INSERT INTO ledger_accounts(id,kind,subject_id,source_id) VALUES (?,'order_escrow',NULL,?)", + (escrow, order_id), write=True) + tx.execute('''INSERT INTO store_orders + (id,buyer,seller,listing_id,listing_revision,package_id,package_revision,package_digest, + quantity,unit_price_minor,total_price_minor,currency_id,escrow_subject,escrow_policy, + dispute_policy,terms_digest,delivery_target,payment_intent_digest,payment_transaction_id, + state,created_at,funded_at,receipt_refs) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL,'created',?,NULL,'[]')''', + (order_id,buyer,listing.owner,listing.id,listing.revision,package_id,package_revision, + package_digest,args['quantity'],body['price_minor'],total,CURRENCY_ID,escrow, + body['escrow_policy'],body['dispute_policy'],digest(body['terms']),canonical(target).decode(), + request.payload_digest,now), write=True) + locked = {'order_id': order_id, 'version': 2, 'buyer': buyer, 'seller': listing.owner, + 'listing': body, 'listing_revision': listing.revision, + 'listing_digest': revision.content.digest, 'package_id': package_id, + 'package_revision': package_revision, 'package_digest': package_digest, + 'quantity': args['quantity'], 'total_price_minor': total, + 'terms_digest': digest(body['terms']), 'policy': policy, + 'recipient_key': recipient_key, 'handle_snapshot': target['handle_snapshot'], + 'buyer_principal': wire(ctx.principal), 'created_at': now} + tx.execute('INSERT INTO order_contracts(order_id,body,digest) VALUES (?,?,?)', + (order_id,canonical(locked).decode(),digest(locked)), write=True) + tx.execute('INSERT INTO order_deadlines(order_id,expires_at) VALUES (?,?)', + (order_id, wire(ctx.now+timedelta(seconds=policy['policy']['funding_timeout_seconds']))), write=True) + from msg.market.email import verification_message + order = _row(tx, order_id, buyer) + await verification_message(app, tx, ctx, request, order) + return order + + +async def fund(app, tx, ctx, request, order): + require(order['state'] == 'created', 'order_not_fundable') + locked = contract(tx, order['id']) + require(ctx.now < parse_time(order['created_at']) + timedelta( + seconds=locked['policy']['policy']['funding_timeout_seconds']), 'payment_intent_expired') + validate_target(tx, order) + receipt = _post_transfer(tx, sender=order['buyer'], recipient=order['escrow_subject'], + amount=order['total_price_minor'], actor=order['buyer'], request_id=request.request_id, + now=ctx.now, receipt_signer=app.receipt_signer, reference='order_fund:'+order['id']) + tx.execute('''UPDATE store_orders SET payment_transaction_id=?,payment_intent_digest=?, + funded_at=?,receipt_refs=? WHERE id=?''', (receipt['body']['transaction_id'], + request.payload_digest,wire(ctx.now),canonical([receipt['body']['transaction_id']]).decode(), + order['id']), write=True) + order.update(payment_transaction_id=receipt['body']['transaction_id'], + payment_intent_digest=request.payload_digest, funded_at=wire(ctx.now), + receipt_refs=[receipt['body']['transaction_id']]) + await transition(tx, order, 'funded', now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='signed_payment_intent') + tx.execute('UPDATE order_deadlines SET expires_at=? WHERE order_id=?', + (wire(ctx.now+timedelta(seconds=locked['policy']['policy']['delivery_timeout_seconds'])), order['id']), write=True) + if locked['listing']['delivery_mode'] == 'managed_instant': + from msg.market.delivery import automatic + await automatic(app, tx, ctx, request, order) + return receipt + + +def install(app, op): + from msg.market.email import install as install_email + install_email(app, op) + + @op('orders.create', obj(INTENT, INTENT_REQUIRED), signature=True) + async def reserve(ctx, request, tx): + order = await create(app, tx, ctx, request) + return HandlerOutput(data={'order': view(tx, order, order['buyer'])}) + + @op('orders.buy', obj(INTENT, INTENT_REQUIRED), signature=True, version=2) + async def buy(ctx, request, tx): + order = await create(app, tx, ctx, request) + receipt = await fund(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], order['buyer']), order['buyer']), + 'payment': receipt}) + + @op('orders.fund', obj({'order_id': IDENTIFIER, 'order_digest': HASH, + 'total_price_minor': INTENT['total_price_minor'], 'currency_id': {'const': CURRENCY_ID}}, + ('order_id','order_digest','total_price_minor','currency_id')), signature=True) + async def pay(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + locked = contract(tx, order['id']) + require(digest(locked) == request.arguments['order_digest'] and + order['total_price_minor'] == request.arguments['total_price_minor'], + 'payment_intent_mismatch') + payment = await fund(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], buyer), buyer), + 'payment': payment}) + + @op('orders.cancel', ORDER, signature=True, version=2) + async def cancel(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + if order['state'] == 'created': + await transition(tx, order, 'cancelled', now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_cancelled') + elif order['state'] not in {'cancelled', 'refunded'}: + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_cancellable') + await settle(app, tx, order, now=ctx.now, actor=buyer, request_id=request.request_id, + reason='buyer_cancelled', refund_minor=order['total_price_minor']) + return HandlerOutput(data={'order': view(tx, order, buyer)}) + + @op('orders.email_bind', obj({'order_id': IDENTIFIER, 'email': INTENT['email']}, + ('order_id','email')), signature=True) + async def email_bind(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + contract(tx, order['id']) + old = order['delivery_target'].get('email', {}) + require(order['state'] in {'created','funded','delivered','settled'}, 'order_closed') + require(order['delivered_at'] is None or + (old.get('endpoint_id') is None and + old.get('address_snapshot') == request.arguments['email']), + 'delivery_target_locked') + target = await target_for(app, tx, buyer, request.arguments['email']) + target['handle_snapshot'] = order['delivery_target']['handle_snapshot'] + require(target['email'].get('endpoint_id') is not None, 'email_not_verified') + save_target(tx, order, target) + validate_target(tx, order, email=True) + if order['delivered_at']: + await enqueue_notification(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, order, buyer)}) + + @op('orders.contract', ORDER, effect='read') + async def get_contract(ctx, request, tx): + viewer = _viewer(ctx) + order = _row(tx, request.arguments['order_id'], viewer) + locked = contract(tx, order['id']) + # Neither endpoint nor credential ceilings are part of seller projections. + return HandlerOutput(data={'order': view(tx, order, viewer), 'contract': { + k:v for k,v in locked.items() if k not in {'buyer_principal','handle_snapshot'}}}) diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py new file mode 100644 index 00000000..0a51ec5b --- /dev/null +++ b/src/msg/market/policy.py @@ -0,0 +1,76 @@ +"""Small, immutable policies. No expressions, callbacks or model discretion.""" +from __future__ import annotations + +from msg.core.codec import canonical, digest, loads +from msg.core.errors import require + +DEFAULT_POLICY = { + 'id': 'dispute-v1', 'version': 1, + 'funding_timeout_seconds': 900, 'delivery_timeout_seconds': 86400, + 'case_timeout_seconds': 604800, 'decision_lifetime_seconds': 86400, + 'panel_size': 3, 'quorum': 2, 'candidates': [], + 'allow_appeal': False, 'appeal_seconds': 3600, + 'selection': 'sha256-order-policy-round-subject-v1', + 'member_invalidation': 'hold-no-replacement-v1', + 'rules': 'objective-faults-v1', +} + + +def validate(policy): + require(isinstance(policy, dict) and set(policy) == set(DEFAULT_POLICY), + 'arbitration_policy_invalid') + require(isinstance(policy['id'], str) and 1 <= len(policy['id']) <= 80 and + type(policy['version']) is int and policy['version'] == 1 and policy['rules'] == 'objective-faults-v1' and + policy['selection'] == DEFAULT_POLICY['selection'] and + policy['member_invalidation'] == DEFAULT_POLICY['member_invalidation'], + 'arbitration_policy_invalid') + for name, maximum in (('funding_timeout_seconds', 86400), + ('delivery_timeout_seconds', 2592000), + ('case_timeout_seconds', 2592000), + ('decision_lifetime_seconds', 604800), + ('appeal_seconds', 86400)): + require(type(policy[name]) is int and 60 <= policy[name] <= maximum, + 'arbitration_policy_invalid', name) + size, quorum, candidates = policy['panel_size'], policy['quorum'], policy['candidates'] + require(type(size) is int and 1 <= size <= 9 and type(quorum) is int and + size // 2 < quorum <= size and type(policy['allow_appeal']) is bool and + isinstance(candidates, list) and len(candidates) <= 100 and + all(isinstance(s, str) and s.startswith('u_') and s != 'u_root' + for s in candidates) and len(candidates) == len(set(candidates)), + 'arbitration_policy_invalid') + require(not policy['allow_appeal'] or + policy['decision_lifetime_seconds'] > policy['appeal_seconds'], + 'arbitration_policy_invalid') + return policy + + +def load_policy(tx, policy_id): + row = tx.one('SELECT body,digest FROM arbitration_policies WHERE id=?', (policy_id,)) + if row: + policy = loads(row[0]) + require(digest(policy) == row[1], 'arbitration_policy_corrupt') + else: + require(policy_id == DEFAULT_POLICY['id'], 'arbitration_policy_unknown') + policy = loads(canonical(DEFAULT_POLICY)) + return validate(policy) + + +def snapshot(tx, policy_id): + policy = load_policy(tx, policy_id) + # Freeze the grant epoch, not merely the subject: revoke+regrant cannot revive + # a vote or silently replace a member in an existing order. + epochs = {} + for candidate in policy['candidates']: + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (candidate,)) + if row and row[1]: + epochs[candidate] = row[0] + return {'policy': policy, 'policy_digest': digest(policy), 'candidate_epochs': epochs} + + +def contract(tx, order_id): + row = tx.one('SELECT body,digest FROM order_contracts WHERE order_id=?', (order_id,)) + require(row is not None, 'order_contract_required') + body = loads(row[0]) + require(digest(body) == row[1] and body['order_id'] == order_id, + 'order_contract_corrupt') + return body diff --git a/src/msg/market/references.py b/src/msg/market/references.py new file mode 100644 index 00000000..8980f222 --- /dev/null +++ b/src/msg/market/references.py @@ -0,0 +1,19 @@ +"""SQL-authoritative market content roots, shared by backup and collection. + +Order IDs, digests and pins never confer read access. Pins protect a blob before +SQL commits; this inventory protects it after the originating file is purged. +""" +from msg.core.codec import loads + + +def content_references(execute): + for table, column in (('store_packages', 'payload_refs'), + ('store_deliveries', 'payload_refs')): + for (raw,) in execute(f'SELECT {column} FROM {table} ORDER BY id'): + for ref in loads(raw): + yield ref['blob'] + # Legacy archives can be inspected before the new schema is installed. + exists = next(iter(execute("SELECT to_regclass('arbitration_evidence')")))[0] + if exists is not None: + for (raw,) in execute('SELECT body FROM arbitration_evidence ORDER BY id'): + yield loads(raw)['blob'] diff --git a/src/msg/market/targets.py b/src/msg/market/targets.py new file mode 100644 index 00000000..b5f32526 --- /dev/null +++ b/src/msg/market/targets.py @@ -0,0 +1,145 @@ +"""Buyer-owned endpoint binding shared by checkout, delivery and SMTP worker.""" +from __future__ import annotations + +import re +from dataclasses import replace + +from msg.core.codec import canonical, decode, digest, loads +from msg.core.errors import Failure, require +from msg.core.models import EffectJob, EmailSettings +from msg.market.policy import contract +from msg.plugins.common import new_id + + +def _email(tx, subject): + row = tx.one('SELECT generation,body FROM emails WHERE subject=?', (subject,)) + if not row: + return None, None + settings = decode(EmailSettings, loads(row[1])) + endpoint = 'em_' + digest((subject, settings.address, settings.verified_at, row[0]))[7:39] + return settings, endpoint + + +async def target_for(app, tx, buyer, email_address=None): + handle = (await tx.resource(buyer)).name + target = {'subject_id': buyer, 'channel': 'site', 'handle_snapshot': handle, + 'email': {'state': 'disabled'}} + if email_address is None: + return target + require(isinstance(email_address, str) and len(email_address) <= 254 and + re.fullmatch(r'[^\s@]+@[^\s@]+\.[^\s@]+', email_address) is not None and + '\r' not in email_address and '\n' not in email_address, 'invalid_email') + email, endpoint = _email(tx, buyer) + verified = (email is not None and email.subject_id == buyer and + email.address == email_address and email.verified_at is not None) + target['email'] = {'state': 'pending' if app.settings.server.mail else 'disabled', + 'subject_id': buyer, 'address_snapshot': email_address, + 'endpoint_id': endpoint if verified else None} + # Verification is a separate, order-information-free external effect. + return target + + +def save_target(tx, order, target): + tx.execute('UPDATE store_orders SET delivery_target=? WHERE id=?', + (canonical(target).decode(), order['id']), write=True) + order['delivery_target'] = target + + +def validate_target(tx, order, *, email=False, encryption=False): + target = order['delivery_target'] + require(target.get('subject_id') == order['buyer'] and target.get('channel') == 'site', + 'delivery_recipient_mismatch') + if encryption: + locked = contract(tx, order['id']).get('recipient_key') + row = tx.one('SELECT subject,recipient,retired_at FROM encryption_subkeys WHERE key_id=?', + (locked['key_id'],)) if locked else None + require(row is not None and row[0] == order['buyer'] and row[1] == locked['recipient'] + and row[2] is None, 'delivery_recipient_mismatch') + if email: + chosen = target.get('email', {}) + require(not chosen.get('notification_disabled'), 'delivery_recipient_mismatch') + endpoint_id = chosen.get('endpoint_id') or '' + if endpoint_id.startswith('dep_'): + row = tx.one('SELECT subject,address,active FROM delivery_email_endpoints WHERE id=?', + (endpoint_id,)) + require(row is not None and row[2] and + row[0] == chosen.get('subject_id') == order['buyer'] and + row[1] == chosen.get('address_snapshot'), 'delivery_recipient_mismatch') + return row[1] + current, endpoint = _email(tx, order['buyer']) + require(chosen.get('subject_id') == order['buyer'] and current is not None and + current.subject_id == order['buyer'] and current.verified_at is not None and + chosen.get('endpoint_id') == endpoint and + chosen.get('address_snapshot') == current.address, + 'delivery_recipient_mismatch') + return current.address + return None + + +async def enqueue_notification(app, tx, ctx, request, order): + chosen = order['delivery_target'].get('email', {}) + if (not app.settings.server.mail or not chosen.get('endpoint_id') or + chosen.get('notification_disabled')): + return + try: + validate_target(tx, order, email=True) + except Failure as exc: + if exc.code != 'delivery_recipient_mismatch': + raise + save_target(tx, order, {**order['delivery_target'], 'email': { + **chosen, 'state': 'pending', 'error': exc.code}}) + return + # Only the buyer can bind/queue the auxiliary channel, even when the seller + # uploads ciphertext later. Checkout's signed principal is persisted privately. + locked = contract(tx, order['id']) + from msg.core.models import Principal + principal = decode(Principal, locked['buyer_principal']) + dedupe = 'market-mail:'+order['id']+':'+chosen['endpoint_id'] + if tx.one('SELECT 1 FROM jobs WHERE dedupe=?', (dedupe,)): + return + job = EffectJob(id=new_id('job'), event_id='delivery:'+order['id'], kind='market_mail', + dedupe_key=dedupe, principal=principal, + operation='orders.buy', arguments={'order_id': order['id'], 'endpoint_id': chosen['endpoint_id']}, + state='pending', attempts=0, next_attempt_at=ctx.now, lease_until=None) + await tx.enqueue(job) + target = {**order['delivery_target'], 'email': {**chosen, 'state': 'queued'}} + save_target(tx, order, target) + + +async def render_notification(app, tx, job): + from msg.plugins.delivery import _delivery + from msg.plugins.orders import _row + order = _row(tx, job.arguments['order_id'], job.principal.subject) + require(order['buyer'] == job.principal.subject and job.principal.actor == order['buyer'], + 'delivery_recipient_mismatch') + delivery = _delivery(tx, order['id']) + require(order['state'] in {'delivered','accepted','settled'} and delivery is not None and + delivery['recipient_subject'] == order['buyer'], 'delivery_not_available') + require(order['delivery_target'].get('email', {}).get('endpoint_id') == job.arguments['endpoint_id'], + 'delivery_recipient_mismatch') + locked = contract(tx, order['id']) + address = validate_target(tx, order, email=True, + encryption=locked['listing']['delivery_mode'] == 'sealed_manual') + # A stable site URL is not a capability. GET always requires fresh authentication. + link = app.settings.service_url.rstrip('/') + '/_orders/' + order['id'] + '/_delivery' + body = {'order_id': order['id'], 'handle': locked['handle_snapshot'], 'pickup': link} + return replace(job, kind='mail', arguments={'recipient': address, + 'recipient_subject': order['buyer'], 'subject': 'msg order notification', + 'text': canonical(body).decode()}) + + +def notification_status(tx, job, state, code): + """Update only the endpoint this attempt was authorised to notify.""" + if job.kind != 'market_mail': + return + from msg.plugins.orders import _row + order = _row(tx, job.arguments['order_id'], job.principal.subject) + chosen = order['delivery_target'].get('email', {}) + if (chosen.get('endpoint_id') != job.arguments['endpoint_id'] or + chosen.get('notification_disabled')): + return + status = ('smtp_accepted' if state == 'done' and code == 'sent' else + 'delivered_unknown' if state == 'uncertain' else + 'disabled' if code == 'mail_disabled' else 'pending') + save_target(tx, order, {**order['delivery_target'], 'email': { + **chosen, 'state': status, 'last_result': code}}) diff --git a/src/msg/plugins/delivery.py b/src/msg/plugins/delivery.py index dde3dfcd..beaca793 100644 --- a/src/msg/plugins/delivery.py +++ b/src/msg/plugins/delivery.py @@ -142,6 +142,9 @@ async def prepare(ctx, request, tx): @op('delivery.get', obj({'order_id': IDENTIFIER}, ('order_id',)), effect='read') async def get(ctx, request, tx): + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (request.arguments['order_id'],)): + from msg.market.delivery import read + return await read(app, tx, ctx, request) buyer = ctx.principal.subject require(buyer is not None and ctx.principal.actor == buyer, 'order_not_found') @@ -162,6 +165,9 @@ async def get(ctx, request, tx): 'delivery_digest': {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'}}, ('order_id', 'delivery_digest')), signature=True) async def accept(ctx, request, tx): + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (request.arguments['order_id'],)): + from msg.market.delivery import accept as market_accept + return await market_accept(app, tx, ctx, request) buyer = _subject(ctx) order = _buyer_order(tx, request.arguments['order_id'], buyer) delivery = _delivery(tx, order['id']) @@ -174,24 +180,16 @@ async def accept(ctx, request, tx): 'delivery_recipient_mismatch') require(_balance(tx, order['escrow_subject']) == order['total_price_minor'], 'escrow_balance_mismatch') - receipt = _post_transfer(tx, sender=order['escrow_subject'], - recipient=order['seller'], amount=order['total_price_minor'], - actor=buyer, request_id=request.request_id, now=ctx.now, - receipt_signer=app.receipt_signer, - reference='order_release:' + order['id']) - receipt_id = receipt['body']['transaction_id'] - changed = tx.execute('''UPDATE store_deliveries - SET state='claimed',claimed_at=?,receipt=? - WHERE order_id=? AND recipient_subject=? AND state='prepared' ''', - (wire(ctx.now), canonical(receipt).decode(), order['id'], buyer), write=True) - require(changed.rowcount == 1, 'delivery_not_acceptable') - changed = tx.execute('''UPDATE store_orders - SET state='settled',settled_at=?,receipt_refs=? - WHERE id=? AND buyer=? AND state='delivered' AND delivered_at IS NOT NULL''', - (wire(ctx.now), canonical([*order['receipt_refs'], receipt_id]).decode(), - order['id'], buyer), write=True) - require(changed.rowcount == 1, 'delivery_not_acceptable') + from msg.market.escrow import settle, transition + await transition(tx, order, 'accepted', now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance') + receipt = (await settle(app, tx, order, now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance'))[0] + tx.execute("UPDATE store_deliveries SET state='claimed',claimed_at=?,receipt=? WHERE order_id=?", + (wire(ctx.now), canonical(receipt).decode(), order['id']), write=True) return HandlerOutput(data={'order_id': order['id'], 'state': 'settled', 'delivery_id': delivery['id'], 'receipt': receipt}) + from msg.market.delivery import install as install_market + install_market(app, op) finish() diff --git a/src/msg/plugins/money.py b/src/msg/plugins/money.py index 059d16df..b589e881 100644 --- a/src/msg/plugins/money.py +++ b/src/msg/plugins/money.py @@ -51,10 +51,14 @@ def _supply(tx): def _post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer'): + receipt_signer, reference=None, kind='transfer', escrow_authority=None): """Called only within the executor's serialized write transaction.""" require(kind in {'transfer', 'refund'}, 'invalid_money_kind') amount = _amount(amount) + account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) + if account and account[0] == 'order_escrow': + from msg.market.escrow import _ESCROW_WRITE + require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') require(sender != recipient, 'self_transfer_forbidden') require(sender not in {ROOT_SUBJECT,'@root'}, 'root_local_only') require(_balance(tx, sender) >= amount, 'insufficient_funds') diff --git a/src/msg/plugins/orders.py b/src/msg/plugins/orders.py index c85a0b8a..f4a8d08c 100644 --- a/src/msg/plugins/orders.py +++ b/src/msg/plugins/orders.py @@ -173,27 +173,19 @@ async def cancel(ctx, request, tx): 'order_not_cancellable') require(tx.one('SELECT 1 FROM store_deliveries WHERE order_id=?', (row['id'],)) is None, 'order_not_cancellable') - require(_balance(tx, row['escrow_subject']) == row['total_price_minor'], - 'escrow_balance_mismatch') - refund = _post_transfer(tx, sender=row['escrow_subject'], - recipient=buyer, amount=row['total_price_minor'], actor=buyer, - request_id=request.request_id, now=ctx.now, - receipt_signer=app.receipt_signer, - reference='order_refund:' + row['id'], kind='refund') - refs = [*row['receipt_refs'], refund['body']['transaction_id']] - changed = tx.execute('''UPDATE store_orders SET state='refunded',receipt_refs=? - WHERE id=? AND buyer=? AND state='funded' AND delivered_at IS NULL - AND NOT EXISTS (SELECT 1 FROM store_deliveries WHERE order_id=?)''', - (canonical(refs).decode(), row['id'], buyer, row['id']), write=True) - require(changed.rowcount == 1, 'order_not_cancellable') + from msg.market.escrow import settle + refunds = await settle(app, tx, row, now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_cancelled', + refund_minor=row['total_price_minor']) + refund = refunds[0] return HandlerOutput(data={'order': _view(_row(tx, row['id'], buyer), buyer), 'refund': refund}) @op('orders.get', obj({'order_id': IDENTIFIER}, ('order_id',)), effect='read') async def get(ctx, request, tx): viewer = _viewer(ctx) - return HandlerOutput(data={'order': _view(_row(tx, request.arguments['order_id'], - viewer), viewer)}) + from msg.market.orders import view + return HandlerOutput(data={'order': view(tx, _row(tx, request.arguments['order_id'], viewer), viewer)}) @op('orders.list', obj({'role': {'enum': ['buy', 'sell']}, 'status': {'enum': ['open', 'completed', 'disputed']}, @@ -210,7 +202,7 @@ async def list_orders(ctx, request, tx): where = ('buyer=?' if role == 'buy' else 'seller=?') values = [viewer] if status: - where += (' AND state IN (\'funded\',\'delivered\',\'accepted\')' + where += (' AND state IN (\'created\',\'funded\',\'delivered\',\'accepted\')' if status == 'open' else ' AND state IN (\'settled\',\'cancelled\',\'refunded\')' if status == 'completed' else @@ -242,4 +234,8 @@ async def payment(ctx, request, tx): result['escrow_balance_minor'] = _balance(tx, row['escrow_subject']) return HandlerOutput(data={'payment': result}) + from msg.market.orders import install as install_market + install_market(app, op) + from msg.market.arbitration import install as install_arbitration + install_arbitration(app, op) finish() diff --git a/src/msg/plugins/store.py b/src/msg/plugins/store.py index 9113ae9c..2aaacbbb 100644 --- a/src/msg/plugins/store.py +++ b/src/msg/plugins/store.py @@ -190,7 +190,7 @@ async def package_deposit(ctx, request, tx): require(resource.revision == request.arguments['listing_revision'], 'listing_revision_conflict') require(body['delivery_mode'] == 'managed_instant' and - body['item_kind'] in {'file','bundle'} and body['state'] != 'closed', + body['item_kind'] in {'file','bundle','text'} and body['state'] != 'closed', 'package_mode_unsupported') manifest = request.arguments['manifest'] require(len(canonical(manifest)) <= MAX_MANIFEST_BYTES, 'manifest_too_large') @@ -206,6 +206,8 @@ async def package_deposit(ctx, request, tx): require(await direct_ancestor(tx, rid) is None, 'payload_private_conversation') await check_access(app, ctx, request, tx, rid, 'read') revision=await tx.revision(ResourceRef(id=rid,revision=ref['revision'])) + if body['item_kind'] == 'text': + require(revision.content.media_type.startswith('text/'), 'text_payload_required') blobs.append(revision.content) total+=revision.content.size require(total <= 1024**4, 'package_too_large') diff --git a/src/msg/storage/ledger_migration.py b/src/msg/storage/ledger_migration.py index ed8caacd..d16865c6 100644 --- a/src/msg/storage/ledger_migration.py +++ b/src/msg/storage/ledger_migration.py @@ -111,7 +111,7 @@ def _validate_identity_references(conn, escrow): 'from_subject', 'to_subject', 'publisher', 'buyer', 'seller', 'parent', 'source_id', 'target_id', 'participant_a', 'participant_b', 'initiator', 'blocker', 'blocked', - 'invited_by', 'granted_by'} + 'invited_by', 'granted_by', 'arbitrator', 'party', 'author'} columns = [(table, column) for table, column in conn.execute(''' SELECT table_name,column_name FROM information_schema.columns WHERE table_schema=current_schema()''') @@ -224,6 +224,22 @@ def _install_subject_account_trigger(conn): 'custodial_upgrades': ('subject', 'subject_id'), 'legacy_directive_versions': ('subject',), 'sync_checkpoints': ('subject',), + # Active market owners/panel authority are not financial account references. + # Settlement receipts retain historical debit/credit IDs intentionally. + 'order_contracts': ('buyer', 'seller', 'buyer_principal.actor', + 'buyer_principal.subject', 'buyer_principal.certificates.*', + 'buyer_principal.ceiling.*.scope.resource_id', + 'policy.policy.candidates.*'), + 'order_settlements': (), + 'order_transitions': ('actor',), + 'delivery_envelopes': ('recipient_subject',), + 'arbitration_policies': ('candidates.*',), + 'arbitration_cases': ('buyer', 'seller', 'opened_by', 'policy.policy.candidates.*', + 'panels.0.*', 'panels.1.*'), + 'arbitration_votes': ('arbitrator', 'principal.actor', 'principal.subject', + 'principal.certificates.*', 'principal.ceiling.*.scope.resource_id'), + 'arbitration_decisions': ('voters.*',), + 'arbitration_evidence': ('source.id',), } diff --git a/src/msg/storage/postgres.py b/src/msg/storage/postgres.py index d4dd80a7..a1425fc1 100644 --- a/src/msg/storage/postgres.py +++ b/src/msg/storage/postgres.py @@ -199,6 +199,61 @@ channel TEXT NOT NULL CHECK(channel='site'), state TEXT NOT NULL CHECK(state IN ('prepared','claimed')), prepared_at TEXT NOT NULL, claimed_at TEXT, receipt TEXT); +CREATE TABLE IF NOT EXISTS order_deadlines ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), expires_at TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS order_deadlines_due ON order_deadlines(expires_at,order_id); +-- Market v2 keeps v1 signed contracts intact. Creation reserves stock without money. +ALTER TABLE store_orders ALTER COLUMN payment_transaction_id DROP NOT NULL; +ALTER TABLE store_orders ALTER COLUMN funded_at DROP NOT NULL; +ALTER TABLE store_orders DROP CONSTRAINT IF EXISTS store_orders_state_check; +ALTER TABLE store_orders ADD CONSTRAINT store_orders_state_check CHECK(state IN + ('created','funded','delivered','accepted','settled','cancelled','refunded','disputed')); +CREATE TABLE IF NOT EXISTS delivery_email_endpoints ( + id TEXT PRIMARY KEY, subject TEXT NOT NULL REFERENCES identities(id), + address TEXT NOT NULL, verified_at TEXT NOT NULL, active BOOLEAN NOT NULL DEFAULT TRUE); +CREATE TABLE IF NOT EXISTS delivery_email_challenges ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL REFERENCES store_orders(id), + subject TEXT NOT NULL REFERENCES identities(id), address TEXT NOT NULL, + verifier TEXT NOT NULL, expires_at TEXT NOT NULL, consumed BOOLEAN NOT NULL DEFAULT FALSE); +CREATE TABLE IF NOT EXISTS order_contracts ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), body TEXT NOT NULL, digest TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS order_settlements ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), decision_id TEXT UNIQUE, body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS order_transitions ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL REFERENCES store_orders(id), body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS delivery_envelopes ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitration_policies ( + id TEXT PRIMARY KEY, body TEXT NOT NULL, digest TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitrator_roles ( + subject TEXT PRIMARY KEY REFERENCES identities(id), epoch TEXT NOT NULL, active BOOLEAN NOT NULL); +CREATE TABLE IF NOT EXISTS arbitrator_conflicts ( + arbitrator TEXT NOT NULL REFERENCES identities(id), party TEXT NOT NULL REFERENCES identities(id), + PRIMARY KEY(arbitrator,party)); +CREATE TABLE IF NOT EXISTS arbitration_cases ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL UNIQUE REFERENCES store_orders(id), + round INTEGER NOT NULL DEFAULT 0, state TEXT NOT NULL, deadline TEXT NOT NULL, body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitration_votes ( + case_id TEXT NOT NULL REFERENCES arbitration_cases(id), round INTEGER NOT NULL, + arbitrator TEXT NOT NULL REFERENCES identities(id), body TEXT NOT NULL, + PRIMARY KEY(case_id,round,arbitrator)); +CREATE TABLE IF NOT EXISTS arbitration_decisions ( + id TEXT PRIMARY KEY, case_id TEXT NOT NULL REFERENCES arbitration_cases(id), + round INTEGER NOT NULL, body TEXT NOT NULL, revoked BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE(case_id,round)); +CREATE TABLE IF NOT EXISTS arbitration_evidence ( + id TEXT PRIMARY KEY, case_id TEXT NOT NULL REFERENCES arbitration_cases(id), + author TEXT NOT NULL REFERENCES identities(id), visibility TEXT NOT NULL, body TEXT NOT NULL); +DO $$ DECLARE tbl TEXT; BEGIN + FOREACH tbl IN ARRAY ARRAY['order_contracts','order_settlements','order_transitions', + 'arbitration_policies','arbitration_votes'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname=tbl||'_append_only' + AND tgrelid=tbl::regclass) THEN + EXECUTE format('CREATE TRIGGER %I BEFORE UPDATE OR DELETE ON %I FOR EACH ROW + EXECUTE FUNCTION msg_money_ledger_append_only()', tbl||'_append_only',tbl); + END IF; + END LOOP; +END $$; CREATE TABLE IF NOT EXISTS server_offers ( offer_id TEXT PRIMARY KEY, resource_kind TEXT NOT NULL, unit TEXT NOT NULL, diff --git a/src/msg/workers/effects.py b/src/msg/workers/effects.py index 92af7cb3..88dfbebe 100644 --- a/src/msg/workers/effects.py +++ b/src/msg/workers/effects.py @@ -120,6 +120,8 @@ async def _claim(self): if job.lease_until is None or job.lease_until <= self.app.clock(): await tx.save_job(replace(job, state='uncertain', lease_until=None)) tx.set_setting('job_status:' + job.id, {'code': 'expired_execution_lease'}) + from msg.market.targets import notification_status + notification_status(tx, job, 'uncertain', 'expired_execution_lease') return job, False row = tx.one("SELECT id FROM jobs WHERE state='pending' AND next_at<=? ORDER BY next_at,id LIMIT 1", (wire(self.app.clock()),)) @@ -140,6 +142,8 @@ async def _finish(self, job, state, code): return await tx.save_job(replace(current, state=state, lease_until=None)) tx.set_setting('job_status:' + job.id, {'code': code}) + from msg.market.targets import notification_status + notification_status(tx, job, state, code) async def _retry_mail(self,job,code): async with self.app.metadata.transaction(write=True) as tx: @@ -153,6 +157,8 @@ async def _retry_mail(self,job,code): await tx.save_job(replace(current,state='pending',lease_until=None, next_attempt_at=self.app.clock()+timedelta(seconds=delay))) tx.set_setting('job_status:'+job.id,{'code':code}) + from msg.market.targets import notification_status + notification_status(tx, job, 'pending', code) async def _retry_webhook(self,job): async with self.app.metadata.transaction(write=True) as tx: @@ -263,6 +269,18 @@ async def _mail(self, job): require(state in {'sent', 'uncertain'}, 'invalid_delivery_result') await self._finish(job, 'done' if state == 'sent' else 'uncertain', state) + async def _market_mail(self, job): + from msg.market.targets import render_notification + from msg.market.email import render_verification + require(self.app.settings.server.mail is not None, 'mail_disabled') + async with self.app.metadata.transaction(write=False) as tx: + await current_principal(self.app, job.principal, tx) + outgoing = await (render_verification(self.app,tx,job) if job.kind=='market_email_verify' + else render_notification(self.app,tx,job)) + state = await self.mail_sender.send(outgoing) + require(state in {'sent', 'uncertain'}, 'invalid_delivery_result') + await self._finish(job, 'done' if state == 'sent' else 'uncertain', state) + async def _webhook(self, job): from msg.core.codec import canonical,decode,loads from msg.core.models import Event @@ -336,6 +354,13 @@ async def _webhook(self, job): async def run_once(self): if self.app.executor.recovery_drill_active(): return False + if 'orders' in self.app.settings.server.plugins: + from msg.market.escrow import resolve_due + if await resolve_due(self.app): + return True + from msg.market.arbitration import resolve_cases + if await resolve_cases(self.app): + return True job, execute = await self._claim() if job is None: return False @@ -346,6 +371,8 @@ async def run_once(self): self.app.settings.server.staging_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='effect-', dir=self.app.settings.server.staging_dir) as temp: await self._tool(job, Path(temp)) + elif job.kind in {'market_mail','market_email_verify'}: + await self._market_mail(job) elif job.kind == 'mail': await self._mail(job) elif job.kind == 'webhook': @@ -373,7 +400,7 @@ async def run_once(self): except Failure as exc: # Known pre-execution rejection is a definite failure. External calls # surface uncertain explicitly rather than inventing an exactly-once promise. - if job.kind=='mail' and exc.retryable and exc.code=='mail_connection_failed': + if job.kind in {'mail','market_mail','market_email_verify'} and exc.retryable and exc.code=='mail_connection_failed': await self._retry_mail(job,exc.code) return True await self._finish(job, 'uncertain' if exc.code in {'external_uncertain','job_lease_lost'} else 'failed', exc.code) diff --git a/src/msg/workers/maintenance.py b/src/msg/workers/maintenance.py index 4b708c9e..dae8f32a 100644 --- a/src/msg/workers/maintenance.py +++ b/src/msg/workers/maintenance.py @@ -190,6 +190,9 @@ async def _collect(app,tx, *, grace_seconds=3600): # In-flight chunks and sealed uploads are roots until the transfer expires. for raw, in tx.execute('SELECT c.body FROM chunks c JOIN transfers t ON t.id=c.transfer_id'): live.add(loads(raw)['content']['digest'][7:]) + from msg.market.references import content_references + for content in content_references(tx.execute): + live.add(content['digest'][7:]) recent=time.time()-grace_seconds removed=0 # Explicit pins are live roots even before a Revision/Transfer pointer is diff --git a/tests/test_market_arbitration.py b/tests/test_market_arbitration.py new file mode 100644 index 00000000..666acb5b --- /dev/null +++ b/tests/test_market_arbitration.py @@ -0,0 +1,210 @@ +"""Real signed panel votes; no fake principal or direct settlement mutation.""" +import asyncio +from copy import deepcopy +from datetime import timedelta + +import pytest +from test_market_lifecycle import buy, market +from test_service import NOW, call, register + +from msg.admin.market import apply_market +from msg.core.codec import canonical, digest, unb64, wire +from msg.core.errors import Failure +from msg.market.policy import DEFAULT_POLICY +from msg.plugins.money import _balance + + +async def configured(installed, *, panel_size=3, count=4, appeal=False): + app, root = installed + members = {} + for i in range(count): + key, uid, _ = await register(app, f'arbitrator-{i}') + members[uid] = key + await apply_market(app, root, action='grant', operator='isolated-fixture', subject=uid) + policy = deepcopy(DEFAULT_POLICY) + policy.update(id='test-panel-v1', candidates=list(members), panel_size=panel_size, + quorum=panel_size//2+1, allow_appeal=appeal) + await apply_market(app, root, action='publish', operator='isolated-fixture', policy=policy) + sk, seller, bk, buyer, listing, _ = await market(app, root, mode='service', kind='service', + policy=policy['id']) + purchase = await buy(app, bk, buyer, listing) + assert purchase.status == 'ok', wire(purchase) + opened = await call(app, 'orders.dispute_open', { + 'order_id':purchase.data['order']['id'], 'reason':'quality'}, key=bk, subject=buyer) + assert opened.status == 'ok', wire(opened) + return app, root, members, sk, seller, bk, buyer, purchase.data['order'], opened.data + + +async def details(app, key, subject, case_id): + result = await call(app,'orders.dispute_get',{'case_id':case_id},key=key,subject=subject) + assert result.status == 'ok', wire(result) + return result.data + + +async def vote(app, key, subject, base, *, refund=2_000_000, outcome='split'): + proposal = {**base, 'refund_minor':refund, 'outcome':outcome} + return await call(app,'orders.dispute_vote',{'proposal':proposal, + 'signature':wire(key.sign(canonical(proposal),purpose='arbitration-decision'))}, + key=key,subject=subject) + + +@pytest.mark.asyncio +async def test_fixed_panel_split_quorum_and_concurrent_execution(installed): + app, _root, members, _sk, seller, bk, buyer, order, opened = await configured(installed) + data = await details(app,bk,buyer,opened['case_id']) + panel = list(opened['panel']) + expected = sorted(members,key=lambda uid:(digest({'version':1,'order_id':order['id'], + 'policy_digest':opened['policy_digest'],'round':0,'subject':uid}),uid))[:3] + assert panel == expected and data['panel_valid'] + base = data['proposal_base'] + # The second signed statement by the same member cannot create a new vote. + first = await vote(app,members[panel[0]],panel[0],base) + assert first.status=='ok' and first.data['decision'] is None + conflict = await vote(app,members[panel[0]],panel[0],base,refund=0,outcome='release') + assert conflict.error.code=='conflicting_vote' + second = await vote(app,members[panel[1]],panel[1],base) + assert second.status=='ok', wire(second) + decision = second.data['decision'] + assert decision and decision['refund_minor']==2_000_000 + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,order['escrow_subject'] if 'escrow_subject' in order else + tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0])==5_000_000 + assert _balance(tx,seller)==0 # A vote is never a payment. + args={'case_id':opened['case_id'],'decision_id':decision['id']} + results=await asyncio.gather(*[call(app,'orders.dispute_execute',args,key=bk, + subject=buyer,rid=f'execution-{i}') for i in range(3)]) + assert all(r.status=='ok' for r in results), [wire(r) for r in results] + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,buyer)==17_000_000 and _balance(tx,seller)==3_000_000 + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?',(order['id'],))[0]==1 + assert tx.one('SELECT COUNT(*) FROM money_ledger WHERE reference LIKE ?', + ('order_%:'+order['id'],))[0]==3 # fund + split's two legs + + +@pytest.mark.asyncio +async def test_role_isolated_evidence_and_summary_has_no_private_fields(installed): + app, root, members, sk, seller, bk, buyer, order, opened = await configured(installed) + cid = opened['case_id'] + statement=await call(app,'orders.dispute_statement',{'case_id':cid, + 'visibility':'panel','text':'Only the panel may see the buyer testimony.'},key=bk,subject=buyer) + assert statement.status=='ok',wire(statement) + eid=statement.data['evidence_id'] + seller_view=await details(app,sk,seller,cid) + assert not seller_view['evidence'] + denied=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=sk,subject=seller) + missing=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':'unknown'},key=sk,subject=seller) + assert denied.error.code==missing.error.code=='evidence_not_found' + arb=opened['panel'][0] + allowed=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=members[arb],subject=arb) + assert unb64(allowed.data['data'])==b'Only the panel may see the buyer testimony.' + outsider=next(uid for uid in members if uid not in opened['panel']) + for case_id in (cid,'case_unknown'): + hidden=await call(app,'orders.dispute_get',{'case_id':case_id},key=members[outsider],subject=outsider) + assert hidden.error.code=='case_not_found' + public=await call(app,'orders.dispute_summary',{'case_id':cid}) + assert public.status=='ok' + assert set(public.data)=={'case_id','state','policy_digest','outcome'} + text=canonical(public.data).decode() + assert all(private not in text for private in (buyer,seller,order['id'],eid,'testimony')) + await apply_market(app,root,action='revoke',operator='test',subject=arb) + after=await call(app,'orders.dispute_get',{'case_id':cid},key=members[arb],subject=arb) + assert after.error.code=='case_not_found' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('invalidation',['revoke','regrant','conflict','decision_revoke','expire']) +async def test_invalidated_decision_never_releases_funds(installed,invalidation): + app, root, members, _sk, seller, bk, buyer, order, opened = await configured( + installed,panel_size=1,count=2) + cid, arb=opened['case_id'],opened['panel'][0] + base=(await details(app,bk,buyer,cid))['proposal_base'] + result=await vote(app,members[arb],arb,base,refund=0,outcome='release') + decision=result.data['decision'] + if invalidation in {'revoke','regrant'}: + await apply_market(app,root,action='revoke',operator='test',subject=arb) + if invalidation=='regrant': + await apply_market(app,root,action='grant',operator='test',subject=arb) + elif invalidation=='conflict': + declared=await call(app,'orders.arbitrator_conflict',{'party':buyer},key=members[arb],subject=arb) + assert declared.status=='ok' + elif invalidation=='decision_revoke': + revoked=await call(app,'orders.dispute_revoke',{'case_id':cid,'decision_id':decision['id']},key=members[arb],subject=arb) + assert revoked.status=='ok' + else: + app.clock=lambda:NOW+timedelta(days=10) + from msg.market.arbitration import resolve_cases + assert await resolve_cases(app)==[cid] + if invalidation!='expire': + failed=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':decision['id']},key=bk,subject=buyer) + assert failed.status=='error',wire(failed) + async with app.metadata.transaction(write=False) as tx: + escrow=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0] + assert _balance(tx,escrow)==5_000_000 and _balance(tx,seller)==0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 + + +@pytest.mark.asyncio +async def test_one_appeal_disjoint_panel_and_old_decision_is_unusable(installed): + app, _root, members, sk, seller, bk, buyer, _order, opened=await configured( + installed,panel_size=1,count=2,appeal=True) + cid,arb=opened['case_id'],opened['panel'][0] + base=(await details(app,bk,buyer,cid))['proposal_base'] + result=await vote(app,members[arb],arb,base,refund=0,outcome='release') + old=result.data['decision']['id'] + premature=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':old},key=bk,subject=buyer) + assert premature.error.code=='appeal_window_open' + appealed=await call(app,'orders.dispute_appeal',{'case_id':cid},key=bk,subject=buyer) + assert appealed.status=='ok',wire(appealed) + assert set(appealed.data['panel']).isdisjoint(opened['panel']) + again=await call(app,'orders.dispute_appeal',{'case_id':cid},key=sk,subject=seller) + assert again.error.code=='appeal_not_available' + old_attempt=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':old},key=bk,subject=buyer) + assert old_attempt.error.code=='decision_not_current' + current=(await details(app,bk,buyer,cid))['proposal_base'] + reviewer=appealed.data['panel'][0] + appeal_vote=await vote(app,members[reviewer],reviewer,current,refund=5_000_000,outcome='refund') + assert appeal_vote.status=='ok',wire(appeal_vote) + final=await call(app,'orders.dispute_execute',{'case_id':cid, + 'decision_id':appeal_vote.data['decision']['id']},key=bk,subject=buyer) + assert final.status=='ok',wire(final) + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,buyer)==20_000_000 and _balance(tx,seller)==0 + + +@pytest.mark.asyncio +async def test_split_rollback_restores_escrow_decision_and_retry(installed,monkeypatch): + app, _root, members, _sk, _seller, bk, buyer, order, opened=await configured(installed,panel_size=1,count=1) + cid,arb=opened['case_id'],opened['panel'][0] + result=await vote(app,members[arb],arb,(await details(app,bk,buyer,cid))['proposal_base']) + args={'case_id':cid,'decision_id':result.data['decision']['id']} + from msg.market import escrow + original=escrow._post_transfer + def fail_after_first_leg(*args,**kwargs): + original(*args,**kwargs) + raise Failure('forced_split_rollback') + monkeypatch.setattr(escrow,'_post_transfer',fail_after_first_leg) + failed=await call(app,'orders.dispute_execute',args,key=bk,subject=buyer) + assert failed.error.code=='forced_split_rollback' + async with app.metadata.transaction(write=False) as tx: + account=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0] + assert _balance(tx,account)==5_000_000 and _balance(tx,buyer)==15_000_000 + assert tx.one('SELECT state FROM arbitration_cases WHERE id=?',(cid,))[0]=='decided' + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 + monkeypatch.setattr(escrow,'_post_transfer',original) + retried=await call(app,'orders.dispute_execute',args,key=bk,subject=buyer) + assert retried.status=='ok',wire(retried) + + +@pytest.mark.asyncio +async def test_default_no_panel_holds_without_choosing_new_candidates(installed): + app,root=installed + _sk,seller,bk,buyer,listing,_=await market(app,root,mode='service',kind='service') + order=(await buy(app,bk,buyer,listing)).data['order'] + opened=await call(app,'orders.dispute_open',{'order_id':order['id'],'reason':'quality'},key=bk,subject=buyer) + assert opened.status=='ok' and opened.data['state']=='held' and not opened.data['panel'] + _key,uid,_=await register(app,'late-arbitrator') + await apply_market(app,root,action='grant',operator='test',subject=uid) + data=await details(app,bk,buyer,opened.data['case_id']) + assert not data['case']['panels']['0'] and not data['panel_valid'] + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,seller)==0 and tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 diff --git a/tests/test_market_delivery.py b/tests/test_market_delivery.py new file mode 100644 index 00000000..f26647e3 --- /dev/null +++ b/tests/test_market_delivery.py @@ -0,0 +1,228 @@ +"""Site authority, recipient ownership, SMTP minimisation and resumable payloads.""" +import asyncio +import subprocess +from dataclasses import replace + +import pytest +from test_market_lifecycle import buy, market +from test_service import call + +from msg.core.codec import b64, canonical, digest, loads, unb64, wire +from msg.core.errors import Failure +from msg.core.models import MailConfig +from msg.plugins.money import _balance +from msg.workers.effects import EffectWorker + + +class Sender: + def __init__(self, result='sent'): + self.messages = [] + self.result = result + + async def send(self, job): + self.messages.append(dict(job.arguments)) + if isinstance(self.result, Exception): + raise self.result + return self.result + + +def enable_mail(app): + config = MailConfig(enabled=True,host='localhost',port=2525,tls='starttls', + sender='test@example.invalid',credential_file=None) + app.settings = replace(app.settings,server=replace(app.settings.server,mail=config)) + + +async def verified_address(app,key,subject,address): + # Test the real signed account-email flow; don't manufacture verification rows. + result=await call(app,'identity.email_set',{'address':address},key=key,subject=subject) + assert result.status=='ok',wire(result) + async with app.metadata.transaction(write=False) as tx: + raw=tx.one("SELECT body FROM jobs WHERE kind='mail' AND body::jsonb->'principal'->>'subject'=? ORDER BY id DESC",(subject,)) + if raw is None: # Jobs index actor/subject in the canonical Principal, not columns. + raise AssertionError('verification job missing') + job=loads(raw[0]) + token=job['arguments']['text'].rsplit(': ',1)[1] + result=await call(app,'identity.email_verify',{'token':token},key=key,subject=subject) + assert result.status=='ok',wire(result) + + +async def drain(app,sender): + worker=EffectWorker(app,mail_sender=sender) + for _ in range(100): + if not await worker.run_once(): + return + raise AssertionError('worker did not converge') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('smtp', ['sent','uncertain','connection_failed','disabled']) +async def test_verified_checkout_mail_is_minimal_and_not_claimed(installed,smtp): + app,root=installed + enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root) + await verified_address(app,bk,buyer,'buyer@example.invalid') + if smtp=='disabled': + app.settings=replace(app.settings,server=replace(app.settings.server,mail=None)) + result=await buy(app,bk,buyer,listing,email='buyer@example.invalid') + assert result.status=='ok' and result.data['order']['state']=='settled',wire(result) + order=result.data['order']; oid=order['id'] + sender=Sender(Failure('mail_connection_failed',retryable=True) if smtp=='connection_failed' else smtp) + await drain(app,sender) + expected={'sent':'smtp_accepted','uncertain':'delivered_unknown', + 'connection_failed':'pending','disabled':'disabled'}[smtp] + got=await call(app,'orders.get',{'order_id':oid},key=bk,subject=buyer) + assert got.data['order']['email_status']==expected,wire(got) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?',(oid,))==('prepared',None) + assert _balance(tx,buyer)==15_000_000 and _balance(tx,seller)==5_000_000 + if smtp=='disabled': + assert not sender.messages + else: + assert len(sender.messages)==1 + mail=sender.messages[0] + body=loads(mail['text']) + assert body=={'order_id':oid,'handle':'@order-buyer', + 'pickup':app.settings.service_url+'/_orders/'+oid+'/_delivery'} + assert mail['recipient']=='buyer@example.invalid' + assert all(secret not in mail['text'] for secret in + ('delivery-ok','store selftest','token','buyer@example.invalid',seller)) + assert set(mail)=={'recipient','recipient_subject','subject','text'} + seller_view=await call(app,'orders.get',{'order_id':oid},key=sk,subject=seller) + assert 'buyer@example.invalid' not in canonical(seller_view).decode() + + +@pytest.mark.asyncio +async def test_unverified_checkout_has_no_order_information_until_owner_verifies(installed): + app,root=installed; enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root) + bought=await buy(app,bk,buyer,listing,email='new@example.invalid') + assert bought.status=='ok' and bought.data['order']['state']=='settled',wire(bought) + oid=bought.data['order']['id']; sender=Sender() + await drain(app,sender) + assert len(sender.messages)==1 + body=loads(sender.messages[0]['text']) + assert set(body)=={'challenge_id','token','purpose'} + assert all(value not in sender.messages[0]['text'] for value in + (oid,buyer,seller,'order-buyer','delivery-ok','store selftest','pickup')) + args={'order_id':oid,'challenge_id':body['challenge_id'],'token':body['token']} + wrong=await call(app,'orders.email_verify',args,key=sk,subject=seller) + assert wrong.error.code=='order_not_found' + confirmed=await call(app,'orders.email_verify',args,key=bk,subject=buyer,rid='verify-order') + assert confirmed.status=='ok',wire(confirmed) + repeat=await call(app,'orders.email_verify',args,key=bk,subject=buyer,rid='verify-order') + assert repeat.replayed + other=await call(app,'orders.email_verify',args,key=bk,subject=buyer) + assert other.error.code=='email_challenge_invalid' + await drain(app,sender) + assert len(sender.messages)==2 and loads(sender.messages[1]['text'])['order_id']==oid + # Order verification does not overwrite the subject's account email preferences. + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT 1 FROM emails WHERE subject=?',(buyer,)) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('change',['disable','address','wrong_owner']) +async def test_send_time_revalidation_prevents_stale_or_cross_subject_delivery(installed,change): + app,root=installed; enable_mail(app) + _sk,seller,bk,buyer,listing,_=await market(app,root) + await verified_address(app,bk,buyer,'buyer@example.invalid') + bought=await buy(app,bk,buyer,listing,email='buyer@example.invalid'); oid=bought.data['order']['id'] + if change=='disable': + result=await call(app,'orders.email_disable',{'order_id':oid},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + elif change=='address': + result=await call(app,'identity.email_set',{'address':'replacement@example.invalid'},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + else: + async with app.metadata.transaction(write=True) as tx: + row=tx.one('SELECT delivery_target FROM store_orders WHERE id=?',(oid,)) + target=loads(row[0]); target['email']['subject_id']=seller + tx.execute('UPDATE store_orders SET delivery_target=? WHERE id=?',(canonical(target).decode(),oid),write=True) + sender=Sender(); await drain(app,sender) + assert all(oid not in message['text'] for message in sender.messages) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state FROM store_orders WHERE id=?',(oid,))[0]=='settled' + assert _balance(tx,seller)==5_000_000 + + +@pytest.mark.asyncio +async def test_sealed_secret_requires_buyer_key_and_explicit_acceptance(installed,monkeypatch,tmp_path): + import msg.security.age_keys as ages + original=ages.generate_age_key; pairs=[] + def generate(): + pair=original(); pairs.append(pair); return pair + monkeypatch.setattr(ages,'generate_age_key',generate) + app,root=installed; enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root,mode='sealed_manual',kind='secret') + private,recipient=pairs[-1] + bought=await buy(app,bk,buyer,listing,email='new@example.invalid') + assert bought.status=='ok',wire(bought) + order=bought.data['order']; oid=order['id'] + encrypted=await asyncio.to_thread(subprocess.run, ['age','-r',recipient], + input=b'private-secret-123',capture_output=True,check=True) + ciphertext=encrypted.stdout + private_file=tmp_path/'identity'; private_file.write_text(private+'\n'); private_file.chmod(0o600) + decrypted=await asyncio.to_thread(subprocess.run, ['age','-d','-i',str(private_file)], + input=ciphertext,capture_output=True,check=True) + assert decrypted.stdout==b'private-secret-123' + uploaded=await call(app,'content.file_put',{'parent':'/@order-seller/files','name':'encrypted.age', + 'data':b64(ciphertext),'media_type':'application/age'},key=sk,subject=seller) + assert uploaded.status=='ok',wire(uploaded) + args={'order_id':oid,'payload_ref':wire(uploaded.resources[0]),'payload_digest':digest(ciphertext), + 'recipient_key_id':order['recipient_key']['key_id']} + wrong=await call(app,'delivery.submit',{**args,'recipient_key_id':'other-key'},key=sk,subject=seller) + assert wrong.error.code=='delivery_recipient_mismatch' + sent=await call(app,'delivery.submit',args,key=sk,subject=seller) + assert sent.status=='ok' and sent.data['state']=='delivered',wire(sent) + delivered=await call(app,'delivery.get',{'order_id':oid},key=bk,subject=buyer,contract_version=2) + assert unb64(delivered.data['delivery']['payloads'][0]['data'])==ciphertext + sender=Sender(); await drain(app,sender) + assert all('private-secret-123' not in m['text'] for m in sender.messages) + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,seller)==0 + accepted=await call(app,'delivery.accept',{'order_id':oid,'delivery_digest':sent.data['delivery_digest']}, + key=bk,subject=buyer,contract_version=2) + assert accepted.status=='ok' and accepted.data['state']=='settled',wire(accepted) + + +@pytest.mark.asyncio +async def test_large_service_transfer_has_buyer_acl_and_never_implies_acceptance(installed): + app,root=installed + sk,seller,bk,buyer,listing,_=await market(app,root,mode='service',kind='service') + body=bytes(range(256))*8193 # Larger than 2 MiB; no oversized JSON result. + opened=await call(app,'transfer.open',{'direction':'upload','size':len(body),'digest':digest(body), + 'requested_part_bytes':65536},key=sk,subject=seller) + assert opened.status=='ok',wire(opened) + tid=opened.data['transfer_id']; part=opened.data['part_bytes'] + for offset in range(0,len(body),part): + chunk=body[offset:offset+part] + result=await call(app,'transfer.part_put',{'transfer_id':tid,'offset':offset, + 'data':b64(chunk),'digest':digest(chunk)},key=sk,subject=seller) + assert result.status=='ok',wire(result) + sealed=await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(body),'final_digest':digest(body)}, + key=sk,subject=seller) + assert sealed.status=='ok',wire(sealed) + bought=await buy(app,bk,buyer,listing); oid=bought.data['order']['id'] + submitted=await call(app,'delivery.submit',{'order_id':oid,'payload_ref':wire(sealed.output), + 'payload_digest':digest(body)},key=sk,subject=seller) + assert submitted.status=='ok',wire(submitted) + before=await call(app,'delivery.get',{'order_id':oid},key=bk,subject=buyer,contract_version=2) + assert before.status=='ok' and 'data' not in before.data['delivery']['payloads'][0],wire(before) + args=before.data['delivery']['payloads'][0]['transfer']['arguments'] + exported=await call(app,'delivery.transfer_open',args,key=bk,subject=buyer) + assert exported.status=='ok',wire(exported) + transfer=exported.data['transfer_id']; output=bytearray(); part=exported.data['part_bytes'] + for offset in range(0,len(body),part): + result=await call(app,'transfer.part_get',{'transfer_id':transfer,'offset':offset, + 'length':min(part,len(body)-offset)},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + output.extend(unb64(result.data['data'])) + assert bytes(output)==body + wrong=await call(app,'transfer.part_get',{'transfer_id':transfer,'offset':0,'length':1},key=sk,subject=seller) + assert wrong.error.code=='transfer_owner_required' + async with app.metadata.transaction(write=False) as tx: + resource=await tx.resource(exported.resources[0].id) + assert resource.owner==buyer and resource.mode==0o600 + assert _balance(tx,seller)==0 + assert tx.one('SELECT state FROM store_deliveries WHERE order_id=?',(oid,))[0]=='prepared' + assert tx.one('SELECT state FROM store_orders WHERE id=?',(oid,))[0]=='delivered' diff --git a/tests/test_market_lifecycle.py b/tests/test_market_lifecycle.py new file mode 100644 index 00000000..f9d3df2e --- /dev/null +++ b/tests/test_market_lifecycle.py @@ -0,0 +1,141 @@ +"""The v2 market contract: real signatures, immutable snapshots and conserved money.""" +import asyncio +from datetime import timedelta + +import pytest +from test_orders import _intent, _sale +from test_service import NOW, call, register + +from msg.admin.money import apply_money +from msg.core.codec import unb64, wire + + +async def market(app, root, *, mode='managed_instant', kind='bundle', quantity=2, + policy='dispute-v1'): + seller_key, seller, _ = await register(app, 'order-seller') + buyer_key, buyer, _ = await register(app, 'order-buyer') + if mode == 'managed_instant': + listing, package = await _sale(app, seller_key, seller, quantity=quantity) + else: + created = await call(app, 'store.listing_create', { + 'name': 'sealed sale', 'item_kind': kind, 'price_minor': 5_000_000, + 'currency_id': 'primary', 'quantity': quantity, 'delivery_mode': mode, + 'escrow_policy': 'escrow-v1', 'dispute_policy': policy, + 'terms': 'Delivery within one day; manual acceptance.'}, + key=seller_key, subject=seller) + assert created.status == 'ok', wire(created) + active = await call(app, 'store.listing_update', { + 'id': created.resources[0].id, 'state': 'active'}, + key=seller_key, subject=seller, + expected=((created.resources[0].id, created.data['generation']),)) + assert active.status == 'ok', wire(active) + listing, package = active.data['listing'], None + await apply_money(app, root, action='mint', operator='test-console', amount_minor=20_000_000) + await apply_money(app, root, action='transfer', operator='test-console', + subject_id=buyer, amount_minor=20_000_000) + return seller_key, seller, buyer_key, buyer, listing, package + + +async def buy(app, key, buyer, listing, **extra): + return await call(app, 'orders.buy', {**_intent(listing), **extra}, + key=key, subject=buyer, contract_version=2) + + +@pytest.mark.asyncio +async def test_instant_delivery_settlement_and_claim_are_distinct(installed): + app, root = installed + _sk, seller, bk, buyer, listing, package = await market(app, root) + args = _intent(listing) + result = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=2, rid='instant-once') + assert result.status == 'ok', wire(result) + order = result.data['order'] + assert order['state'] == 'settled' + assert order['package_digest'] == package['digest'] + replay = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=2, rid='instant-once') + assert replay.replayed and replay.data == result.data + delivery = await call(app, 'delivery.get', {'order_id': order['id']}, + key=bk, subject=buyer, contract_version=2) + assert delivery.status == 'ok', wire(delivery) + data = delivery.data['delivery'] + assert data['state'] == 'prepared' and data['claimed_at'] is None + assert data['manifest']['text'] == 'msg.lmm.best store selftest' + assert unb64(data['payloads'][0]['data']) == b'delivery-ok\n' + async with app.metadata.transaction(write=False) as tx: + escrow = tx.one('SELECT escrow_subject FROM store_orders WHERE id=?', (order['id'],))[0] + from msg.plugins.money import _balance + assert _balance(tx, buyer) == 15_000_000 + assert _balance(tx, seller) == 5_000_000 + assert _balance(tx, escrow) == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?', (order['id'],))[0] == 1 + count = tx.one('SELECT COUNT(*) FROM money_ledger')[0] + for _ in range(2): + claimed = await call(app, 'delivery.accept', {'order_id': order['id'], + 'delivery_digest': data['delivery_digest']}, key=bk, subject=buyer, contract_version=2) + assert claimed.status == 'ok', wire(claimed) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM money_ledger')[0] == count + + +@pytest.mark.asyncio +async def test_created_payment_snapshot_and_cancel_before_fund(installed): + app, root = installed + sk, seller, bk, buyer, listing, _ = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + assert created.status == 'ok', wire(created) + order = created.data['order'] + assert order['state'] == 'created' and order['funded_at'] is None + changed = await call(app, 'store.listing_update', { + 'id': listing['listing_id'], 'price_minor': 7_000_000, 'terms': 'Later terms'}, + key=sk, subject=seller, expected=((listing['listing_id'], 2),)) + assert changed.status == 'ok', wire(changed) + funded = await call(app, 'orders.fund', {'order_id': order['id'], + 'order_digest': order['order_digest'], 'total_price_minor': 5_000_000, + 'currency_id': 'primary'}, key=bk, subject=buyer) + assert funded.status == 'ok' and funded.data['order']['state'] == 'settled', wire(funded) + assert funded.data['order']['listing_revision'] == listing['listing_revision'] + assert funded.data['order']['terms_digest'] == listing['terms_digest'] + other = await call(app, 'orders.create', _intent(changed.data['listing']), key=bk, subject=buyer) + cancelled = await call(app, 'orders.cancel', {'order_id': other.data['order']['id']}, + key=bk, subject=buyer, contract_version=2) + assert cancelled.status == 'ok' and cancelled.data['order']['state'] == 'cancelled' + + +@pytest.mark.asyncio +async def test_objective_missing_package_refunds_atomically(installed): + app, root = installed + _sk, seller, bk, buyer, listing, package = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + order = created.data['order'] + async with app.metadata.transaction(write=True) as tx: + tx.execute('DELETE FROM store_packages WHERE id=?', (package['id'],), write=True) + funded = await call(app, 'orders.fund', {'order_id': order['id'], + 'order_digest': order['order_digest'], 'total_price_minor': 5_000_000, + 'currency_id': 'primary'}, key=bk, subject=buyer) + assert funded.status == 'ok' and funded.data['order']['state'] == 'refunded', wire(funded) + async with app.metadata.transaction(write=False) as tx: + from msg.plugins.money import _balance + assert _balance(tx, buyer) == 20_000_000 and _balance(tx, seller) == 0 + assert tx.one('SELECT COUNT(*) FROM store_deliveries')[0] == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0] == 1 + + +@pytest.mark.asyncio +async def test_last_quantity_and_timeout_do_not_double_spend(installed): + app, root = installed + _sk, seller, bk, buyer, listing, _ = await market(app, root, mode='service', kind='service', quantity=1) + results = await asyncio.gather(*(buy(app, bk, buyer, listing) for _ in range(2))) + assert sum(r.status == 'ok' for r in results) == 1 + assert [r.error.code for r in results if r.error] == ['quantity_unavailable'] + order = next(r.data['order'] for r in results if r.status == 'ok') + assert order['state'] == 'funded' + app.clock = lambda: NOW + timedelta(days=2) + from msg.market.escrow import resolve_due + await resolve_due(app) + await resolve_due(app) + async with app.metadata.transaction(write=False) as tx: + from msg.plugins.money import _balance + assert tx.one('SELECT state FROM store_orders WHERE id=?', (order['id'],))[0] == 'refunded' + assert _balance(tx, buyer) == 20_000_000 and _balance(tx, seller) == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0] == 1 diff --git a/tests/test_market_recovery.py b/tests/test_market_recovery.py new file mode 100644 index 00000000..e1f3d12c --- /dev/null +++ b/tests/test_market_recovery.py @@ -0,0 +1,89 @@ +"""Real restart/pg_dump/restore and readonly diagnostics for private market state.""" +import asyncio + +import pytest +from test_market_arbitration import configured, details, vote +from test_service import NOW, call + +from msg.admin.backups import backup, restore +from msg.admin.market_check import inspect_market +from msg.application import Application +from msg.config import load_settings +from msg.core.codec import unb64, wire +from msg.core.errors import Failure +from msg.workers.effects import EffectWorker + + +async def snapshot(tx): + tables=('store_orders','order_contracts','order_settlements','order_transitions', + 'arbitration_cases','arbitration_votes','arbitration_decisions','arbitration_evidence', + 'money_ledger','jobs','events','audit','results','settings') + return {name:tx.rows('SELECT * FROM '+name+' ORDER BY 1') for name in tables} + + +@pytest.mark.asyncio +async def test_restart_and_backup_preserve_private_case_decision_without_reexecution(installed,tmp_path,pg_dsn): + app,_root,members,sk,seller,bk,buyer,_order,opened=await configured(installed) + cid=opened['case_id']; text='Private sealed dispute evidence. Never public.' + stored=await call(app,'orders.dispute_statement',{'case_id':cid,'visibility':'panel','text':text},key=bk,subject=buyer) + assert stored.status=='ok',wire(stored) + eid=stored.data['evidence_id'] + data=await details(app,bk,buyer,cid) + for uid in opened['panel'][:2]: + cast=await vote(app,members[uid],uid,data['proposal_base']) + assert cast.status=='ok' and cast.data['decision'],wire(cast) + args={'case_id':cid,'decision_id':cast.data['decision']['id']} + # A new Application reads the same authoritative metadata; no in-memory panel. + restarted=Application(app.settings,clock=lambda:NOW) + await restarted.load() + try: + result=await call(restarted,'orders.dispute_execute',args,key=bk,subject=buyer) + assert result.status=='ok',wire(result) + finally: + await restarted.close() + archive=tmp_path/'market.zip' + await backup(app,archive) + async with app.metadata.transaction(write=False) as tx: + original=await snapshot(tx) + report=await inspect_market(tx) + assert report['orders']==1 + assert await snapshot(tx)==original + restored_config=tmp_path/'restored-etc'; restored_data=tmp_path/'restored-data' + await asyncio.to_thread(restore,archive,restored_config,restored_data,postgres_dsn=pg_dsn) + restored=Application(load_settings(restored_config),clock=lambda:NOW) + await restored.load() + try: + async with restored.metadata.transaction(write=False) as tx: + await inspect_market(tx) + # Recovery intentionally changes safety settings, not the ledger/case. + after=await snapshot(tx) + for table in set(original)-{'settings','results','events','audit'}: + assert after[table]==original[table],table + evidence=await call(restored,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=bk,subject=buyer) + assert evidence.status=='ok' and unb64(evidence.data['data']).decode()==text,wire(evidence) + other=await call(restored,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=sk,subject=seller) + assert other.error.code=='evidence_not_found' + assert await EffectWorker(restored).run_once() is False + rejected=await call(restored,'orders.dispute_execute',args,key=bk,subject=buyer) + assert rejected.error.code=='writes_paused' + finally: + await restored.close() + + +@pytest.mark.asyncio +async def test_market_doctor_cannot_mutate_or_synthesize_a_repair(installed): + from test_market_lifecycle import buy, market + app,root=installed + _sk,_seller,bk,buyer,listing,_=await market(app,root) + result=await buy(app,bk,buyer,listing) + assert result.status=='ok' + async with app.metadata.transaction(write=False) as tx: + before=await snapshot(tx) + assert (await inspect_market(tx))['escrow_invariants'] + assert await snapshot(tx)==before + # Snapshot/settlement facts are append-only, even to a privileged DB session. + with pytest.raises(Failure,match='constraint_conflict'): + async with app.metadata.transaction(write=True) as tx: + tx.execute("UPDATE order_contracts SET digest='forged'",write=True) + async with app.metadata.transaction(write=False) as tx: + assert await snapshot(tx)==before diff --git a/tests/test_orders.py b/tests/test_orders.py index 7885d67a..9e7933b8 100644 --- a/tests/test_orders.py +++ b/tests/test_orders.py @@ -301,7 +301,7 @@ async def test_cancel_refuses_delivery_and_rolls_back_after_refund_post(installe key=buyer_key, subject=buyer) assert first.status == second.status == 'ok' first_id, second_id = first.data['order']['id'], second.data['order']['id'] - from msg.plugins import orders + from msg.market import escrow as orders original = orders._post_transfer def abort_after_refund(*args, **kwargs): From c1de28f052ef59971b45213f242749bc5810a2db Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 04:42:00 +0800 Subject: [PATCH 6/9] test: namespace nested regression fixtures to preserve root conftest imports --- tests/issue_78_80/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/issue_78_80/__init__.py diff --git a/tests/issue_78_80/__init__.py b/tests/issue_78_80/__init__.py new file mode 100644 index 00000000..8eb92cfd --- /dev/null +++ b/tests/issue_78_80/__init__.py @@ -0,0 +1,6 @@ +"""Keep this isolated suite's conftest out of the root fixture namespace. + +The full PostgreSQL suite imports installed from tests/conftest.py. Without +this package boundary, pytest loads the nested fixture as top-level conftest +and breaks collection of the full suite. No tests or fixtures are excluded. +""" From 914cad6ab2acf14d056134f7bec6f234c0e23d01 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 05:18:23 +0800 Subject: [PATCH 7/9] test: resolve read-tree helpers inside their package --- tests/issue_78_80/test_read_tree_transports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/issue_78_80/test_read_tree_transports.py b/tests/issue_78_80/test_read_tree_transports.py index 61299f8b..1bcacac7 100644 --- a/tests/issue_78_80/test_read_tree_transports.py +++ b/tests/issue_78_80/test_read_tree_transports.py @@ -12,7 +12,7 @@ from msg.transports.client import HTTPTransport,PathGETTransport,MCPHTTPTransport from msg.transports.dictionary import build_dictionary,read_query_path_document from msg.transports.read_tree_path import decode_read_tree_path -from test_read_query_tree import tree,query +from .test_read_query_tree import tree,query PATH='/_r/q/3/r/r_tree/n/1/f/i/x/children/n/1/f/i/x/children/n/1/f/i/up/1/up/1' @@ -28,7 +28,7 @@ def test_path_tree_is_readable_and_round_trips_without_json_payload(): args,proof=decode_read_tree_path(PATH.encode()) assert args=={**query(),'query_version':3} assert proof is None - aliased,suffix=decode_read_tree_path(PATH.replace('/_r/','/_read/').encode()+b'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/p/signed') + aliased,suffix=decode_read_tree_path(PATH.replace('/_r/','/_read/') .encode()+b'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/p/signed') assert aliased==args and suffix==b'signed' From 0b88bbfbd56c5d28c5fd65b7ee24a43320a97b70 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 05:18:45 +0800 Subject: [PATCH 8/9] test: use package-relative QueryRef regression helpers --- tests/issue_78_80/test_tree_query_ref.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/issue_78_80/test_tree_query_ref.py b/tests/issue_78_80/test_tree_query_ref.py index fa267c83..de0f6651 100644 --- a/tests/issue_78_80/test_tree_query_ref.py +++ b/tests/issue_78_80/test_tree_query_ref.py @@ -7,8 +7,8 @@ from msg.core.models import ResourceRef from msg.transports.http import create_app from msg.transports.client import HTTPTransport -from test_read_query_tree import tree,query -from test_read_tree_transports import business_state +from .test_read_query_tree import tree,query +from .test_read_tree_transports import business_state async def seal_query(h,args): From 5983f31f38abfe8b4f8b9c1d013dd41a8b9d80fe Mon Sep 17 00:00:00 2001 From: LIghtJUNction <106986785+LIghtJUNction@users.noreply.github.com> Date: Mon, 28 Sep 2026 05:46:15 +0000 Subject: [PATCH 9/9] fix(market): make emailed pickup and unpaid payment paths usable Bridge authenticated lifecycle delivery reads without changing legacy checkout contracts. Return pending payment with no fabricated receipt before funding. Add real HTTP email-link, privacy, HEAD and read-only regressions. Refs #73, #74, #75. --- src/msg/plugins/delivery.py | 3 ++ src/msg/plugins/orders.py | 7 ++- tests/test_market_paths.py | 89 +++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 tests/test_market_paths.py diff --git a/src/msg/plugins/delivery.py b/src/msg/plugins/delivery.py index 7d6fc448..3658777d 100644 --- a/src/msg/plugins/delivery.py +++ b/src/msg/plugins/delivery.py @@ -159,6 +159,9 @@ async def get(ctx, request, tx): require(buyer is not None and ctx.principal.actor == buyer, 'order_not_found') order = _buyer_order(tx, request.arguments['order_id'], buyer) + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (order['id'],)): + from msg.market.delivery import read + return await read(app, tx, ctx, request) delivery = _delivery(tx, order['id']) require(delivery is not None, 'delivery_not_found') await _verified_delivery(app, tx, order, delivery) diff --git a/src/msg/plugins/orders.py b/src/msg/plugins/orders.py index 9777d7ce..3c4d0594 100644 --- a/src/msg/plugins/orders.py +++ b/src/msg/plugins/orders.py @@ -243,8 +243,11 @@ async def list_orders(ctx, request, tx): async def payment(ctx, request, tx): viewer = _viewer(ctx) row = _row(tx, request.arguments['order_id'], viewer) + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (row['id'],)): + from msg.market.policy import contract + contract(tx, row['id']) result = {'order_id': row['id'], - 'status': 'refunded' if row['state'] == 'refunded' else 'funded', + 'status': _view(row, viewer)['payment_status'], 'amount_minor': row['total_price_minor'], 'currency_id': row['currency_id']} if viewer == row['buyer']: @@ -254,7 +257,7 @@ async def payment(ctx, request, tx): (transaction_id,)) require(ledger is not None, 'order_not_found') receipts.append(loads(ledger[0])) - result['receipt'] = receipts[0] + result['receipt'] = receipts[0] if receipts else None result['receipts'] = receipts result['escrow_balance_minor'] = _balance(tx, row['escrow_subject']) return HandlerOutput(data={'payment': result}) diff --git a/tests/test_market_paths.py b/tests/test_market_paths.py new file mode 100644 index 00000000..6229d171 --- /dev/null +++ b/tests/test_market_paths.py @@ -0,0 +1,89 @@ +"""Published pickup/payment URLs remain private read-only lifecycle projections.""" +import httpx +import pytest +from test_market_delivery import Sender, drain, enable_mail, verified_address +from test_market_lifecycle import buy, market +from test_order_paths import _headers +from test_orders import _intent +from test_service import call, register + +from msg.core.codec import loads, unb64, wire +from msg.transports.http import create_app + + +async def snapshot(app): + async with app.metadata.transaction(write=False) as tx: + return {table: tx.rows(f'SELECT * FROM {table} ORDER BY 1') for table in ( + 'store_orders', 'store_deliveries', 'money_ledger', 'events', 'audit')} + + +@pytest.mark.asyncio +async def test_emailed_pickup_url_reads_new_delivery_without_claiming(installed): + app, root = installed + enable_mail(app) + sk, seller, bk, buyer, listing, _ = await market(app, root) + ok, outsider, _ = await register(app, 'order-outsider') + await verified_address(app, bk, buyer, 'buyer@example.invalid') + bought = await buy(app, bk, buyer, listing, email='buyer@example.invalid') + assert bought.status == 'ok', wire(bought) + oid = bought.data['order']['id'] + sender = Sender() + await drain(app, sender) + assert len(sender.messages) == 1 + message = loads(sender.messages[0]['text']) + assert message['order_id'] == oid + before = await snapshot(app) + args = {'order_id': oid} + headers = _headers(app, 'delivery.get', args, bk, buyer) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + response = await http.get(message['pickup'], headers=headers) + assert response.status_code == 200, response.text + delivery = response.json()['delivery'] + assert delivery['order_id'] == oid and delivery['state'] == 'prepared' + assert delivery['claimed_at'] is None + assert unb64(delivery['payloads'][0]['data']) == b'delivery-ok\n' + alias = f'/@order-buyer/orders/{oid}/_delivery' + assert (await http.get(alias, headers=headers)).json() == response.json() + head = await http.head(message['pickup'], headers=headers) + assert head.status_code == 200 and head.headers['etag'] == response.headers['etag'] + for key, subject in ((None, None), (sk, seller), (ok, outsider)): + auth = {} if key is None else _headers(app, 'delivery.get', args, key, subject) + denied = await http.get(message['pickup'], headers=auth) + missing_id = 'ord_' + 'z' * 32 + missing_auth = {} if key is None else _headers(app, 'delivery.get', + {'order_id': missing_id}, key, subject) + missing = await http.get(f'/_orders/{missing_id}/_delivery', headers=missing_auth) + assert denied.status_code in {403, 404} and denied.status_code == missing.status_code + assert await snapshot(app) == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize('cancelled', [False, True]) +async def test_unfunded_payment_url_has_no_fabricated_receipt_or_payment(installed, cancelled): + app, root = installed + sk, seller, bk, buyer, listing, _ = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + assert created.status == 'ok', wire(created) + oid = created.data['order']['id'] + if cancelled: + result = await call(app, 'orders.cancel', {'order_id': oid}, key=bk, + subject=buyer, contract_version=2) + assert result.status == 'ok', wire(result) + before = await snapshot(app) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + for key, subject in ((bk, buyer), (sk, seller)): + response = await http.get(f'/_orders/{oid}/_payment', headers=_headers( + app, 'orders.payment', {'order_id': oid}, key, subject)) + assert response.status_code == 200, response.text + payment = response.json()['payment'] + assert payment['status'] == 'pending' + assert payment['amount_minor'] == 5_000_000 + assert payment['currency_id'] == 'primary' + if subject == buyer: + assert payment['receipt'] is None and payment['receipts'] == [] + assert payment['escrow_balance_minor'] == 0 + else: + assert 'receipts' not in payment and 'escrow_balance_minor' not in payment + assert await snapshot(app) == before