diff --git a/.github/workflows/recovery-safety.yml b/.github/workflows/recovery-safety.yml new file mode 100644 index 00000000..ef4c703e --- /dev/null +++ b/.github/workflows/recovery-safety.yml @@ -0,0 +1,75 @@ +name: Recovery safety +on: + push: + branches: [main, fix/issues-68-70-recovery-safety] + pull_request: +permissions: + contents: read +concurrency: + group: recovery-safety-${{ github.ref }} + cancel-in-progress: true +jobs: + recovery: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest -d postgres" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install real cryptographic and database tools + run: | + sudo apt-get update + sudo apt-get install -y age postgresql-client + python -m pip install -e '.[dev]' + - name: Compile and collect the complete suite + run: | + python -m compileall -q src + mkdir -p artifacts + python -m pytest --collect-only -q tests >artifacts/recovery-collection.txt 2>&1 || { cat artifacts/recovery-collection.txt; exit 1; } + tail -n 3 artifacts/recovery-collection.txt + - name: Verify recovery against real PostgreSQL and age + run: | + python -m pytest -q \ + tests/test_recovery_safety_units.py \ + tests/test_custody_exit_evidence.py \ + tests/test_custodial_upgrade.py \ + tests/test_custodial_historical_migration.py \ + tests/test_custodial_inventory_ack.py \ + tests/test_custodial_rewrap.py \ + tests/test_root_rotation.py \ + tests/test_root_rotation_resume.py \ + tests/test_root_storage_layout.py \ + tests/test_authority_snapshot_drift.py \ + tests/test_backup_drill.py \ + tests/test_backup_daemon.py \ + --junitxml=artifacts/recovery-safety.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: recovery-safety-verification + path: artifacts/ diff --git a/docs/ARCHITECTURE_REUSE.md b/docs/ARCHITECTURE_REUSE.md new file mode 100644 index 00000000..2e371de9 --- /dev/null +++ b/docs/ARCHITECTURE_REUSE.md @@ -0,0 +1,55 @@ +# Architecture reuse + +The authoritative design approves fewer independent mechanisms, not fewer +visible capabilities. Keep published operation versions, short codes, signed +bytes, IDs and historical receipts compatible. Existing completed-contract +regressions remain binding. + +## Implemented slice: order notifications + +`market/delivery_targets.py` owns shared account-email lookup, address validation, +mail enablement and authenticated pickup-link construction. Legacy checkout and +v3 keep their endpoint encoding and message formats; sharing a helper does not +reinterpret their signed contracts. Endpoint generation, owner, current key and +notification opt-in are still revalidated before sending. Control characters, +URL credentials, query/fragment secrets and malformed origins fail closed. + +A Delivery says what was provided to which buyer. A notification EffectJob says +whether SMTP work was queued, accepted, failed or uncertain. Neither proves buyer +acceptance or permits settlement. V3 order reads derive notification state from +the current endpoint's job and result, without editing order rows or trusting a +stale cached success. Missing jobs cannot prove SMTP delivery. Failed jobs stay +failed; disabled endpoints and uncertain attempts are not reactivated by reads. + +The shared effect completion/retry/lease fence writes job facts, not market +presentation fields. Its attempt, state, deadline and current-authority checks +remain unchanged. The notification read path checks the linked job's order, +endpoint, kind and principal before displaying it. Seller views still exclude +buyer email addresses. + +## Remaining convergence + +ServerOffer should become a Listing compatibility view, Purchase an Order, +and subject/order/bounty accounts use one protected ledger implementation. +Retain typed account purposes, server entitlement authority, atomic accounting +and explicit migrations. This slice does not implement those model migrations. + +ConsignmentPackage is immutable content; Delivery is a recipient-bound fact. +Recovery and delivery envelopes can share codecs, not authority or secret-release +policies. Notes and similar content share Resource/Revision, while membership, +certificates, grants and money keep their controlled operations and constraints. + +OperationSpec is the source of operation metadata; routes/help derive from it. +Specific business functions use the existing executor transaction. They do not +need another Engine/Manager/Repository hierarchy. Worker scheduling is shared; +channel executors and secret/network permissions remain isolated. + +New default explicit acceptance requires a new operation or policy version. +Never change orders.buy@1/@2/@3 in place. Keep automatic-delivery/settlement +regressions and distinguish their policy settlement from signed buyer claims. +Money held before fulfillment may be refunded from escrow; already-settled root +funds still require the local root command. Same request and payload replay the +original non-secret result, not another payout or another one-time secret. + +Refs #85, #71, #72 and #81. No production deployment or financial operation is +part of this refactor. diff --git a/docs/AUTHORIZATION_SOURCES.md b/docs/AUTHORIZATION_SOURCES.md new file mode 100644 index 00000000..a036dad3 --- /dev/null +++ b/docs/AUTHORIZATION_SOURCES.md @@ -0,0 +1,61 @@ +# Current authorization sources (issue #76) + +Every adapter uses OperationExecutor and the current Authorizer. A signature, +resource ID, cursor, historical revision or role name is never a grant by itself. +Credential ceilings, entry restrictions, live security-parent traversal and +special type/state gates apply before an alternative source can authorize access. + +| Source | Live conditions | Losing the source | +|---|---|---| +| Owner/mode | Current owner, mode, active security-parent chain | Owner/mode/state change takes effect immediately | +| Organization/group | Active organization and matching active membership; `public` remains virtual | Leave, removal, archived/missing organization or mismatched restored row denies that source | +| Topic governance | Current membership policy; bans on every ancestor topic constrain writes | A nested topic cannot escape a parent ban | +| Certificate | Current issuer chain, grants, authority sources, scope and credential ceiling | Revocation, expiry, source loss or narrower scope is rechecked | +| ShareGrant@2 | Ordinary shareable resource; current owner at root, active user/group, exact read operation and empty constraints | Expiry/revocation/owner change removes this source, not an independent one | +| Reshare | Same resource, fixed source chain, each parent currently allows reshare; child expiry no later than current parent | Cycles, missing parents, shortened expiry, unknown operations/constraints fail closed | +| ShareLink | Disabled by default; live owner, source credential and resource checks | Token is not public access; expiry/revocation/owner or credential changes deny it | + +The first supported ShareGrant@2 contract remains **read only, empty constraints**. +Unknown constraints and future operation names are rejected, not silently ignored. +This is not a general delegation evaluator. Containers, controlled/system facts, +DMs, SOUL/AGENTS personal roots, Todo and preview resources are not shareable. +Grant creation, source-chain validation and shared/link reads use one boundary +predicate. Sharing a child does not open its parent or another attachment. + +Ordinary read/write capabilities replace only their named checks. A group or +Topic role never implies CA, system, root, purge, chown or credential-management +rights. Topic bans constrain writes even when an outer source permits them; +independent read sources remain independent. Losing a group role does not remove +an otherwise active membership. Owner transfer and leaving are separate facts. + +## Regression evidence + +`tests/test_authorization_matrix.py` uses one nine-source table: owner, mode, +inherited traversal, organization member/maintainer/owner, scoped certificate, +user ShareGrant and group ShareGrant. Each source is established and removed by +real operations (with one explicit current-parent relocation fixture). It checks +current content, history and pinned revisions through HTTP, PathGET, GraphQL and +MCP, plus the actual CLI parser/dispatcher with its real signed client. It also +checks Search, Sync, Inbox and Outbox after revocation. Reused adapters may cache +transport descriptions, never positive resource authorization. + +`tests/test_authorization_sources.py` covers archived organizations with stale +membership rows, independent source preservation, restored unknown operations +and constraints, contracted parent expiry, newly protected previews and ancestor +topic bans. `tests/test_authorization_recovery_boundaries.py` proves that a shared +post does not open an attachment; concurrent reads cannot retain access after a +completed revocation; ShareLinks work only through supported body transports; +and a real pg_dump/restore preserves denial of content/history/old revisions and +filters notifications. Restored workers remain disabled in a recovery drill. + +These 20 new cases passed locally with real PostgreSQL 14.24, Git and signatures +on Python 3.13.5. That evidence does not replace the final commit's Python 3.15, +PostgreSQL 16, Valkey, full core suite, conformance and build CI. + +The new matrix complements `test_share_grants*`, `test_share_links`, +`test_organization_governance`, `test_topic_governance`, `test_authorization`, +`test_linkset_diff` and `test_search_source_relation_filters`, including explicit +role boundaries, organization policy changes, Topic virtual-event privacy and +old search cursor revalidation. Reads do not repair stale facts or append +business events, jobs, audit records or revisions. No production data, Root +credentials, deployment configuration or signed CA ceilings are changed. diff --git a/docs/DESIGN_GAP_REPORT.md b/docs/DESIGN_GAP_REPORT.md index baec327e..75e42baf 100644 --- a/docs/DESIGN_GAP_REPORT.md +++ b/docs/DESIGN_GAP_REPORT.md @@ -1,6 +1,18 @@ # 项目设计差距报告 -## 当前工作树进度与验收边界 +## #73–75：版本化市场实现（2026-09-28） + +订单快照、自动交付、独立 checkout 邮箱验证和确定性仲裁已实现；接口、最终性、默认无仲裁员、旧 `@1` 兼容及运维边界集中见 [MARKET_CONTRACTS.md](MARKET_CONTRACTS.md)。下方 `135a190` 及更早测试数字属于历史提交，不能用于本批验收。 + +| Issue | 实现与测试入口 | 保留的部署边界 | +| --- | --- | --- | +| #73 | `market/orders.py`、`escrow.py`；不可变签名快照、显式状态表、单一资金释放、原子分账；`test_market_lifecycle.py` | 不改已签名 `orders.buy@1` 的含义；生产库迁移单独审查 | +| #74 | `market/delivery.py`、`targets.py`、`email.py`；managed 自动结算但不 claimed，sealed/service 显式验收，真实 Transfer 和买家 endpoint 复核；`test_market_delivery.py` | SMTP 只发最小通知；本地 fake sender 不代表真实 SMTP 验收 | +| #75 | `market/policy.py`、`arbitration.py`；固定候选/epoch/panel/quorum、一次申诉、私有证据、有效签名 Decision 由 escrow 原子消费；`test_market_arbitration.py` | 默认候选为空；撤权/缺 quorum 持款，不临时挑人或让 AI 自由裁量 | + +`market_lifecycle` 隔离 selftest 覆盖 20 MSG 注资、10 MSG PoP 奖励、5 MSG 固定 bundle 自动交易及签名仲裁退款；只读 doctor 检查市场不变量。`test_market_recovery.py` 验证真实备份恢复、CAS 证据及恢复演练副作用隔离。验证结果以本批 PR 的具体 head 和 CI 为准，不能把这些模块扩写为整个项目或生产交易已验收。 + +## 历史：135a190 时点的工作树进度与验收边界 本次重新读取[ChatGPT权威设计](https://docs.google.com/document/d/1EM5Qr5qdg6tAFi2wvY0EBm6zxMj6DTBMc_dybU5qkz0/edit)，当前191个非空段；[已完成部分](https://docs.google.com/document/d/1FtTdF5uhBPAsi-so-jOfpsiVI19RWKgx6bzIEFvpR2E/edit)当前71个非空段，含A19/A20等归档条款。历史段数和旧CI仅作沿革；归档不取消未迁出需求及回归要求。 diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index b1ae1574..54df37d6 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,6 +1,18 @@ # 实现范围与需求差异 -## 当前工作树进度与验收边界 +## #73–75：版本化市场实现（2026-09-28） + +订单快照、自动交付、独立 checkout 邮箱验证和确定性仲裁已实现；接口、最终性、默认无仲裁员、旧 `@1` 兼容及运维边界集中见 [MARKET_CONTRACTS.md](MARKET_CONTRACTS.md)。下方 `135a190` 及更早测试数字属于历史提交，不能用于本批验收。 + +| Issue | 实现与测试入口 | 保留的部署边界 | +| --- | --- | --- | +| #73 | `market/orders.py`、`escrow.py`；不可变签名快照、显式状态表、单一资金释放、原子分账；`test_market_lifecycle.py` | 不改已签名 `orders.buy@1` 的含义；生产库迁移单独审查 | +| #74 | `market/delivery.py`、`targets.py`、`email.py`；managed 自动结算但不 claimed，sealed/service 显式验收，真实 Transfer 和买家 endpoint 复核；`test_market_delivery.py` | SMTP 只发最小通知；本地 fake sender 不代表真实 SMTP 验收 | +| #75 | `market/policy.py`、`arbitration.py`；固定候选/epoch/panel/quorum、一次申诉、私有证据、有效签名 Decision 由 escrow 原子消费；`test_market_arbitration.py` | 默认候选为空；撤权/缺 quorum 持款，不临时挑人或让 AI 自由裁量 | + +`market_lifecycle` 隔离 selftest 覆盖 20 MSG 注资、10 MSG PoP 奖励、5 MSG 固定 bundle 自动交易及签名仲裁退款；只读 doctor 检查市场不变量。`test_market_recovery.py` 验证真实备份恢复、CAS 证据及恢复演练副作用隔离。验证结果以本批 PR 的具体 head 和 CI 为准，不能把这些模块扩写为整个项目或生产交易已验收。 + +## 历史：135a190 时点的工作树进度与验收边界 本次实时读取[ChatGPT权威设计](https://docs.google.com/document/d/1EM5Qr5qdg6tAFi2wvY0EBm6zxMj6DTBMc_dybU5qkz0/edit)，修订 `ANLCKQmHD9c-ju8OTbJ2QIem9CjzQKIpNwVNIxKrhZ47O54VGNYCNnqIaJ8czWILEXFuaBxzxZ8_A9oocGySQwmmVmTnXAubcqo05Xpa6o8`；[已完成部分](https://docs.google.com/document/d/1FtTdF5uhBPAsi-so-jOfpsiVI19RWKgx6bzIEFvpR2E/edit)修订`ANLCKQmjW4TG7EcyxL6DyAXC86ij36Ex01x5DzJ1NjrlYcJOn4GJcC_12tos9Q12O-Vv5NXH6SBjID-dAdVA13vnqhISDbYzDf_1QJXUKE4`。归档文件明确：首次迁出由e126539/CI36306888836支撑，后续条款按各自提交和CI；已迁入条款的完成范围不延伸到整章。 diff --git a/docs/ISSUES_68_70_PROGRESS.md b/docs/ISSUES_68_70_PROGRESS.md new file mode 100644 index 00000000..94c5e9e7 --- /dev/null +++ b/docs/ISSUES_68_70_PROGRESS.md @@ -0,0 +1,96 @@ +# Recovery safety: issues #68–70 + +This patch is an isolated implementation increment, not production promotion or +acceptance of the three umbrella issues. No production data, root key, PIN, +certificate, balance or outbound endpoint was used. + +## Implemented + +### #68 — independently verifiable migration evidence + +`identity.custodial_upgrade_inventory` now re-verifies each persisted ACK against +its original subject, challenge, mapping, plaintext digest, request ID and new +signing key. Counting rows is not verification. Changed, missing, malformed or +cross-challenge evidence remains pending. New mappings explicitly identify both +encryption subkeys; retained legacy mappings are bound through their challenge. +An inventory delta also sets `historical_revisions_require_migration`, even if +all ACKs for the old list were valid. + +The original token may inspect a pending challenge; after an identity switch, +only the challenge's new signing key may inspect its completed result. The +read reports `identity_switched`, `history_recoverable`, `online_retired`, +`backup_retired`, `server_key_retired` and a separate `completion_status`. +Online retirement requires the new primary keys, retired old keys, revoked old +signing credential and empty destroyed vault fields. No backup-destruction +attestation exists yet, so `backup_retired` and `server_key_retired` remain false. +The legacy `status=completed` from an empty-inventory identity switch is not a +claim that every server-held backup has been destroyed; use the independent +retirement fields. + +Coverage is explicitly `enumerated_keystore_age_revisions`; external ciphertext +coverage is unknown. A valid ACK is a client's signed declaration, not proof +that every unknown ciphertext or recipient is recoverable. + +### #69 — transactional, persistent restore quarantine + +The v4 dump is converted to a protected SQL file, then imported with the mandatory +`recovery_quarantine` setting in one `psql --single-transaction` commit. A failure +before that commit rolls back both. The SQL file is not retained. Existing v4 +archives remain supported; no root private material is added to data backups. +`psql` is now required alongside the existing PostgreSQL backup tools. + +Deleting the config marker, copying the configuration or restarting does not +clear this database gate. Unknown, malformed and even null/false gate values +remain quarantined. Startup does not synchronize release-owned resources while +quarantined. Authentication/authorization, SSH principals, share-link authority, +external job claims and scheduled maintenance fail closed. Process-local gating +also covers old marker-only recovery drills. + +Only the HTTP health response remains available; it returns 503 with +`status=recovery_quarantined`, `ready=false`, and disabled writes/outbound. +Business reads are blocked too: a resource that was public in an old snapshot +may have become private since. Restore returns `revocation_replay=required` and +`promotion=blocked`. No network operation clears this state. + +This is containment, **not** an implementation or proof of monotonic revocation +replay. Do not remove the gate to promote an unverified restore. + +### #70 — root rotation storage and restart integrity + +Rotation now writes the new encrypted root key to the same protected directory +used by `root_envelope` and the pending journal, rather than creating an unrelated +`config/root/key.json`. New independent layouts remain independent; existing +legacy layouts retain their explicit lookup behavior. + +A v2 journal signs commitments to the old certificate, new trust document, +encrypted envelope, exact online CSR and prior envelope digest. Resume verifies +those commitments, self-signature, CSR possession proof, service/subject/key +bindings and the already committed database state. Old key history remains +0600 under 0700 directories, is not overwritten on retry, and is never replaced +by an already installed new key. The audit stores public commitments, not the +encrypted root envelope. A pending v1 journal is rejected with +`rotation_journal_upgrade_required`; it is not silently treated as a v2 approval. +Such a journal needs a separately reviewed local recovery based on its actual +commit/file stage, not merely a version-number edit. + +Root operations remain restricted to the existing local-console boundary. +Old chains remain invalidated on rotation; no automatic dual-trust window or +silent expansion of existing signed grants is introduced. + +## Evidence and outstanding acceptance + +Local evidence: 34 cryptographic/state/SQL-command unit cases passed on Python +3.13.5; compilation and `git diff --check` passed. This does not substitute for +the project's Python 3.15/PostgreSQL/age CI. Added integration regressions cover +real age decryption before ACK, completed identity versus pending backup +retirement, deleted-marker restore/restart, no outbound jobs or maintenance, +atomic restore failure, and root rotation interruption before key/trust writes. +CI results must be taken from the actual run, not this document. + +#68 remains open for complete migration finalization, frozen-scope deltas/new-write +key enforcement, compatible old-encryption-key envelopes, irrecoverable-item +choices, and independently auditable backup retirement. #69 remains open for a +durable independently trusted anti-rollback checkpoint, monotonic replay of all +revocation/retirement/authorization facts, controlled promotion and a production +recovery drill. #70 remains open for real OS-console acceptance, operator-approved +existing-CA governance/migration, and a reviewed trust-transition/rollback plan. diff --git a/docs/MARKET_CONTRACTS.md b/docs/MARKET_CONTRACTS.md new file mode 100644 index 00000000..b4588d49 --- /dev/null +++ b/docs/MARKET_CONTRACTS.md @@ -0,0 +1,219 @@ +# Market contracts: orders, delivery and disputes + +This is the implementation contract for #73–75. It does not enable production +fixtures, mint funds, install arbitrators or configure an outbound mail server. +Amounts are integer MSG minor units; one MSG is 1,000,000 minor units. + +## Choosing a checkout contract + +`orders.buy@1` retains its original signed meaning: fund an order, then explicitly +prepare and accept a supported managed package. Previously signed requests are +not silently reinterpreted as permission for automatic settlement. +The published `orders.buy@2` keeps its digest-bound managed checkout and +optional `auto_accept` under `escrow-instant-v1`; `delivery.claim@1` remains +its separate signed acknowledgement. Its schema, receipts and decision journal +are preserved rather than reinterpreted by the new arbitration flow. + +New clients select `orders.buy@3`, or use `orders.create@1` followed by +`orders.fund@1`. The latter signs the returned `order_digest`, exact total and +currency. Read `orders.contract` or `orders.get` as the buyer or seller. The +existing CLI's schema-driven invocation supports these operations; obtain each +schema from discovery rather than constructing URLs containing credentials. +All private reads use current authentication; an order ID is never a capability. + +At creation, the immutable contract locks buyer, seller, quantity, listing +revision/content/terms, package revision/digest, policy/digest/grant epochs, +buyer handle and, for sealed delivery, the buyer's primary encryption subkey. +The private checkout principal is not included in seller projections. Listing +edits do not rewrite a reserved order. Funding still revalidates the current +site recipient, payment signature, balance and expiry. + +Unfunded reservations count against quantity until cancelled/expired. The +serialized PostgreSQL write transaction covers quantity, balance, ledger, +transitions, receipts and the executor's idempotent result. The funding timer +starts at persisted creation; the delivery timer starts at persisted funding. +The bounded worker resolves overdue orders; GET never advances a timer. + +## State and accounting + +The transition table is in `msg.market.escrow.TRANSITIONS`: + +``` +created -> funded | cancelled +funded -> delivered | refunded | disputed +delivered -> accepted | refunded | disputed +accepted -> settled +disputed -> settled | refunded +``` + +Each order owns an `order_escrow` LedgerAccount, with no Subject, key or +credentials. Only the EscrowEngine's internal settlement authority can debit it. +One `order_settlements` row per order and a unique consumed decision prevent +repeated settlement, including retry with another request ID. A split appends +both ledger legs atomically. Snapshot, transition, vote, policy and settlement +facts reject SQL updates/deletes. Historical financial receipts are not rewritten. + +A funded, undelivered order can be cancelled by its buyer. Missing/corrupt pinned +packages or delivery bytes are checked against server storage, not accepted on a +complainant's word. Overdue, undelivered orders refund. Unexpected programming or +infrastructure errors roll back; they are not converted into discretionary refunds. + +## Delivery and claiming are different facts + +For `managed_instant` file/bundle/text listings, funding verifies the complete +pinned manifest and payload digests, creates the buyer-only delivery and settles +the funds in the same transaction. Delivery remains **prepared**, not claimed. +The buyer explicitly signs `delivery.accept@2` with the delivery digest to record +claiming. Reading a page, an SMTP response and opening a Transfer do not claim it. +Automatic managed settlement is final for this escrow; it is not a promise to +refund money that has already left the escrow account. + +For `sealed_manual` and `service`, funding holds the balance. The seller uploads +an immutable File (ordinary upload or Transfer), then signs `delivery.submit` +with its revision and digest. Sealed delivery additionally binds the checkout +buyer's still-active key ID and requires an age ciphertext envelope. The server +cannot prove that opaque age bytes decrypt correctly: the buyer must decrypt, +check the content and explicitly accept, or open a dispute. Services use an +immutable evidence file and the same explicit acceptance boundary. No plaintext +secret, attachment or goods content is mailed. + +`delivery.get@2` has a 64 KiB total inline budget. Larger payloads advertise +`delivery.transfer_open` and bounded `delivery.part_get`. The signed Transfer +entry creates a private buyer-owned File referencing the same BlobRef, then +uses the existing TransferService. Continuations recheck the buyer's current +credential and file ACL; this does not duplicate the payload or expose the +seller's other files. Package, delivery and private evidence references are +SQL-authoritative roots for both backup and garbage collection. + +## Optional checkout email + +No email is required for purchase or site delivery. An existing verified email +must belong to the same buyer and match its endpoint generation. An unverified +checkout address receives only a verification challenge, never order ID, handle, +goods, pickup URL or an authentication capability. The original buyer must also +sign `orders.email_verify` with that proof. This creates an order-only endpoint; +it does not replace their account's email settings. + +After verification, `orders.email_bind` can bind a current verified account +endpoint. `orders.email_disable` disables the selected notification channel and +revokes an order-only endpoint. The worker rechecks buyer/actor, delivery owner, +endpoint owner/generation/address, revocation and (for sealed goods) encryption +key ownership before sending. A stale job cannot overwrite a newly chosen or +disabled endpoint. Seller projections never contain the address. + +The order notification contains exactly `order_id`, checkout `handle` and an +authenticated site `pickup` URL. Status distinguishes disabled, pending, queued, +SMTP acceptance and uncertain delivery. SMTP failure, disabled configuration or +an uncertain response does not undo site delivery or financial settlement. + +## Versioned objective resolution and arbitration + +`orders.dispute_open` first checks objective faults. Otherwise it creates one +private Case per order. Statements and immutable file evidence are submitted +through `orders.dispute_statement` / `orders.dispute_evidence` with `panel` or +`parties` visibility. `orders.dispute_get` and `orders.dispute_evidence_get` enforce +those roles; unauthorized and missing objects return the same error. Public +`orders.dispute_summary` exposes only random case ID, state, policy digest and +the executed outcome, not participants, order ID, votes, testimony or payloads. + +The builtin `dispute-v1` has no arbitrators. It safely holds subjective cases +rather than choosing an administrator or AI. Before offering arbitrated goods, +the operator uses a physical console (preview, typed digest approval and Root +PIN) to grant registered arbitrators and publish an immutable policy: + +``` +msgd --config-dir /etc/msg.lmm.best market grant @arbitrator +msgd --config-dir /etc/msg.lmm.best market publish /secure/policy.json +msgd --config-dir /etc/msg.lmm.best market revoke @arbitrator +``` + +A policy has exactly the keys in `msg.market.policy.DEFAULT_POLICY`; copy that +shape, use a new ID, actual candidate subject IDs and explicit durations. There +are no evaluation expressions, scripts, model calls or hidden fallback members. +A grant/regrant has a new epoch; orders snapshot the active epochs at checkout. +The policy's candidate set and SHA256(order, policy, round, subject) ordering +select the panel. Buyer, seller, declared conflicts and currently invalid roles +are excluded. `orders.arbitrator_conflict` is a signed self-declaration. It does +not claim knowledge of undisclosed real-world relationships. + +Panel size is bounded and quorum is a strict majority. A member signs the exact +proposal with purpose `arbitration-decision`, then submits it via +`orders.dispute_vote`. Each member has one immutable vote per round. Votes are +facts, never ledger writes. A matching quorum forms a release/refund/split +Decision; `orders.dispute_execute` or the due worker revalidates signatures, +credential ceilings, grant epochs, conflicts, round, policy and deadlines before +the EscrowEngine consumes it. Conflicting, expired, revoked or stale decisions +cannot pay again. + +The explicit member-invalidation policy is **hold-no-replacement-v1**: a panel +that loses authority is held, not silently reselected. A policy may permit one +appeal with a disjoint panel selected from the same original candidate snapshot; +there is no second appeal. Old-round decisions become unusable. If no valid +quorum or eligible appeal panel exists, funds remain held and doctor reports the +held-case count. Arbitrators cannot override this by directly moving money. + +## Operations and verification + +Doctor's `market` check is read-only: account separation, escrow balances, +contract/policy digests, settlement conservation and consumed decisions. The +isolated `market_lifecycle` selftest creates disposable identities and a Test +Root, registers/funds a bank with 20 MSG, awards the buyer 10 MSG for a real PoP, +then buys the fixed 5 MSG bundle (`msg.lmm.best store selftest`, `hello.txt` = +`delivery-ok\n`). It checks prepared-not-claimed delivery, an in-memory SMTP sink, +verification-only email, one real signed panel refund, final bank 15/buyer 5 and +zero escrow. It never sends to a real address and cleans its namespace. + +Regression files: `test_market_lifecycle.py`, `test_market_delivery.py`, +`test_market_arbitration.py`, `test_market_recovery.py`, plus the existing order, +ledger, worker, backup and transport suites. Recovery tests use a real backup and +restore; a recovery drill still blocks outbound effects and timer execution. +The migration classifies all new authoritative identity references while retaining +the fail-closed guard for unknown tables. It does not rewrite old receipts. + +Existing credentials' finite ceilings do not expand merely because operations +were installed. Follow the existing local authority review/reissuance procedure +before allowing new operations. Test success is not production deployment, +real-SMTP acceptance, a production schema rehearsal or permission to mint money. + +## Signed arbitration reasons + +Before voting, a current panel member writes a nonempty UTF-8 `text/plain` or +`text/markdown` File (at most 64 KiB), then signs `orders.dispute_rationale@1` +with `{case_id, ref: {id, revision}}`. This pins exactly that immutable revision +as case evidence visible to both parties and the current authorized panel. It +does not share the author's file tree or disclose the reason in public summaries. + +The returned `rationale_ref` and `rationale_digest` are required fields in the +canonical proposal signed with purpose `arbitration-decision`, alongside the +case/order/round/policy, allocation and expiry. Every quorum member must sign the +same reason and allocation. A later draft, another case/round, a different digest +or signature substitution cannot replace the committed reason. Both voting and +escrow execution verify the pinned bytes. Missing/corrupt reasons stop payment; +the due worker holds the case rather than inventing a replacement ruling. + +Reason blobs use the existing case-evidence retention/backup roots. Restoring a +case retains its exact reason and visibility without executing it again. The +market doctor checks bindings without writing or retroactively applying current +role grants to an already-executed historical decision. This completes the new +arbitration contract in this unreleased PR; legacy `orders.buy@1` is unchanged. + +## Integration with published managed checkout + +The full lifecycle uses `orders.buy@3`; `orders.buy@2` keeps the already-published +package-digest/optional-auto-accept contract and `escrow-instant-v1` policy. Their +immutable decision journals are retained. Both execution paths share the same +internal OrderEscrow account check; neither path can settle the other's order. +Existing `delivery.claim@1` remains an acknowledgement, not another payment. + +Version-3 contract reads and execution recheck every immutable checkout column +against its pinned contract. Funding references must identify the corresponding +buyer-to-escrow ledger entry. Settlement facts bind funding, receipt references, +settlement time and prepared-delivery identity/content, while allowing subsequent +buyer ACKs and independent email status updates. A mixed restored projection +fails closed without another ledger entry, receipt, Event or outbound job. + +Mail configuration must be present **and enabled** at checkout and immediately +before invoking the sender. Turning SMTP off after enqueueing prevents the send +and marks the selected notification disabled without undoing delivery/payment. +The existing completion fences are retained for tools, Git, webhooks and mail; +expired market-mail leases also converge the selected order's notification state. diff --git a/docs/system/rules/_index.md b/docs/system/rules/_index.md index da724ec1..522af93e 100644 --- a/docs/system/rules/_index.md +++ b/docs/system/rules/_index.md @@ -1,4 +1,4 @@ - + # Platform rules This is the authoritative index. Read only the rules relevant to your task. Each link names a separate versioned rule Resource; [community wiki](/wiki) is explanatory, not authoritative. @@ -13,5 +13,5 @@ This is the authoritative index. Read only the rules relevant to your task. Each | msg.topics | Topic membership and moderation | topics | content.topic_*, discussion.* | 1 | [topics](/_rules/topics) | | msg.files | Transfers and encrypted personal data | files | transfer.*, keystore.* | 1 | [files](/_rules/files) | | msg.recovery | Custodians and recovery boundaries | recovery | identity.recover | 1 | [recovery](/_rules/recovery) | -| msg.security | Secrets and trusted execution | security | system.*, tool.* | 1 | [security](/_rules/security) | +| msg.security | Secrets and trusted execution | security | system.*, tool.* | 2 | [security](/_rules/security) | | msg.protocol | Registry and network entrypoints | protocol | all operations | 1 | [protocol](/_rules/protocol) | diff --git a/docs/system/rules/security.md b/docs/system/rules/security.md index 0b713f35..aece52d1 100644 --- a/docs/system/rules/security.md +++ b/docs/system/rules/security.md @@ -1,6 +1,8 @@ - + # Security boundaries Root signing, PINs, and recovery secrets stay in local root administration, outside the ordinary Resource tree and network service. Do not expose complete tokens or private keys in posts, logs, cursors, indexes, or tool output. Source-managed platform rules are release artifacts; ordinary users, Topics, plugins, and wiki edits cannot change them. Only installed trusted code may implement a handler. +Order emails are notifications, not ownership or acceptance. Pickup links must contain no credentials and still require buyer authentication. SMTP status comes from the delivery job, not a second order state machine. Disabled endpoints, failed jobs and uncertain attempts are not revived by reads; buyer acceptance is a separate signed operation. + See [authorization](/_rules/auth) before granting capabilities. diff --git a/scripts/ci_shards.py b/scripts/ci_shards.py index d6fd86a5..8b45b623 100644 --- a/scripts/ci_shards.py +++ b/scripts/ci_shards.py @@ -1,16 +1,16 @@ -"""Run disjoint pytest partitions and verify their complete evidence at the CI gate. +"""Run complete, disjoint pytest partitions and verify each executed node ID. -Every runner collects the entire suite. Partitioning is by stable node ID, not a -hand-maintained filename list, so new/parameterized/nested tests cannot disappear. +Adapted from PR #104. Every runner collects the complete suite; the gate rejects +skips, duplicate IDs, count-only substitutions and missing or failed evidence. """ from __future__ import annotations import argparse import hashlib import json -from pathlib import Path import sys import xml.etree.ElementTree as ET +from pathlib import Path def partition(nodeids, index, count): @@ -28,7 +28,24 @@ def evidence(nodeids, index, count): 'selected': partition(all_nodes, index, count)} +def report_nodes(path): + root = ET.parse(path).getroot() + if any(root.findall('.//' + tag) for tag in ('error', 'failure', 'skipped')): + raise ValueError('JUnit result contains failed or skipped tests') + nodes = [] + for case in root.findall('.//testcase'): + ids = [p.get('value') for p in case.findall('./properties/property') + if p.get('name') == 'msg.nodeid'] + if len(ids) != 1 or not ids[0]: + raise ValueError('JUnit result is missing an unambiguous test node ID') + nodes.append(ids[0]) + if len(nodes) != len(set(nodes)): + raise ValueError('JUnit result contains duplicate test node IDs') + return set(nodes) + + def verify(directory, count): + partition([], 0, count) directory = Path(directory) expected = None seen = set() @@ -44,18 +61,8 @@ def verify(directory, count): if seen & selected: raise ValueError('test executed in multiple shards') seen.update(selected) - root = ET.parse(directory / f'tests-{index}.xml').getroot() - cases = root.findall('.//testcase') - if len(cases) != len(selected) or root.findall('.//error') or root.findall('.//failure'): - raise ValueError('JUnit result is incomplete or failed') - observed = [] - for case in cases: - identities = case.findall('./properties/property[@name="msg.nodeid"]') - if len(identities) != 1 or identities[0].get('value') is None: - raise ValueError('JUnit test identity is missing or ambiguous') - observed.append(identities[0].get('value')) - if sorted(observed) != sorted(selected): - raise ValueError('JUnit test identities do not match the selected shard') + if report_nodes(directory / f'tests-{index}.xml') != selected: + raise ValueError('JUnit result does not match selected test node IDs') if not expected or seen != expected: raise ValueError('test coverage is incomplete') return len(seen) @@ -64,7 +71,7 @@ def verify(directory, count): def run(index, count, directory): import pytest - partition([], index, count) # Validate before starting tests or writing files. + partition([], index, count) directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -75,8 +82,6 @@ def pytest_collection_modifyitems(self, config, items): rejected = [item for item in items if item.nodeid not in chosen] items[:] = [item for item in items if item.nodeid in chosen] for item in items: - # Preserve the exact pytest identity, including class/parameter - # punctuation, instead of reverse-engineering JUnit display names. item.user_properties.append(('msg.nodeid', item.nodeid)) config.hook.pytest_deselected(items=rejected) (directory / f'shard-{index}.json').write_text( diff --git a/src/msg/admin/backups.py b/src/msg/admin/backups.py index 36ebf610..2b5612a7 100644 --- a/src/msg/admin/backups.py +++ b/src/msg/admin/backups.py @@ -14,6 +14,8 @@ from msg.core.codec import canonical,loads from msg.core.errors import Failure,require from msg.config import load_settings,write_example +from msg.admin.restore_database import restore_dump +from msg.storage.git import durable_write _FORMAT = 'msg-data-backup-v4' _KEYS = ('online.key', 'receipt.key', 'tokens.key') @@ -43,6 +45,10 @@ def _db_refs(connection): content=loads(raw)['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') contents[content['digest'][7:]]=content['size'] + from msg.market.references import content_references + for content in content_references(connection.execute): + require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') + contents[content['digest'][7:]]=content['size'] live={row[0] for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL')} require(live.issubset(set(revisions)), 'backup_revision_missing') return {'contents':contents, 'revisions':revisions} @@ -291,9 +297,11 @@ def restore(source,config_dir,data_dir,*,postgres_dsn='service=msgd'): require(connection.execute( "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() LIMIT 1" ).fetchone() is None,'restore_database_not_empty') - subprocess.run(['pg_restore','--single-transaction','--exit-on-error','--no-owner','--no-acl', - '--dbname',safe_dsn,str(directory/'metadata.dump')], - env=env,check=True,capture_output=True) + quarantine={'format':'msg-recovery-quarantine-v1','outbound_enabled':False, + 'source_backup_sha256':_hash(Path(source)), + 'revocation_replay':'required','authority':'health_only'} + restore_dump(directory/'metadata.dump',safe_dsn=safe_dsn,env=env, + quarantine=quarantine) with psycopg.connect(postgres_dsn) as connection: require(_db_refs(connection)==manifest['references'], 'backup_database_reference_mismatch') @@ -309,10 +317,8 @@ def restore(source,config_dir,data_dir,*,postgres_dsn='service=msgd'): settings=write_example(config_dir,data_dir,manifest['service_url'],postgres_dsn=postgres_dsn) # Install the fail-closed marker as soon as a runnable config exists, # including if a later filesystem verification fails. - (config_dir/'recovery-drill.json').write_bytes(canonical({ - 'format':'msg-recovery-drill-v1','outbound_enabled':False, - 'source_backup_sha256':_hash(Path(source))})) - os.chmod(config_dir/'recovery-drill.json',0o600) + durable_write(config_dir/'recovery-drill.json',canonical({ + **quarantine,'format':'msg-recovery-drill-v1'}),mode=0o600) _write_restored_config(directory/'source-config.toml',settings,postgres_dsn) settings=load_settings(config_dir) for name,target in (('content',settings.server.content_dir), @@ -331,4 +337,5 @@ def restore(source,config_dir,data_dir,*,postgres_dsn='service=msgd'): _relink_lfs(data_dir,settings) _verify_storage(data_dir,manifest['references'],settings=settings) return {'status':'restored','root_admin_material':'restore_separate_encrypted_root_backup_locally', - 'mail':'disabled_until_configured','outbound':'disabled_recovery_drill'} + 'mail':'disabled_until_configured','outbound':'disabled_recovery_drill', + 'revocation_replay':'required','promotion':'blocked'} diff --git a/src/msg/admin/diagnostics.py b/src/msg/admin/diagnostics.py index 6ee58448..4e2c3f29 100644 --- a/src/msg/admin/diagnostics.py +++ b/src/msg/admin/diagnostics.py @@ -173,6 +173,14 @@ async def inspect(tx): from msg.admin.custodial_check import inspect_custodial_history success('custodial_history', **await inspect_custodial_history(tx)) except Failure as exc:failed('custodial_history',exc.code) + if 'orders' in settings.server.plugins: + try: + from msg.admin.market_check import inspect_market + success('market',**(await inspect_market(tx))) + except (Failure,KeyError,ValueError,psycopg.Error) as exc: + failed('market',getattr(exc,'code','market_inspection_failed')) + else: + checks['market']={'ok':True,'status':'disabled'} try: from msg.core.requests import SECRET_DELIVERY_MIN_VERSION columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " @@ -606,6 +614,8 @@ async def approve(key,uid,grants): checks['credential_delivery_recovery']=await check_token_delivery(app,now) from msg.admin.custodial_check import check_custodial_history checks['custodial_history_recovery']=await check_custodial_history(app,now) + from msg.admin.market_check import check_market + checks['market_lifecycle']=await check_market(app,root,call,register,now) except Failure as exc: checks['failure']={'code':exc.code} finally: diff --git a/src/msg/admin/market.py b/src/msg/admin/market.py new file mode 100644 index 00000000..1bc971f0 --- /dev/null +++ b/src/msg/admin/market.py @@ -0,0 +1,91 @@ +"""Root-signed, physical-console arbitration configuration; never a network API.""" +from __future__ import annotations + +import asyncio +import getpass + +from msg.admin.money import _root_signer +from msg.admin.root import RootAdmin, require_local_console, root_envelope +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, loads, wire +from msg.core.errors import require +from msg.core.models import AuditEvent, Event, ResourceRef +from msg.market.policy import DEFAULT_POLICY, validate +from msg.plugins.common import new_id +from msg.security.crypto import Ed25519Signer, open_private_key + + +async def apply_market(app, signer, *, action, operator, subject=None, policy=None, expected=None): + _root_signer(app, signer) + require(action in {'grant','revoke','publish'}, 'market_admin_action_invalid') + if action == 'publish': + policy = validate(policy) + require(policy['id'] != DEFAULT_POLICY['id'], 'builtin_policy_immutable') + async with app.metadata.transaction(write=True) as tx: + if action == 'publish': + require(tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, + 'arbitration_policy_exists') + for candidate in policy['candidates']: + identity = await tx.subject(candidate) + require(identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator') + tx.execute('INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', + (policy['id'],canonical(policy).decode(),digest(policy)), write=True) + before, after = None, policy + else: + identity = await tx.subject(subject) + require(identity.kind == 'registered' and not identity.local_only and subject != ROOT_SUBJECT, + 'invalid_arbitrator') + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (subject,)) + before = {'epoch':row[0],'active':row[1]} if row else None + if expected is not None: + require(before == expected, 'market_preview_stale') + require((action=='grant' and (not row or not row[1])) or + (action=='revoke' and row and row[1]), 'arbitrator_role_unchanged') + after = {'epoch': new_id('role') if action=='grant' else row[0], 'active': action=='grant'} + tx.execute('''INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) + ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active''', + (subject,after['epoch'],after['active']), write=True) + statement = {'action':action,'subject':subject,'before':before,'after':after, + 'operator':operator,'at':wire(app.clock())} + proof = signer.sign(canonical(statement), purpose='market-admin') + event = Event(id=new_id('audit'), type='root.market.'+action, time=app.clock(), + request_id=new_id('local-market'), actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), data={**statement,'root_signature':wire(proof)}) + await tx.append_event(event) + await tx.append_audit(AuditEvent(event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(before),after_digest=digest(after),previous_digest=None, + entry_digest='',result=action)) + return {'action':action,'audit_event_id':event.id,'result':after} + + +class MarketAdmin: + def __init__(self, config_dir): + self.config_dir = config_dir + + def execute(self, action, *, subject=None, policy_file=None): + operator = require_local_console(self.config_dir) + app = RootAdmin(self.config_dir)._app() + async def run(): + await app.load() + try: + policy = loads(policy_file.read_bytes()) if policy_file is not None else None + resolved, before = subject, None + async with app.metadata.transaction(write=False) as tx: + if subject and subject.startswith('@'): + resolved = await tx.resolve('/'+subject) + if resolved: + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,)) + before = {'epoch':row[0],'active':row[1]} if row else None + preview = {'action':action,'subject':resolved,'before':before,'policy':policy} + print(canonical(preview).decode()) + approval = 'CONFIRM MARKET '+digest(preview) + require(input('Type '+approval+' to continue: ') == approval, 'approval_cancelled') + pin = getpass.getpass('Root PIN/passphrase: ') + signer = Ed25519Signer.from_bytes(open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), pin)) + return await apply_market(app, signer, action=action, operator=operator, + subject=resolved,policy=policy,expected=before) + finally: + await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/market_check.py b/src/msg/admin/market_check.py new file mode 100644 index 00000000..06c00c67 --- /dev/null +++ b/src/msg/admin/market_check.py @@ -0,0 +1,163 @@ +"""Read-only financial invariants and a deliberately isolated Test Root scenario.""" +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace + +from msg.core.codec import b64, canonical, digest, loads, wire +from msg.core.errors import require +from msg.core.models import MailConfig +from msg.market.policy import contract, validate +from msg.plugins.money import _balance + + +async def inspect_market(tx): + """No initialization, repair, timer, statement, event, job or network call.""" + count = 0 + for oid,buyer,total,escrow,state in tx.rows( + 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id'): + expected = total if state in {'funded','delivered','accepted','disputed'} else 0 + require(_balance(tx,escrow)==expected, 'market_escrow_invariant') + require(tx.one('SELECT 1 FROM identities WHERE id=?',(escrow,)) is None and + tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?',(escrow,))==('order_escrow',None), + 'market_escrow_identity') + snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?',(oid,)) + if snapshot: + locked = contract(tx,oid) + policy = locked['policy'] + validate(policy['policy']) + require(digest(policy['policy'])==policy['policy_digest'] and + locked['buyer']==buyer and locked['total_price_minor']==total, 'market_snapshot_invariant') + row = tx.one('SELECT body FROM order_settlements WHERE order_id=?',(oid,)) + if row: + fact = loads(row[0]) + require(state in {'settled','refunded'} and fact['refund_minor']+fact['release_minor']==total, + 'market_settlement_invariant') + actual = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?",(escrow,))[0] + require(actual==total,'market_settlement_invariant') + elif snapshot: + require(state not in {'settled','refunded'},'market_settlement_missing') + count += 1 + for raw,expected in tx.rows('SELECT body,digest FROM arbitration_policies'): + policy = loads(raw); validate(policy) + require(digest(policy)==expected,'arbitration_policy_corrupt') + for case_id,order_id,state in tx.rows('SELECT id,order_id,state FROM arbitration_cases'): + settlement = tx.one('SELECT decision_id FROM order_settlements WHERE order_id=?',(order_id,)) + if state=='executed': + require(settlement is not None and settlement[0] is not None,'market_decision_unfunded') + from msg.market.arbitration import _read + from msg.market.rationale import record + row=tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=?', + (settlement[0],case_id)) + require(row is not None,'market_decision_missing') + decision=loads(row[0]) + require(all(decision.get(k)==decision['proposal'].get(k) + for k in ('rationale_ref','rationale_digest')),'decision_rationale_mismatch') + record(tx,_read(tx,case_id),decision['proposal']) + return {'orders':count,'held_cases':tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], + 'escrow_invariants':True,'private_reads_only':True} + + +async def check_market(app,root,call,register,now): + """20 MSG issue -> bank-funded 10 reward -> 5 bundle, entirely disposable.""" + from msg.admin.market import apply_market + from msg.admin.money import apply_money + from msg.market.policy import DEFAULT_POLICY + from msg.workers.effects import EffectWorker + require(app.selftest_run_id is not None,'selftest_namespace_required') + bank,bank_id = await register('market-test-bank') + buyer,buyer_id = await register('market-test-buyer') + async def invoke(name,args,key=buyer,subject=buyer_id,**kw): + result = await call(name,args,key,subject,**kw) + require(result.status=='ok',result.error.code if result.error else 'market_selftest_failed') + return result + await apply_money(app,root,action='bank_add',operator='isolated-selftest',subject_id=bank_id) + await apply_money(app,root,action='mint',operator='isolated-selftest',amount_minor=20_000_000) + await apply_money(app,root,action='transfer',operator='isolated-selftest',subject_id=bank_id,amount_minor=20_000_000) + bounty = await invoke('bounty.create',{'name':'identity-test','terms':'One-use signature test', + 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1},bank,bank_id) + challenge = await invoke('bounty.challenge',{'listing_id':bounty.data['bounty']['listing_id']}) + payload = challenge.data['challenge'] + await invoke('bounty.claim',{'challenge_id':payload['challenge_id'], + 'proof':wire(buyer.sign(canonical(payload),purpose='bounty-pop-v1'))}) + listing = await invoke('store.listing_create',{'name':'store-selftest','item_kind':'bundle', + 'price_minor':5_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'managed_instant', + 'escrow_policy':'escrow-v1','dispute_policy':'dispute-v1','terms':'Immutable test bundle.'},bank,bank_id) + async with app.metadata.transaction(write=False) as tx: + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(bank_id,))[0] + file = await invoke('content.file_put',{'parent':parent,'name':'hello.txt','media_type':'text/plain', + 'data':b64(b'delivery-ok\n')},bank,bank_id) + package = await invoke('store.package_deposit',{'listing_id':listing.resources[0].id, + 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, + 'payload_refs':[wire(file.resources[0])]},bank,bank_id) + active = await invoke('store.listing_update',{'id':listing.resources[0].id,'state':'active', + 'package_ref':package.data['package']['id']},bank,bank_id, + expected=((listing.resources[0].id,listing.data['generation']),)) + old_settings = app.settings + class NoNetworkMail: + def __init__(self): self.messages=[] + async def send(self,job): + self.messages.append(dict(job.arguments)); return 'sent' + sender=NoNetworkMail() + app.settings=replace(app.settings,server=replace(app.settings.server,mail=MailConfig(enabled=True, + host='selftest.invalid',port=2525,tls='starttls',sender='test@selftest.invalid',credential_file=None))) + try: + sale = active.data['listing'] + bought = await invoke('orders.buy',{'listing_id':sale['listing_id'], + 'listing_revision':sale['listing_revision'],'quantity':1,'total_price_minor':5_000_000, + 'currency_id':'primary','email':'buyer@selftest.invalid'},contract_version=3) + oid = bought.data['order']['id'] + worker=EffectWorker(app,mail_sender=sender) + while await worker.run_once(): pass + require(len(sender.messages)==1 and oid not in sender.messages[0]['text'],'market_email_leak') + proof=loads(sender.messages[0]['text']) + await invoke('orders.email_verify',{'order_id':oid,'challenge_id':proof['challenge_id'],'token':proof['token']}) + while await worker.run_once(): pass + require(len(sender.messages)==2 and set(loads(sender.messages[1]['text']))=={'order_id','handle','pickup'}, + 'market_email_leak') + delivery=await invoke('delivery.get',{'order_id':oid},contract_version=2) + require(delivery.data['delivery']['state']=='prepared' and + delivery.data['delivery']['payloads'][0]['data']==b64(b'delivery-ok\n'),'market_claim_invariant') + finally: + app.settings=old_settings + # A separate 1 MSG service dispute refunds the test buyer through a real + # fixed 2-of-3 panel. It leaves the official 5/15 final balances unchanged. + members={} + for i in range(3): + key,uid=await register('market-test-arbitrator-'+str(i)); members[uid]=key + await apply_market(app,root,action='grant',operator='isolated-selftest',subject=uid) + policy=deepcopy(DEFAULT_POLICY); policy.update(id='selftest-panel-v1',candidates=list(members)) + await apply_market(app,root,action='publish',operator='isolated-selftest',policy=policy) + service=await invoke('store.listing_create',{'name':'case-selftest','item_kind':'service', + 'price_minor':1_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'service', + 'escrow_policy':'escrow-v1','dispute_policy':policy['id'],'terms':'Test service dispute.'},bank,bank_id) + service=await invoke('store.listing_update',{'id':service.resources[0].id,'state':'active'},bank,bank_id, + expected=((service.resources[0].id,service.data['generation']),)) + sale=service.data['listing'] + bought=await invoke('orders.buy',{'listing_id':sale['listing_id'],'listing_revision':sale['listing_revision'], + 'quantity':1,'total_price_minor':1_000_000,'currency_id':'primary'},contract_version=3) + opened=await invoke('orders.dispute_open',{'order_id':bought.data['order']['id'],'reason':'quality'}) + case=await invoke('orders.dispute_get',{'case_id':opened.data['case_id']}) + author=opened.data['panel'][0] + async with app.metadata.transaction(write=False) as tx: + parent=tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(author,))[0] + reason=await invoke('content.file_put',{'parent':parent,'name':'case-reason.txt', + 'media_type':'text/plain','data':b64(b'Selftest evidence warrants a full refund.')},members[author],author) + bound=await invoke('orders.dispute_rationale',{'case_id':opened.data['case_id'], + 'ref':wire(reason.resources[0])},members[author],author) + proposal={**case.data['proposal_base'],'outcome':'refund','refund_minor':1_000_000, + **{name:bound.data[name] for name in ('rationale_ref','rationale_digest')}} + for uid in opened.data['panel'][:2]: + voted=await invoke('orders.dispute_vote',{'proposal':proposal, + 'signature':wire(members[uid].sign(canonical(proposal),purpose='arbitration-decision'))},members[uid],uid) + args={'case_id':opened.data['case_id'],'decision_id':voted.data['decision']['id']} + await invoke('orders.dispute_execute',args) + await invoke('orders.dispute_execute',args) + async with app.metadata.transaction(write=False) as tx: + await inspect_market(tx) + require(_balance(tx,buyer_id)==5_000_000 and _balance(tx,bank_id)==15_000_000 and + tx.one("SELECT SUM(amount_minor) FROM money_ledger WHERE kind='mint'")[0]==20_000_000, + 'market_conservation_failed') + for (escrow,) in tx.rows("SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')"): + require(_balance(tx,escrow)==0,'market_escrow_invariant') + return True diff --git a/src/msg/admin/restore_database.py b/src/msg/admin/restore_database.py new file mode 100644 index 00000000..7b7886c8 --- /dev/null +++ b/src/msg/admin/restore_database.py @@ -0,0 +1,37 @@ +"""Import an offline dump and its quarantine gate in the same PostgreSQL commit.""" +from pathlib import Path +import os +import subprocess + +from msg.core.codec import canonical +from msg.security.quarantine import SETTING + + +def restore_dump(dump, *, safe_dsn, env, quarantine): + """Stream through a protected SQL file; COPY data requires psql, not execute(). + + pg_restore does not connect to PostgreSQL here. psql's single transaction + contains both the dump and the mandatory gate. An interruption before commit + leaves neither an imported snapshot nor a runnable ungated database. + """ + script = Path(dump).parent / 'quarantined-restore.sql' + script.touch(mode=0o600, exist_ok=False) + try: + subprocess.run(['pg_restore', '--exit-on-error', '--no-owner', '--no-acl', + '--file', str(script), str(dump)], + env=env, check=True, capture_output=True) + # Values are fixed recovery metadata, not secrets. Escape as a SQL + # literal (including backslashes) independently of session string flags. + value = canonical(quarantine).decode().replace('\\', '\\\\').replace("'", "''") + gate = ("\nSET search_path = public, pg_catalog;\n" + f"INSERT INTO settings(key,value) VALUES('{SETTING}',E'{value}') " + "ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value;\n") + with script.open('ab') as stream: + stream.write(gate.encode()) + stream.flush() + os.fsync(stream.fileno()) + subprocess.run(['psql', '--no-psqlrc', '--quiet', '--single-transaction', + '--set', 'ON_ERROR_STOP=1', '--dbname', safe_dsn, '--file', str(script)], + env=env, check=True, capture_output=True) + finally: + script.unlink(missing_ok=True) diff --git a/src/msg/admin/rotation.py b/src/msg/admin/rotation.py index 41de5159..804023bb 100644 --- a/src/msg/admin/rotation.py +++ b/src/msg/admin/rotation.py @@ -7,16 +7,17 @@ from __future__ import annotations from dataclasses import replace from datetime import timedelta -from pathlib import Path +import os from msg.constants import ROOT_SUBJECT, ONLINE_CA, ADMINS_GROUP, CSR_SPACE -from msg.core.codec import canonical, decode, digest, wire, b64, unb64, loads +from msg.core.codec import canonical, decode, digest, wire, b64 from msg.core.errors import require from msg.core.models import Certificate, CertificateRequest, Credential, Signature, IssuancePolicy, AuditEvent, Event, ResourceRef from msg.plugins.common import new_id from msg.plugins.identity import certificate_resource -from msg.security.crypto import seal_private_key, open_private_key, Ed25519Signer +from msg.security.crypto import seal_private_key, Ed25519Signer from msg.security.certificates import sign_certificate, csr_body +from msg.security.rotation_journal import authorization, validate as validate_journal from msg.security.capabilities import grant_for from msg.storage.git import durable_write from msg.bootstrap import seed_resource @@ -47,45 +48,73 @@ def prepare(app, new_signer, new_pin, *, old_signer, operator): online = app.online_signer csr = CertificateRequest(resource_id=new_id('csr'), applicant=ONLINE_CA, subject_id=ONLINE_CA, requested_issuer=ROOT_SUBJECT, public_key=online.public_key, kind='ca', - grants=(grant_for(app.registry.capability('cert.issue')),), + grants=(grant_for(app.registry.capability('cert.issue'),scope=app.default_scope()),), issuance=IssuancePolicy(issue_grants=app.base_grants(),max_cert_ttl_seconds=app.settings.base_certificate_ttl, max_child_ca_depth=0,max_delegation_depth=8), requested_ttl_seconds=31536000, target_service=app.settings.service_url,delegation_depth=0,authority_sources=(),request_digest='', possession_proof=Signature(key_id=online.key_id,algorithm='ed25519',value=b'')) csr = replace(csr, request_digest=digest(csr_body(csr)), possession_proof=online.sign(canonical(csr_body(csr)),purpose='csr')) - journal={'version':1,'old_certificate':wire(old),'new_trust':{'version':1,'public_key':b64(new_signer.public_key), - 'certificate':wire(new)},'new_envelope':seal_private_key(new_signer.private_bytes(),new_pin), - 'online_csr':wire(csr),'statement':statement, - 'new_signature':wire(new_signer.sign(canonical(statement),purpose='root-rotation')), - 'old_signature':wire(old_signer.sign(canonical(statement),purpose='root-rotation')) if old_signer else None} + from msg.admin.root import root_envelope + key_file=root_envelope(app.settings.config_dir) + require(not key_file.is_symlink(),'unsafe_root_private_path') + journal={'version':2,'old_certificate':wire(old),'new_trust':{'version':1, + 'public_key':b64(new_signer.public_key),'certificate':wire(new)}, + 'new_envelope':seal_private_key(new_signer.private_bytes(),new_pin), + 'previous_envelope_digest':digest(key_file.read_bytes()) if key_file.exists() else None, + 'online_csr':wire(csr),'statement':statement} + proof=canonical(authorization(journal)) + journal['new_signature']=wire(new_signer.sign(proof,purpose='root-rotation')) + journal['old_signature']=wire(old_signer.sign(proof,purpose='root-rotation')) if old_signer else None + protected=journal_path(app).parent + require(not protected.is_symlink(),'unsafe_root_private_path') + protected.mkdir(parents=True,exist_ok=True,mode=0o700) + os.chmod(protected,0o700) durable_write(journal_path(app),canonical(journal),mode=0o600) return journal async def complete(app, journal, *, pin): from msg.security.crypto import verify - private=open_private_key(journal['new_envelope'],pin) - signer=Ed25519Signer.from_bytes(private) - require(signer.public_key==unb64(journal['new_trust']['public_key']),'root_key_mismatch') - require(journal['version']==1,'unknown_rotation_journal') - verify(signer.public_key,canonical(journal['statement']),decode(Signature,journal['new_signature']),purpose='root-rotation') - old=decode(Certificate,journal['old_certificate']) - new=decode(Certificate,journal['new_trust']['certificate']) - require(new.subject_id==ROOT_SUBJECT and new.target_service==app.settings.service_url,'rotation_service_mismatch') - csr=decode(CertificateRequest,journal['online_csr']) + pending=journal_path(app) + protected=pending.parent + require(not protected.is_symlink() and not pending.is_symlink(), + 'unsafe_root_private_path') + online=Ed25519Signer.from_bytes((app.settings.service_keys/'online.key').read_bytes()) + signer,old,new,csr,proof=validate_journal(journal,pin=pin, + service=app.settings.service_url,online_public=online.public_key) + key_file=protected/'key.json' + require(not key_file.is_symlink(),'unsafe_root_private_path') + if key_file.exists(): + require(digest(key_file.read_bytes()) in { + journal['previous_envelope_digest'],digest(canonical(journal['new_envelope']))}, + 'root_envelope_changed') + else: + require(journal['previous_envelope_digest'] is None,'root_envelope_missing') # This method is called only after the local console has approved the exact - # journal. It also handles the state where SQLite committed but files did not. + # journal. It also handles the state where PostgreSQL committed but protected files did not. await app.open_storage() async with app.metadata.transaction(write=True) as tx: current=tx.setting('active_root_certificate','cert_root') require(current in {old.resource_id,new.resource_id},'trust_anchor_mismatch') - if current != new.resource_id: - existing=await tx.certificate(old.resource_id) - require(canonical(existing)==canonical(old),'trust_anchor_mismatch') - old_key=await tx.credential(old.key_id) - if journal['old_signature']: - verify(old_key.verifier,canonical(journal['statement']),decode(Signature,journal['old_signature']),purpose='root-rotation') + existing=await tx.certificate(old.resource_id) + require(canonical(existing)==canonical(old),'trust_anchor_mismatch') + old_key=await tx.credential(old.key_id) + require(old_key.subject_id==ROOT_SUBJECT and + digest(old_key.verifier)==journal['statement']['old_fingerprint'], + 'rotation_statement_mismatch') + if journal['old_signature'] is not None: + verify(old_key.verifier,canonical(proof),decode(Signature,journal['old_signature']), + purpose='root-rotation') + if current == new.resource_id: + require(canonical(await tx.certificate(new.resource_id))==canonical(new) and + canonical(await tx.csr(csr.resource_id))==canonical(csr) and + old_key.revoked_at is not None and await tx.certificate_revoked(old.resource_id), + 'rotation_resume_mismatch') + installed=await tx.credential(new.key_id) + require(installed.verifier==signer.public_key and installed.revoked_at is None, + 'rotation_resume_mismatch') + else: subject=await tx.subject(ROOT_SUBJECT) require(subject.local_only,'root_policy_corrupt') await tx.save_credential(Credential(id=signer.key_id,subject_id=ROOT_SUBJECT,kind='signing_key', @@ -95,7 +124,7 @@ async def complete(app, journal, *, pin): await tx.register_certificate(new,None,0) event=Event(id=new_id('audit'),type='root.rotate',time=app.clock(),request_id=new.resource_id, actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=new.resource_id),), - data={'statement':journal['statement'],'new_signature':journal['new_signature'], + data={'statement':journal['statement'],'authorization':proof,'new_signature':journal['new_signature'], 'old_signature':journal['old_signature']}) await tx.revoke_certificate(old.resource_id,AuditEvent(event=event,authority=(), before_digest=digest(old),after_digest=digest(new),previous_digest=None,entry_digest='',result='root_rotated')) @@ -107,16 +136,30 @@ async def complete(app, journal, *, pin): tx.set_setting('active_root_certificate',new.resource_id) tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)+1) tx.set_setting('root_rotation:'+new.resource_id,journal['statement']) - protected=app.settings.config_dir/'root' - history=protected/'history'/old.resource_id - history.mkdir(parents=True,exist_ok=True,mode=0o700) - key_file=protected/'key.json' - if key_file.exists() and not (history/'key.json').exists(): - durable_write(history/'key.json',key_file.read_bytes(),mode=0o600) + # The journal and envelope must use the same independently protected root + # directory, including when resuming after the metadata commit. + for directory in (protected,protected/'history',protected/'history'/old.resource_id): + require(not directory.is_symlink(),'unsafe_root_private_path') + directory.mkdir(parents=True,exist_ok=True,mode=0o700) + os.chmod(directory,0o700) + history=protected/'history'/old.resource_id/'key.json' + require(not history.is_symlink(),'unsafe_root_private_path') + if history.exists(): + require(digest(history.read_bytes())==journal['previous_envelope_digest'], + 'rotation_history_mismatch') + elif journal['previous_envelope_digest'] is not None: + # Never archive an already installed new envelope as the old key. + require(key_file.exists() and + digest(key_file.read_bytes())==journal['previous_envelope_digest'], + 'rotation_history_missing') + durable_write(history,key_file.read_bytes(),mode=0o600) durable_write(key_file,canonical(journal['new_envelope']),mode=0o600) durable_write(app.settings.trust_file,canonical(journal['new_trust']),mode=0o444) # Journal survives every earlier failure and is removed only after both # metadata and protected files have durable copies. - journal_path(app).unlink(missing_ok=True) + pending.unlink(missing_ok=True) + directory=os.open(protected,os.O_RDONLY|os.O_DIRECTORY) + try:os.fsync(directory) + finally:os.close(directory) return {'root_id':ROOT_SUBJECT,'certificate_id':new.resource_id,'fingerprint':digest(signer.public_key), 'online_ca_request':csr.resource_id,'restart_required':True,'old_certificate_chains_valid':False} diff --git a/src/msg/application.py b/src/msg/application.py index 390adbaf..591920d2 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -21,6 +21,7 @@ from msg.security.capabilities import install_capabilities, primary_ceiling, base_grants, temporary_ceiling from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer +from msg.security.quarantine import active as quarantine_active from msg.storage.git import GitContentStore @@ -98,21 +99,31 @@ async def load(self): self.cursors=CursorCodec(hmac.digest(self._token_secret,b'cursor-key-v1','sha256')) self.certificates=CertificateValidator(self.registry,root_certificate,root_public,self.settings.service_url,self.clock) async with self.metadata.transaction(write=False) as tx: + quarantined=quarantine_active(tx) await self.certificates.validate(root_certificate.resource_id,tx) root=await tx.subject(ROOT_SUBJECT) require(root.local_only,'root_policy_corrupt') # Only a verified installation may import immutable release-owned rules. # This is idempotent per source digest/version and rejects source deletion. from msg.bootstrap import sync_system_sources - async with self.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root) + marker=self.settings.config_dir/'recovery-drill.json' + quarantined=quarantined or marker.exists() or marker.is_symlink() + if not quarantined: + async with self.metadata.transaction(write=True) as tx: + # Recheck under the writer lock; loading a restore must not + # mutate release resources before recovery has been accepted. + if not quarantine_active(tx): + await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root) + else: + quarantined=True self.authenticator=AuthenticationService(self.registry,self.certificates,self.settings.service_url,self.clock, self.primary_ceiling,self.temporary_ceiling) self.authorizer=AuthorizationService(self.registry,self.certificates) self.executor=OperationExecutor(self.registry,self.metadata,self.contents,self.authenticator,self.authorizer, self.clock,self.receipt_signer) self.executor.application=self - self.executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' + self.executor.recovery_drill_marker=marker + self.executor.recovery_quarantined=quarantined self.executor.response_hook=self._secrets_for_caller self._loaded=True return self diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py index 7d448105..24b533fe 100644 --- a/src/msg/core/executor.py +++ b/src/msg/core/executor.py @@ -19,11 +19,12 @@ def __init__(self,registry,metadata,contents,authenticator,authorizer,clock,rece self.receipt_signer=receipt_signer self.response_hook=None self.recovery_drill_marker=None + self.recovery_quarantined=False self.application=None def recovery_drill_active(self): marker=self.recovery_drill_marker - return marker is not None and (marker.exists() or marker.is_symlink()) + return self.recovery_quarantined or (marker is not None and (marker.exists() or marker.is_symlink())) async def execute(self,request, *, entry='network'): principal=None @@ -38,8 +39,9 @@ async def execute(self,request, *, entry='network'): details={'contract_version':min_version} if min_version is not None else None) require(entry in spec.entries,'entry_not_allowed') self.registry.validate(spec.input_schema,request.arguments) - require(not (spec.effect!='read' and self.recovery_drill_active()), - 'writes_paused') + if self.recovery_drill_active(): + require(spec.effect=='read','writes_paused') + raise Failure('recovery_quarantined') if spec.name=='batch.independent': return await self._independent(request,spec,entry) async with self.metadata.transaction(write=spec.effect!='read') as session: diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 90f479fc..d5070877 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -118,6 +118,11 @@ def parser(): offer_set.add_argument('--max-quantity',type=int,required=True) offer_set.add_argument('--duration-seconds',type=int) ops.add_parser('disable').add_argument('offer_id') + market=sub.add_parser('market',help='Physical-console arbitration configuration') + market_sub=market.add_subparsers(dest='market_command',required=True) + market_sub.add_parser('grant').add_argument('subject_id') + market_sub.add_parser('revoke').add_argument('subject_id') + market_sub.add_parser('publish').add_argument('policy_file',type=Path) backup=sub.add_parser('backup',help='Local service-data backup, excluding root private material') backup.add_argument('destination',type=Path) restore=sub.add_parser('restore',help='Restore service data into new directories only') @@ -162,6 +167,11 @@ def main(argv=None): elif args.root_command=='backup':result=admin.backup(args.destination) else:result=admin.recover(args.source) emit(result);return 0 + if args.command=='market': + from msg.admin.market import MarketAdmin + emit(MarketAdmin(args.config_dir).execute(args.market_command, + subject=getattr(args,'subject_id',None),policy_file=getattr(args,'policy_file',None))) + return 0 if args.command=='money': from msg.admin.money import MoneyAdmin if args.money_command=='offer': diff --git a/src/msg/data/bootstrap.json b/src/msg/data/bootstrap.json index 074406fe..c3785f72 100644 --- a/src/msg/data/bootstrap.json +++ b/src/msg/data/bootstrap.json @@ -43,6 +43,7 @@ }, "features": [ {"feature_id":"bootstrap","enabled_by_default":true,"default_config":{},"sample_resource":"r_root","doctor_check":"bootstrap","selftest_case":"isolated_namespace"}, + {"feature_id":"market","enabled_by_default":true,"default_config":{"checkout_version":2,"production_fixtures":false,"arbitrators":[]},"sample_resource":null,"doctor_check":"market","selftest_case":"market_lifecycle"}, {"feature_id":"postgres","enabled_by_default":true,"default_config":{},"sample_resource":"r_root","doctor_check":"storage","selftest_case":"stable_id_read"}, {"feature_id":"root_trust","enabled_by_default":true,"default_config":{},"sample_resource":"u_root","doctor_check":"root_trust","selftest_case":"root_network_rejected"}, {"feature_id":"credential_delivery","enabled_by_default":true,"default_config":{"credential_delivery_recovery_window":"15m","release":"once","secret_url":false},"sample_resource":null,"doctor_check":"credential_delivery","selftest_case":"credential_delivery_recovery"}, diff --git a/src/msg/market/__init__.py b/src/msg/market/__init__.py index e69de29b..619af723 100644 --- a/src/msg/market/__init__.py +++ b/src/msg/market/__init__.py @@ -0,0 +1 @@ +"""Versioned market facts; only the executor/worker owns SQL transactions.""" diff --git a/src/msg/market/arbitration.py b/src/msg/market/arbitration.py new file mode 100644 index 00000000..09aa4a05 --- /dev/null +++ b/src/msg/market/arbitration.py @@ -0,0 +1,481 @@ +"""Deterministic panels and signed, quorum-bound escrow decisions. + +A Case never holds a key or balance. A panel can attest only one allocation per +member/round. The EscrowEngine consumes the current, revalidated Decision once. +""" +from __future__ import annotations + +from datetime import timedelta + +from msg.core.codec import canonical, decode, digest, loads, parse_time, wire +from msg.core.errors import Failure, require +from msg.core.models import BlobRef, HandlerOutput, Principal, ResourceRef, Signature +from msg.market.orders import HASH, view +from msg.market.policy import contract +from msg.market.rationale import PINNED_REF +from msg.market.rationale import verified as verify_rationale +from msg.plugins.common import check_access, new_id +from msg.plugins.orders import _row, _subject +from msg.plugins.schemas import IDENTIFIER, REF, SIGNATURE, obj +from msg.security.crypto import verify + +REASONS = ('quality','not_as_described','wrong_ciphertext','delivery_missing', + 'delivery_digest_mismatch','package_missing','package_digest_mismatch', + 'delivery_timeout','other') +CASE = obj({'case_id': IDENTIFIER}, ('case_id',)) +PROPOSAL = obj({'case_id': IDENTIFIER, 'order_id': IDENTIFIER, + 'round': {'type':'integer','minimum':0,'maximum':1}, 'policy_digest': HASH, + 'outcome': {'enum':['release','refund','split']}, + 'refund_minor': {'type':'integer','minimum':0,'maximum':2**63-1}, + 'expires_at': {'type':'string','maxLength':40}, + 'rationale_ref': PINNED_REF, 'rationale_digest': HASH}, + ('case_id','order_id','round','policy_digest','outcome','refund_minor','expires_at', + 'rationale_ref','rationale_digest')) + + +def _read(tx, case_id): + row = tx.one('SELECT body,round,state,deadline FROM arbitration_cases WHERE id=?', (case_id,)) + require(row is not None, 'case_not_found') + case = loads(row[0]) + case.update(round=row[1], state=row[2], deadline=row[3]) + return case + + +def _conflict(tx, candidate, case): + return candidate in {case['buyer'],case['seller']} or tx.one('''SELECT 1 + FROM arbitrator_conflicts WHERE arbitrator=? AND party IN (?,?)''', + (candidate,case['buyer'],case['seller'])) is not None + + +async def member_valid(tx, candidate, case): + epoch = case['policy']['candidate_epochs'].get(candidate) + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (candidate,)) + if not epoch or not row or not row[1] or row[0] != epoch or _conflict(tx, candidate, case): + return False + try: + subject, resource = await tx.subject(candidate), await tx.resource(candidate) + return not subject.local_only and subject.kind == 'registered' and resource.state == 'active' + except Failure: + return False + + +async def panel_for(tx, case, round_number): + eligible = [] + previous = set(case.get('panels', {}).get('0', ())) if round_number else set() + for candidate in case['policy']['policy']['candidates']: + if candidate not in previous and await member_valid(tx, candidate, case): + eligible.append(candidate) + def score(candidate): + return digest({'version': 1, 'order_id': case['order_id'], + 'policy_digest': case['policy']['policy_digest'], 'round': round_number, + 'subject': candidate}) + return sorted(eligible, key=lambda candidate: (score(candidate),candidate))[ + :case['policy']['policy']['panel_size']] + + +async def panel_valid(tx, case): + panel = case['panels'][str(case['round'])] + return (len(panel) == case['policy']['policy']['panel_size'] and + all([await member_valid(tx, candidate, case) for candidate in panel])) + + +async def owned_case(tx, ctx, case_id, *, parties=False): + viewer = ctx.principal.subject + require(viewer is not None and ctx.principal.actor == viewer, 'case_not_found') + case = _read(tx, case_id) + if viewer in {case['buyer'],case['seller']}: + return case + if not parties and viewer in case['panels'][str(case['round'])] and await member_valid(tx, viewer, case): + return case + raise Failure('case_not_found') + + +def proposal_base(case): + policy = case['policy']['policy'] + deadline = parse_time(case['round_deadlines'][str(case['round'])]) + return {'case_id': case['id'], 'order_id': case['order_id'], 'round': case['round'], + 'policy_digest': case['policy']['policy_digest'], + 'expires_at': wire(deadline+timedelta(seconds=policy['decision_lifetime_seconds']))} + + +def proposal_valid(case, proposal, total): + base = proposal_base(case) + require(set(proposal) == {*base,'outcome','refund_minor','rationale_ref','rationale_digest'} and + all(proposal[k] == v for k,v in base.items()), 'decision_context_mismatch') + refund, outcome = proposal['refund_minor'], proposal['outcome'] + require(type(refund) is int and ((outcome=='release' and refund==0) or + (outcome=='refund' and refund==total) or (outcome=='split' and 0 now, 'decision_expired') + require(parse_time(decision['not_before']) <= now, 'appeal_window_open') + require(await panel_valid(tx, case), 'decision_panel_invalid') + require(decision['rationale_ref'] == decision['proposal']['rationale_ref'] and + decision['rationale_digest'] == decision['proposal']['rationale_digest'], + 'decision_rationale_mismatch') + await verify_rationale(app, tx, case, decision['proposal']) + quorum = case['policy']['policy']['quorum'] + require(len(decision['voters']) == quorum and len(set(decision['voters'])) == quorum and + set(decision['voters']) <= set(case['panels'][str(case['round'])]), 'decision_quorum_invalid') + cast_times = [] + for arbitrator in decision['voters']: + vote_row = tx.one('SELECT body FROM arbitration_votes WHERE case_id=? AND round=? AND arbitrator=?', + (case['id'],case['round'],arbitrator)) + require(vote_row is not None, 'decision_quorum_invalid') + vote = loads(vote_row[0]) + require(vote['proposal'] == decision['proposal'], 'decision_conflicting_vote') + await verify_vote(app, tx, case, vote) + cast_times.append(parse_time(vote['cast_at'])) + delay = (case['policy']['policy']['appeal_seconds'] + if case['round']==0 and case['policy']['policy']['allow_appeal'] else 0) + require(parse_time(decision['not_before']) == max(cast_times)+timedelta(seconds=delay) and + decision['expires_at'] == decision['proposal']['expires_at'], 'decision_context_mismatch') + + +def notice(tx, case, state, now): + # Private inbox references only. No testimony, vote, email or payload escapes. + for recipient in {case['buyer'],case['seller']}: + record = {'id': new_id('message'), 'sender': recipient, 'actor': recipient, + 'recipient': recipient, 'resource': wire(ResourceRef(id=recipient)), + 'order_id': case['order_id'], 'case_id': case['id'], + 'time': wire(now), 'state': state, 'source': 'system'} + tx.execute('INSERT INTO messages VALUES (?,?,?,?,?,?)', + (record['id'],recipient,recipient,recipient,'case:'+case['id'],canonical(record).decode()), write=True) + + +async def objective_fault(app, tx, order, now): + locked = contract(tx, order['id']) + if order['state'] == 'funded': + deadline = parse_time(order['funded_at']) + timedelta( + seconds=locked['policy']['policy']['delivery_timeout_seconds']) + if now >= deadline: + return 'delivery_timeout' + if locked['listing']['delivery_mode'] == 'managed_instant': + from msg.market.delivery import package + try: + await package(app, tx, order) + except Failure as exc: + if exc.code in {'package_missing','package_digest_mismatch'}: + return exc.code + raise + if order['state'] == 'delivered': + from msg.market.delivery import verified + try: + await verified(app, tx, order) + except Failure as exc: + if exc.code == 'delivery_not_found': + return 'delivery_missing' + if exc.code in {'delivery_digest_mismatch','package_missing','package_digest_mismatch'}: + return exc.code + raise + return None + + +async def execute(app, tx, case, decision_id, now, actor, request_id): + from msg.market.escrow import settle + order = _row(tx, case['order_id'], case['buyer']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=? AND round=?', + (decision_id,case['id'],case['round'])) + require(row is not None, 'decision_not_current') + decision = loads(row[0]) + existing = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (order['id'],)) + if existing: + settlement = loads(existing[0]) + require(settlement['decision_id'] == decision_id, 'order_already_settled') + return settlement['receipts'] + receipts = await settle(app, tx, order, now=now, actor=actor, request_id=request_id, + reason='arbitration_decision', refund_minor=decision['refund_minor'], decision=decision) + notice(tx, case, 'executed', now) + return receipts + + +async def resolve_cases(app, *, limit=100): + changed, now = [], app.clock() + async with app.metadata.transaction(write=True) as tx: + for (case_id,) in tx.rows('''SELECT id FROM arbitration_cases + WHERE state IN ('open','decided') AND deadline<=? ORDER BY deadline,id LIMIT ?''', + (wire(now),limit)): + case = _read(tx, case_id) + reason = 'quorum_timeout' + if case['state'] == 'decided': + row = tx.one('SELECT id FROM arbitration_decisions WHERE case_id=? AND round=?', + (case_id,case['round'])) + try: + require(row is not None, 'decision_not_current') + await execute(app, tx, case, row[0], now, case['buyer'], 'case-resolve:'+case_id) + changed.append(case_id) + continue + except Failure as exc: + # No unexpected SQL/IO failure is converted into a money rule. + if exc.code not in {'decision_revoked_or_unknown','decision_expired', + 'decision_panel_invalid','decision_member_invalid','credential_revoked', + 'credential_expired','invalid_signature','permission_denied', + 'credential_ceiling','decision_not_current','decision_quorum_invalid', + 'decision_signer_mismatch','decision_conflicting_vote','decision_context_mismatch', + 'decision_rationale_missing','decision_rationale_mismatch'}: + raise + reason = exc.code + tx.execute("UPDATE arbitration_cases SET state='held' WHERE id=?", (case_id,), write=True) + tx.set_setting('case_hold:'+case_id, {'reason': reason, 'at': wire(now)}) + notice(tx, case, 'held', now) + changed.append(case_id) + return changed + + +def install(app, op): + from msg.market.rationale import install as install_rationale + install_rationale(app, op) + + @op('orders.dispute_open', obj({'order_id': IDENTIFIER, + 'reason': {'enum': list(REASONS)}}, ('order_id','reason')), signature=True) + async def open_case(ctx, request, tx): + from msg.market.escrow import settle, transition + actor = _subject(ctx) + order = _row(tx, request.arguments['order_id'], actor) + locked = contract(tx, order['id']) + existing = tx.one('SELECT id FROM arbitration_cases WHERE order_id=?', (order['id'],)) + if existing: + return HandlerOutput(data={'case_id': existing[0], 'order_id': order['id']}) + require(order['state'] in {'funded','delivered'}, 'order_not_disputable') + objective = await objective_fault(app, tx, order, ctx.now) + if objective: + receipts = await settle(app, tx, order, now=ctx.now, actor=actor, + request_id=request.request_id, reason=objective, refund_minor=order['total_price_minor']) + return HandlerOutput(data={'resolution': 'objective_refund', 'reason': objective, + 'order': view(tx, order, actor), 'receipts': receipts}) + case_id = new_id('case') + deadline = wire(ctx.now+timedelta(seconds=locked['policy']['policy']['case_timeout_seconds'])) + case = {'id': case_id, 'order_id': order['id'], 'buyer': order['buyer'], + 'seller': order['seller'], 'opened_by': actor, 'opened_at': wire(ctx.now), + 'reason': request.arguments['reason'], 'policy': locked['policy'], + 'panels': {}, 'round_deadlines': {'0': deadline}, 'round': 0} + case['panels']['0'] = await panel_for(tx, case, 0) + state = 'open' if await panel_valid(tx, case) else 'held' + tx.execute('INSERT INTO arbitration_cases(id,order_id,round,state,deadline,body) VALUES (?,?,0,?,?,?)', + (case_id,order['id'],state,deadline,canonical(case).decode()), write=True) + await transition(tx, order, 'disputed', now=ctx.now, actor=actor, + request_id=request.request_id, reason=request.arguments['reason']) + notice(tx, case, state, ctx.now) + return HandlerOutput(data={'case_id': case_id, 'order_id': order['id'], 'state': state, + 'panel': case['panels']['0'], 'policy_digest': locked['policy']['policy_digest'], + 'deadline': deadline}) + + @op('orders.dispute_get', CASE, effect='read') + async def get(ctx, request, tx): + case = await owned_case(tx, ctx, request.arguments['case_id']) + viewer = ctx.principal.subject + panel = viewer in case['panels'][str(case['round'])] + evidence = [] + for eid,author,visibility,body in tx.rows('''SELECT id,author,visibility,body + FROM arbitration_evidence WHERE case_id=? ORDER BY id''', (case['id'],)): + if panel or viewer == author or visibility == 'parties': + record = loads(body) + evidence.append({'id': eid,'author': author,'visibility': visibility, + 'kind': record['kind'],'round': record['round'], + 'digest': record['blob']['digest'],'size': record['blob']['size'], + **({'rationale_ref': record['source']} if record['kind']=='rationale' else {})}) + decisions = [loads(row[0]) for row in tx.rows( + 'SELECT body FROM arbitration_decisions WHERE case_id=? ORDER BY round', (case['id'],))] + votes = [loads(row[0]) for row in tx.rows( + 'SELECT body FROM arbitration_votes WHERE case_id=? AND round=? ORDER BY arbitrator', + (case['id'],case['round']))] + return HandlerOutput(data={'case': {k:v for k,v in case.items() if k!='policy'}, + 'policy': case['policy'], 'panel_valid': await panel_valid(tx, case), + 'proposal_base': proposal_base(case), 'evidence': evidence, + 'votes': [{'arbitrator':v['arbitrator'],'proposal':v['proposal'],'signature':v['signature']} + for v in votes], 'decisions': decisions}) + + @op('orders.dispute_summary', CASE, effect='read') + async def summary(ctx, request, tx): + case = _read(tx, request.arguments['case_id']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE case_id=? AND round=?', + (case['id'],case['round'])) + return HandlerOutput(data={'case_id': case['id'], 'state': case['state'], + 'policy_digest': case['policy']['policy_digest'], + 'outcome': loads(row[0])['proposal']['outcome'] if row and case['state']=='executed' else None}) + + evidence_fields = {'case_id': IDENTIFIER, 'visibility': {'enum':['panel','parties']}} + @op('orders.dispute_statement', obj({**evidence_fields, + 'text': {'type':'string','minLength':1,'maxLength':4096}}, ('case_id','visibility','text')), + signature=True) + @op('orders.dispute_evidence', obj({**evidence_fields, 'ref': REF}, + ('case_id','visibility','ref')), signature=True) + async def add_evidence(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id'], parties=True) + require(case['state'] in {'open','held'} and + ctx.now < parse_time(case['round_deadlines'][str(case['round'])]), 'case_evidence_closed') + require(tx.one('SELECT COUNT(*) FROM arbitration_evidence WHERE case_id=? AND author=?', + (case['id'],actor))[0] < 64, 'case_evidence_limit') + from msg.market.delivery import verify_blob + if 'text' in request.arguments: + blob = await app.contents.put_bytes(request.arguments['text'].encode(), 'text/plain') + kind, source = 'statement', None + else: + ref = decode(ResourceRef, request.arguments['ref']) + require(ref.revision is not None, 'payload_revision_required') + resource = await tx.resource(ref.id) + require(resource.owner == actor and resource.type in {'file','attachment'} and + resource.state == 'active', 'evidence_not_owned') + await check_access(app, ctx, request, tx, ref.id, 'read') + blob = (await tx.revision(ref)).content + kind, source = 'resource', wire(ref) + await verify_blob(app, blob) + eid = new_id('evidence') + await app.contents.pin(blob, eid) + record = {'kind': kind, 'blob': wire(blob), 'source': source, 'round': case['round'], + 'at': wire(ctx.now), 'author_request_digest': request.payload_digest} + tx.execute('INSERT INTO arbitration_evidence(id,case_id,author,visibility,body) VALUES (?,?,?,?,?)', + (eid,case['id'],actor,request.arguments['visibility'],canonical(record).decode()), write=True) + return HandlerOutput(data={'evidence_id': eid, 'digest': blob.digest}) + + @op('orders.dispute_evidence_get', obj({'case_id': IDENTIFIER,'evidence_id': IDENTIFIER, + 'offset': {'type':'integer','minimum':0}, + 'length': {'type':'integer','minimum':1,'maximum':65536}}, + ('case_id','evidence_id')), effect='read') + async def evidence_get(ctx, request, tx): + case = await owned_case(tx, ctx, request.arguments['case_id']) + row = tx.one('SELECT author,visibility,body FROM arbitration_evidence WHERE id=? AND case_id=?', + (request.arguments['evidence_id'],case['id'])) + viewer = ctx.principal.subject + require(row is not None and (viewer==row[0] or row[1]=='parties' or + viewer in case['panels'][str(case['round'])]), 'evidence_not_found') + record = loads(row[2]) + blob = decode(BlobRef, record['blob']) + start = request.arguments.get('offset', 0) + end = min(blob.size, start+request.arguments.get('length', 65536)) + require(0 <= start <= end <= blob.size, 'invalid_byte_range') + raw = b''.join([part async for part in app.contents.read(blob,(start,end))]) + from msg.core.codec import b64 + return HandlerOutput(data={'evidence_id': request.arguments['evidence_id'], 'digest': blob.digest, + 'size': blob.size, 'offset': start, 'data': b64(raw), 'chunk_digest': digest(raw), + 'next_offset': end if end= quorum: + voters = sorted(matching, key=lambda v:v['arbitrator'])[:quorum] + delay = (case['policy']['policy']['appeal_seconds'] + if case['round']==0 and case['policy']['policy']['allow_appeal'] else 0) + decision = {'id': 'dec_'+digest(proposal)[7:39], 'case_id': case['id'], + 'order_id': order['id'], 'round': case['round'], 'proposal': proposal, + 'refund_minor': proposal['refund_minor'], 'policy_digest': proposal['policy_digest'], + 'rationale_ref': proposal['rationale_ref'], 'rationale_digest': proposal['rationale_digest'], + 'voters': [v['arbitrator'] for v in voters], 'expires_at': proposal['expires_at'], + 'not_before': wire(max(parse_time(v['cast_at']) for v in voters)+timedelta(seconds=delay))} + tx.execute('INSERT INTO arbitration_decisions(id,case_id,round,body) VALUES (?,?,?,?)', + (decision['id'],case['id'],case['round'],canonical(decision).decode()), write=True) + tx.execute("UPDATE arbitration_cases SET state='decided',deadline=? WHERE id=?", + (decision['not_before'],case['id']), write=True) + notice(tx, case, 'decided', ctx.now) + return HandlerOutput(data={'case_id': case['id'], 'votes_for_proposal': len(matching), 'decision': decision}) + + @op('orders.dispute_execute', obj({'case_id': IDENTIFIER,'decision_id': IDENTIFIER}, + ('case_id','decision_id')), signature=True) + async def execute_decision(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id']) + receipts = await execute(app, tx, case, request.arguments['decision_id'], ctx.now, actor, request.request_id) + return HandlerOutput(data={'case_id': case['id'],'state':'executed','receipts':receipts}) + + @op('orders.dispute_appeal', CASE, signature=True) + async def appeal(ctx, request, tx): + _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id'], parties=True) + policy = case['policy']['policy'] + require(policy['allow_appeal'] and case['round']==0 and case['state'] in {'decided','held'}, + 'appeal_not_available') + row = tx.one('SELECT body FROM arbitration_decisions WHERE case_id=? AND round=0', (case['id'],)) + until = (parse_time(loads(row[0])['not_before']) if row else + parse_time(case['round_deadlines']['0'])+timedelta(seconds=policy['appeal_seconds'])) + require(ctx.now < until, 'appeal_window_closed') + case['panels']['1'] = await panel_for(tx, case, 1) + case['round'] = 1 + deadline = wire(ctx.now+timedelta(seconds=policy['case_timeout_seconds'])) + case['round_deadlines']['1'] = deadline + state = 'open' if await panel_valid(tx, case) else 'held' + tx.execute('UPDATE arbitration_cases SET round=1,state=?,deadline=?,body=? WHERE id=?', + (state,deadline,canonical(case).decode(),case['id']), write=True) + notice(tx, case, 'appealed', ctx.now) + return HandlerOutput(data={'case_id':case['id'],'round':1,'state':state,'panel':case['panels']['1']}) + + @op('orders.dispute_revoke', obj({'case_id': IDENTIFIER,'decision_id': IDENTIFIER}, + ('case_id','decision_id')), signature=True) + async def revoke_decision(ctx, request, tx): + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id']) + row = tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=? AND round=?', + (request.arguments['decision_id'],case['id'],case['round'])) + require(row is not None and actor in loads(row[0])['voters'], 'decision_signer_required') + require(case['state'] in {'decided','held'}, 'decision_already_executed') + tx.execute('UPDATE arbitration_decisions SET revoked=TRUE WHERE id=?', + (request.arguments['decision_id'],), write=True) + tx.execute("UPDATE arbitration_cases SET state='held' WHERE id=?", (case['id'],), write=True) + notice(tx, case, 'held', ctx.now) + return HandlerOutput(data={'case_id':case['id'],'state':'held','decision_revoked':True}) + + @op('orders.arbitrator_conflict', obj({'party': IDENTIFIER}, ('party',)), signature=True) + async def declare_conflict(ctx, request, tx): + actor = _subject(ctx) + role = tx.one('SELECT active FROM arbitrator_roles WHERE subject=?', (actor,)) + require(role is not None and role[0], 'arbitrator_required') + party = (await tx.subject(request.arguments['party'])).resource_id + tx.execute('INSERT INTO arbitrator_conflicts(arbitrator,party) VALUES (?,?) ON CONFLICT DO NOTHING', + (actor,party), write=True) + return HandlerOutput(data={'declared':True}) diff --git a/src/msg/market/delivery.py b/src/msg/market/delivery.py new file mode 100644 index 00000000..940468c9 --- /dev/null +++ b/src/msg/market/delivery.py @@ -0,0 +1,338 @@ +"""Automatic managed packages and explicitly accepted ciphertext/service evidence.""" +from __future__ import annotations + +import hashlib + +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import Failure, require +from msg.core.models import BlobRef, HandlerOutput, ResourceRef +from msg.market.escrow import settle, transition +from msg.market.orders import HASH, ORDER +from msg.market.policy import contract +from msg.market.targets import enqueue_notification, validate_target +from msg.plugins.common import check_access, new_id +from msg.plugins.delivery import _buyer_order, _delivery +from msg.plugins.orders import _row, _subject, _viewer +from msg.plugins.schemas import IDENTIFIER, REF, obj +from msg.plugins.store import _package_row + +INLINE = 65536 +PART = 65536 + + +async def verify_blob(app, blob): + """Bounded-memory verification, including content loss after a deposit.""" + hasher, size = hashlib.sha256(), 0 + try: + async for chunk in app.contents.read(blob): + hasher.update(chunk) + size += len(chunk) + except FileNotFoundError: + raise Failure('package_missing') from None + except Failure as exc: + if exc.code == 'content_missing': + raise Failure('package_missing') from None + if exc.code in {'content_truncated','content_size_mismatch'}: + raise Failure('package_digest_mismatch') from None + raise + require(size == blob.size and 'sha256:'+hasher.hexdigest() == blob.digest, + 'package_digest_mismatch') + + +async def package(app, tx, order): + row = await _package_row(tx, order['package_id']) + require(row is not None, 'package_missing') + require(row[1] == order['listing_id'] and row[3] == order['seller'] and + row[4] == order['package_revision'] and row[8] == order['package_digest'] and + row[10] == 'managed_instant' and row[5] in {'file','bundle','text'}, + 'package_digest_mismatch') + refs, manifest = loads(row[7]), loads(row[6]) + snapshot = {'seller': row[3], 'listing_id': row[1], 'listing_revision': row[2], + 'revision': row[4], 'kind': row[5], 'manifest': manifest, 'payload_refs': refs, + 'total_size': row[9], 'delivery_mode': row[10]} + require(digest(snapshot) == order['package_digest'] and 0 <= row[9] <= 1024**4, + 'package_digest_mismatch') + require(1 <= len(refs) <= 32, 'package_digest_mismatch') + total = 0 + for ref in refs: + blob = decode(BlobRef, ref['blob']) + await verify_blob(app, blob) + total += blob.size + require(total == row[9], 'package_digest_mismatch') + return row[5], manifest, refs + + +def body_for(order, delivery_id, kind, manifest, refs): + return {'delivery_id': delivery_id, 'order_id': order['id'], + 'recipient_subject': order['buyer'], 'kind': kind, 'manifest': manifest, + 'payload_refs': refs, 'package_digest': order['package_digest'] or + contract_digest(order), 'channel': 'site'} + + +def contract_digest(order): + # Non-managed goods have no deposited package. This is a terms commitment, + # not a false claim that the service/ciphertext content was fixed at checkout. + return digest({'listing_revision': order['listing_revision'], 'terms_digest': order['terms_digest']}) + + +async def prepare(app, tx, ctx, request, order, kind, manifest, refs, *, envelope=None): + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_deliverable') + validate_target(tx, order, encryption=envelope is not None) + delivery_id = new_id('dlv') + for ref in refs: + await app.contents.pin(decode(BlobRef, ref['blob']), delivery_id) + body = body_for(order, delivery_id, kind, manifest, refs) + tx.execute('''INSERT INTO store_deliveries + (id,order_id,recipient_subject,kind,payload_refs,manifest,package_digest, + delivery_digest,channel,state,prepared_at,claimed_at,receipt) + VALUES (?,?,?,?,?,?,?,?,'site','prepared',?,NULL,NULL)''', + (delivery_id,order['id'],order['buyer'],kind,canonical(refs).decode(), + canonical(manifest).decode(),body['package_digest'],digest(body),wire(ctx.now)), write=True) + if envelope: + tx.execute('INSERT INTO delivery_envelopes(order_id,body) VALUES (?,?)', + (order['id'],canonical(envelope).decode()), write=True) + await transition(tx, order, 'delivered', now=ctx.now, actor=ctx.principal.actor, + request_id=request.request_id, reason='verified_delivery') + tx.execute('UPDATE store_orders SET delivered_at=? WHERE id=?', + (wire(ctx.now),order['id']), write=True) + order['delivered_at'] = wire(ctx.now) + # Inbox is a minimal reference to the one authoritative order, not a copy. + message = {'id': new_id('message'), 'sender': order['seller'], 'actor': order['seller'], + 'recipient': order['buyer'], 'order_id': order['id'], 'time': wire(ctx.now), + 'resource': wire(ResourceRef(id=order['buyer'])), 'state': 'delivered', 'source': 'system', + 'href': '/_orders/'+order['id']+'/_delivery'} + tx.execute('INSERT INTO messages VALUES (?,?,?,?,?,?)', + (message['id'],order['seller'],order['buyer'],order['buyer'], + 'delivery:'+order['id'],canonical(message).decode()), write=True) + await enqueue_notification(app, tx, ctx, request, order) + return _delivery(tx, order['id']) + + +async def automatic(app, tx, ctx, request, order): + """An authenticated payment authorizes the pinned managed-instant policy. + + Only verified objective storage faults refund. An arbitrary exception rolls + back the executor transaction, rather than turning software bugs into payouts. + """ + try: + kind, manifest, refs = await package(app, tx, order) + except Failure as exc: + if exc.code not in {'package_missing','package_digest_mismatch'}: + raise + await settle(app, tx, order, now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason=exc.code, refund_minor=order['total_price_minor']) + return + await prepare(app, tx, ctx, request, order, kind, manifest, refs) + await transition(tx, order, 'accepted', now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='managed_instant_verified') + await settle(app, tx, order, now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='managed_instant_verified') + # Delivery remains prepared, not claimed. Server fulfilment is not buyer ACK. + + +async def verified(app, tx, order, *, verify_bytes=True): + delivery = _delivery(tx, order['id']) + require(delivery is not None, 'delivery_not_found') + validate_target(tx, order) + require(delivery['recipient_subject'] == order['buyer'], 'delivery_recipient_mismatch') + expected = body_for(order, delivery['id'], delivery['kind'], + delivery['manifest'], delivery['payload_refs']) + require(delivery['package_digest'] == expected['package_digest'] and + delivery['delivery_digest'] == digest(expected), 'delivery_digest_mismatch') + locked = contract(tx, order['id']) + if locked['listing']['delivery_mode'] == 'managed_instant': + row = await _package_row(tx, order['package_id']) + require(row is not None and row[8] == order['package_digest'] and + loads(row[6]) == delivery['manifest'] and loads(row[7]) == delivery['payload_refs'], + 'delivery_digest_mismatch') + elif locked['listing']['delivery_mode'] == 'sealed_manual': + envelope = tx.one('SELECT body FROM delivery_envelopes WHERE order_id=?', (order['id'],)) + require(envelope is not None, 'delivery_digest_mismatch') + envelope = loads(envelope[0]) + require(envelope['recipient_subject'] == order['buyer'] and + envelope['recipient_key'] == locked['recipient_key'] and + envelope['ciphertext_ref'] == delivery['payload_refs'][0], + 'delivery_recipient_mismatch') + if verify_bytes: + for ref in delivery['payload_refs']: + await verify_blob(app, decode(BlobRef, ref['blob'])) + return delivery + + +async def read(app, tx, ctx, request): + buyer = _viewer(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + delivery = await verified(app, tx, order) + payloads, budget = [], INLINE + for index, ref in enumerate(delivery['payload_refs']): + blob = decode(BlobRef, ref['blob']) + item = {'id': ref['id'], 'revision': ref['revision'], 'digest': blob.digest, + 'size': blob.size, 'media_type': blob.media_type, 'payload_index': index} + if blob.size <= budget: + item['data'] = b64(await app.contents.read_bytes(blob, limit=INLINE)) + budget -= blob.size + else: + item['transfer'] = {'operation':'delivery.transfer_open','arguments':{ + 'order_id':order['id'],'delivery_digest':delivery['delivery_digest'], + 'payload_index':index}} + item['next'] = {'operation': 'delivery.part_get', 'arguments': { + 'order_id': order['id'], 'delivery_digest': delivery['delivery_digest'], + 'payload_index': index, 'offset': 0, 'length': min(PART,blob.size)}} + payloads.append(item) + result = {k:v for k,v in delivery.items() if k not in {'id','payload_refs','receipt'}} + result.update(delivery_id=delivery['id'], payloads=payloads) + return HandlerOutput(data={'delivery': result}) + + +async def accept(app, tx, ctx, request): + buyer = _subject(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + delivery = await verified(app, tx, order) + require(delivery['delivery_digest'] == request.arguments['delivery_digest'], 'delivery_mismatch') + require(order['state'] in {'delivered','settled'}, 'delivery_not_acceptable') + if delivery['state'] == 'claimed': + return HandlerOutput(data={'order_id': order['id'], 'state': order['state'], + 'delivery_id': delivery['id'], 'receipt': delivery['receipt']}) + settlement = [] + if order['state'] == 'delivered': + await transition(tx, order, 'accepted', now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance') + settlement = await settle(app, tx, order, now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_acceptance') + body = {'order_id': order['id'], 'delivery_id': delivery['id'], 'subject': buyer, + 'delivery_digest': delivery['delivery_digest'], 'request_digest': request.payload_digest, + 'claimed_at': wire(ctx.now)} + receipt = {'body': body, + 'signature': wire(app.receipt_signer.sign(canonical(body), purpose='delivery-ack'))} + tx.execute("UPDATE store_deliveries SET state='claimed',claimed_at=?,receipt=? WHERE order_id=?", + (wire(ctx.now),canonical(receipt).decode(),order['id']), write=True) + return HandlerOutput(data={'order_id': order['id'], 'state': order['state'], + 'delivery_id': delivery['id'], 'receipt': receipt, 'settlement_receipts': settlement}) + + +def install(app, op): + @op('delivery.get', ORDER, effect='read', version=2) + async def get(ctx, request, tx): + return await read(app, tx, ctx, request) + + @op('delivery.accept', obj({'order_id': IDENTIFIER, 'delivery_digest': HASH}, + ('order_id','delivery_digest')), signature=True, version=2) + async def claim(ctx, request, tx): + return await accept(app, tx, ctx, request) + + @op('delivery.prepare', ORDER, signature=True, version=2) + async def prepare_managed(ctx, request, tx): + buyer = _subject(ctx) + order = _buyer_order(tx, request.arguments['order_id'], buyer) + locked = contract(tx, order['id']) + require(locked['listing']['delivery_mode'] == 'managed_instant', 'seller_delivery_required') + if order['state'] == 'funded': + await automatic(app, tx, ctx, request, order) + return HandlerOutput(data={'order_id': order['id'], 'state': order['state']}) + + @op('delivery.submit', obj({'order_id': IDENTIFIER, 'payload_ref': REF, + 'payload_digest': HASH, 'recipient_key_id': IDENTIFIER}, + ('order_id','payload_ref','payload_digest')), signature=True) + async def submit(ctx, request, tx): + from msg.plugins.communication import direct_ancestor + seller = _subject(ctx) + order = _row(tx, request.arguments['order_id'], seller) + require(order['seller'] == seller, 'order_not_found') + locked = contract(tx, order['id']) + mode = locked['listing']['delivery_mode'] + require(mode in {'sealed_manual','service'}, 'order_delivery_mode_unsupported') + require(order['state'] == 'funded', 'order_not_deliverable') + from datetime import timedelta + + from msg.core.codec import parse_time + require(ctx.now < parse_time(order['funded_at']) + timedelta( + seconds=locked['policy']['policy']['delivery_timeout_seconds']), 'delivery_deadline_passed') + ref = decode(ResourceRef, request.arguments['payload_ref']) + require(ref.revision is not None, 'payload_revision_required') + resource = await tx.resource(ref.id) + require(resource.owner == seller and resource.type in {'file','attachment'} and + resource.state == 'active', 'payload_not_owned') + require(await direct_ancestor(tx, ref.id) is None, 'payload_private_conversation') + await check_access(app, ctx, request, tx, ref.id, 'read') + revision = await tx.revision(ref) + blob = revision.content + require(blob.digest == request.arguments['payload_digest'], 'delivery_digest_mismatch') + await verify_blob(app, blob) + payload = {'id': ref.id, 'revision': ref.revision, 'blob': wire(blob)} + envelope = None + if mode == 'sealed_manual': + validate_target(tx, order, encryption=True) + require(request.arguments.get('recipient_key_id') == locked['recipient_key']['key_id'], + 'delivery_recipient_mismatch') + prefix = b''.join([piece async for piece in app.contents.read(blob, (0,min(64,blob.size)))]) + require(blob.media_type in {'application/age','application/octet-stream'} and + prefix.startswith(b'age-encryption.org/v1\n'), 'encrypted_delivery_required') + envelope = {'recipient_subject': order['buyer'], 'recipient_key': locked['recipient_key'], + 'ciphertext_ref': payload, 'seller_request_digest': request.payload_digest} + manifest = {'encryption': 'age-x25519', 'recipient_key_id': locked['recipient_key']['key_id']} + else: + require('recipient_key_id' not in request.arguments, 'unexpected_recipient_key') + manifest = {'type': 'service-evidence', 'acceptance': 'explicit-buyer-signature'} + delivery = await prepare(app, tx, ctx, request, order, locked['listing']['item_kind'], + manifest, [payload], envelope=envelope) + return HandlerOutput(data={'order_id': order['id'], 'delivery_id': delivery['id'], + 'delivery_digest': delivery['delivery_digest'], 'state': 'delivered'}) + + @op('delivery.part_get', obj({'order_id': IDENTIFIER, 'delivery_digest': HASH, + 'payload_index': {'type': 'integer','minimum': 0,'maximum': 31}, + 'offset': {'type': 'integer','minimum': 0,'maximum': 1024**4}, + 'length': {'type': 'integer','minimum': 1,'maximum': PART}}, + ('order_id','delivery_digest','payload_index','offset','length')), effect='read') + async def part_get(ctx, request, tx): + a = request.arguments + order = _buyer_order(tx, a['order_id'], _viewer(ctx)) + delivery = await verified(app, tx, order, verify_bytes=False) + require(a['delivery_digest'] == delivery['delivery_digest'], 'delivery_digest_mismatch') + require(a['payload_index'] < len(delivery['payload_refs']), 'payload_not_found') + blob = decode(BlobRef, delivery['payload_refs'][a['payload_index']]['blob']) + start, end = a['offset'], a['offset']+a['length'] + require(start < end <= blob.size, 'invalid_byte_range') + data = b''.join([piece async for piece in app.contents.read(blob, (start,end))]) + require(len(data) == a['length'], 'delivery_digest_mismatch') + following = ({**a, 'offset': end, 'length': min(PART,blob.size-end)} if end 32 and ord(c) != 127 for c in origin) and + '\\' not in origin, 'invalid_delivery_origin') + try: + base = urlsplit(origin) + valid = (base.scheme in {'http', 'https'} and base.hostname and + base.username is None and base.password is None and + not base.query and not base.fragment) + base.port # Validate malformed/out-of-range ports as well as IPv6 syntax. + except ValueError: + raise Failure('invalid_delivery_origin') from None + require(valid, 'invalid_delivery_origin') + return origin.rstrip('/') + '/_orders/' + order_id + '/_delivery' + + +def account_email(tx, buyer): + """Read the live account row once; endpoint encodings remain version-specific.""" + row = tx.one('SELECT generation,body FROM emails WHERE subject=?', (buyer,)) + if row is None: + return None, None + email = decode(EmailSettings, loads(row[1])) + require(email.subject_id == buyer, 'delivery_recipient_mismatch') + return row[0], email + + def validate_target(order, delivery=None): require(order['delivery_target'] == { 'subject_id': order['buyer'], 'channel': 'site'}, @@ -31,16 +65,12 @@ def email_binding(tx, buyer, address): The existing single-email row is the endpoint. Its generation also fences revoke/reverify and address reuse; equality of the address alone is unsafe. """ - row = tx.one('SELECT generation,body FROM emails WHERE subject=?', (buyer,)) - if row is None: - return None - email = decode(EmailSettings, loads(row[1])) - require(email.subject_id == buyer, 'delivery_recipient_mismatch') - if email.address != address or email.verified_at is None: + generation, email = account_email(tx, buyer) + if email is None or email.address != address or email.verified_at is None: return None validate_address(email.address) return {'endpoint_id': 'email:' + buyer, 'owner': buyer, - 'generation': row[0], 'address_snapshot': address, + 'generation': generation, 'address_snapshot': address, 'verified_at': wire(email.verified_at)} diff --git a/src/msg/market/email.py b/src/msg/market/email.py new file mode 100644 index 00000000..b1000832 --- /dev/null +++ b/src/msg/market/email.py @@ -0,0 +1,100 @@ +"""Optional checkout-address verification, separate from account email settings. + +The emailed proof cannot authenticate or identify an order. Verification also +requires the original buyer's current signed request. An endpoint never changes +owner; a disabled endpoint is not reactivated by retries or a restored job. +""" +from __future__ import annotations + +import secrets +from dataclasses import replace +from datetime import timedelta + +from msg.core.codec import b64, canonical, digest, parse_time, wire +from msg.core.errors import require +from msg.core.models import EffectJob, HandlerOutput +from msg.market.targets import enqueue_notification, mail_enabled, save_target, validate_target +from msg.plugins.common import new_id +from msg.plugins.orders import _row, _subject +from msg.plugins.schemas import IDENTIFIER, obj + + +async def verification_message(app, tx, ctx, request, order): + chosen = order['delivery_target'].get('email', {}) + if (not mail_enabled(app) or chosen.get('endpoint_id') or + not chosen.get('address_snapshot')): + return + token, challenge_id = b64(secrets.token_bytes(32)), new_id('evc') + tx.execute('''INSERT INTO delivery_email_challenges + (id,order_id,subject,address,verifier,expires_at) VALUES (?,?,?,?,?,?)''', + (challenge_id,order['id'],order['buyer'],chosen['address_snapshot'],digest(token), + wire(ctx.now+timedelta(minutes=15))), write=True) + save_target(tx, order, {**order['delivery_target'], 'email': { + **chosen, 'challenge_id':challenge_id, 'state':'pending'}}) + await tx.enqueue(EffectJob(id=new_id('job'),event_id=challenge_id,kind='market_email_verify', + dedupe_key='market-verify:'+challenge_id,principal=ctx.principal, + operation=request.operation,arguments={'challenge_id':challenge_id,'token':token}, + state='pending',attempts=0,next_attempt_at=ctx.now,lease_until=None)) + + +async def render_verification(app, tx, job): + row = tx.one('''SELECT order_id,subject,address,verifier,expires_at,consumed + FROM delivery_email_challenges WHERE id=?''', (job.arguments['challenge_id'],)) + require(row is not None and not row[5] and parse_time(row[4])>app.clock() and + row[1] == job.principal.subject == job.principal.actor and + digest(job.arguments['token']) == row[3], 'email_challenge_expired') + order = _row(tx,row[0],row[1]) + chosen = order['delivery_target'].get('email',{}) + require(order['buyer']==row[1] and chosen.get('challenge_id')==job.arguments['challenge_id'] and + chosen.get('address_snapshot')==row[2] and chosen.get('endpoint_id') is None and + order['state'] in {'created','funded','delivered','settled'}, 'email_challenge_expired') + # No order ID, buyer handle, goods, pickup link or authority in this text. + return replace(job,kind='mail',arguments={'recipient':row[2], + 'recipient_subject':row[1], 'subject':'msg address verification', + 'text':canonical({'challenge_id':job.arguments['challenge_id'], + 'token':job.arguments['token'], 'purpose':'verify this address only'}).decode()}) + + +def install(app, op): + from msg.market.orders import ORDER, view + + @op('orders.email_verify',obj({'order_id':IDENTIFIER,'challenge_id':IDENTIFIER, + 'token':{'type':'string','minLength':43,'maxLength':43}}, + ('order_id','challenge_id','token')),signature=True) + async def verify(ctx, request, tx): + a, buyer = request.arguments, _subject(ctx) + order = _row(tx,a['order_id'],buyer) + require(order['buyer']==buyer,'order_not_found') + row = tx.one('''SELECT order_id,subject,address,verifier,expires_at,consumed + FROM delivery_email_challenges WHERE id=?''',(a['challenge_id'],)) + chosen = order['delivery_target'].get('email',{}) + require(row is not None and row[0]==order['id'] and row[1]==buyer and not row[5] and + parse_time(row[4])>ctx.now and secrets.compare_digest(row[3],digest(a['token'])) and + chosen.get('challenge_id')==a['challenge_id'] and + chosen.get('address_snapshot')==row[2] and chosen.get('endpoint_id') is None and + order['state'] in {'created','funded','delivered','settled'},'email_challenge_invalid') + endpoint = new_id('dep') + tx.execute('INSERT INTO delivery_email_endpoints(id,subject,address,verified_at) VALUES (?,?,?,?)', + (endpoint,buyer,row[2],wire(ctx.now)),write=True) + tx.execute('UPDATE delivery_email_challenges SET consumed=TRUE WHERE id=?', + (a['challenge_id'],),write=True) + chosen = {k:v for k,v in chosen.items() if k!='challenge_id'} + save_target(tx,order,{**order['delivery_target'],'email':{ + **chosen,'endpoint_id':endpoint,'state':'pending'}}) + validate_target(tx,order,email=True) + if order['delivered_at']: + await enqueue_notification(app,tx,ctx,request,order) + return HandlerOutput(data={'order':view(tx,order,buyer)}) + + @op('orders.email_disable',ORDER,signature=True) + async def disable(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx,request.arguments['order_id'],buyer) + require(order['buyer']==buyer,'order_not_found') + chosen = order['delivery_target'].get('email',{}) + if (chosen.get('endpoint_id') or '').startswith('dep_'): + tx.execute('UPDATE delivery_email_endpoints SET active=FALSE WHERE id=? AND subject=?', + (chosen['endpoint_id'],buyer),write=True) + save_target(tx,order,{**order['delivery_target'],'email':{ + **chosen,'state':'disabled','notification_disabled':True}}) + return HandlerOutput(data={'order':view(tx,order,buyer)}) diff --git a/src/msg/market/escrow.py b/src/msg/market/escrow.py index c33c91d6..92e0bb59 100644 --- a/src/msg/market/escrow.py +++ b/src/msg/market/escrow.py @@ -1,20 +1,166 @@ -"""Single escrow debit boundary for the implemented objective policy. - -This is not an arbitration service: v1 supports a buyer's pre-delivery -cancellation and explicit acceptance of a verified delivery. Disputed orders, -subjective claims, splits and time-based decisions fail closed. No LLM, role or -administrator can supply an arbitrary payout to this entry point. -""" +"""One escrow release entry point, guarded transitions and append-only facts.""" from __future__ import annotations +from datetime import timedelta + from msg.core.codec import b64, canonical, decode, digest, freeze_json, loads, parse_time, wire from msg.core.errors import require -from msg.core.models import Signature +from msg.core.models import Event, Signature from msg.core.requests import signing_bytes +from msg.market.policy import contract, delivery_snapshot +from msg.plugins.common import new_id from msg.plugins.money import CURRENCY_ID, _balance, _post_transfer from msg.security.crypto import verify +# Not a credential, account or public capability. Only trusted in-process market +# code can reach this token; ordinary signed money.transfer never receives it. +_ESCROW_WRITE = object() +TRANSITIONS = { + 'created': {'funded', 'cancelled'}, + 'funded': {'delivered', 'refunded', 'disputed'}, + 'delivered': {'accepted', 'refunded', 'disputed'}, + 'accepted': {'settled'}, + 'disputed': {'refunded', 'settled'}, + 'cancelled': set(), 'refunded': set(), 'settled': set(), +} +OBJECTIVE_REFUNDS = frozenset({'package_missing', 'package_digest_mismatch', + 'delivery_missing', 'delivery_digest_mismatch', 'delivery_timeout', 'buyer_cancelled'}) + + +def _post_escrow_transfer(tx, *, order, recipient, amount, actor, request_id, now, + receipt_signer, reference, kind): + """The only place that supplies internal authority to debit OrderEscrow. + + Legacy signed decisions and versioned arbitration keep their distinct + journals, signatures and request IDs, but share this account boundary. + """ + account = tx.one('SELECT kind,subject_id,source_id FROM ledger_accounts WHERE id=?', + (order['escrow_subject'],)) + require(account == ('order_escrow', None, order['id']), 'escrow_account_mismatch') + require(recipient in {order['buyer'], order['seller']}, 'escrow_recipient_mismatch') + return _post_transfer(tx, sender=order['escrow_subject'], recipient=recipient, + amount=amount, actor=actor, request_id=request_id, now=now, + receipt_signer=receipt_signer, reference=reference, kind=kind, + escrow_authority=_ESCROW_WRITE) + + +async def transition(tx, order, state, *, now, actor, request_id, reason): + require(state in TRANSITIONS.get(order['state'], ()), 'order_transition_invalid') + changed = tx.execute('UPDATE store_orders SET state=? WHERE id=? AND state=?', + (state, order['id'], order['state']), write=True) + require(changed.rowcount == 1, 'order_transition_conflict') + fact = {'order_id': order['id'], 'from': order['state'], 'to': state, + 'at': wire(now), 'reason': reason, 'policy': order['dispute_policy'], + 'actor': actor, 'request_id': request_id} + tx.execute('INSERT INTO order_transitions(id,order_id,body) VALUES (?,?,?)', + (new_id('ot'), order['id'], canonical(fact).decode()), write=True) + # Events deliberately contain no buyer handle, evidence, goods or address. + await tx.append_event(Event(id=new_id('ev'), type='orders.' + state, time=now, + actor=actor, subject=order['buyer'], request_id=request_id, + resources=(), data={'order_id': order['id'], 'state': state})) + order['state'] = state + + +async def settle(app, tx, order, *, now, actor, request_id, reason, + refund_minor=0, decision=None): + """Validate authority *before* appending either leg of a balanced settlement. + Caller owns the transaction. Any failure rolls back both legs, the decision + consumption, transitions, receipts, events and request result together. + """ + contract(tx, order['id']) + require(tx.one('SELECT 1 FROM order_escrow_decisions WHERE order_id=?', (order['id'],)) is None, + 'order_already_settled') + total = order['total_price_minor'] + require(type(refund_minor) is int and 0 <= refund_minor <= total, + 'settlement_amount_invalid') + existing = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (order['id'],)) + if existing: + body = loads(existing[0]) + require(body['refund_minor'] == refund_minor and body['reason'] == reason and + body['decision_id'] == (decision['id'] if decision else None), + 'order_already_settled') + return body['receipts'] + if decision is not None: + from msg.market.arbitration import validate_decision + await validate_decision(app, tx, order, decision, now) + require(refund_minor == decision['refund_minor'], 'decision_amount_mismatch') + elif refund_minor == total: + require(reason in OBJECTIVE_REFUNDS and order['state'] in {'funded', 'delivered'}, + 'escrow_release_forbidden') + else: + require(refund_minor == 0 and order['state'] == 'accepted' and + reason in {'buyer_acceptance', 'managed_instant_verified'}, + 'escrow_release_forbidden') + require(_balance(tx, order['escrow_subject']) == total, 'escrow_balance_mismatch') + receipts = [] + for suffix, recipient, amount, kind in ( + ('refund', order['buyer'], refund_minor, 'refund'), + ('release', order['seller'], total-refund_minor, 'transfer'), + ): + if amount: + receipts.append(_post_escrow_transfer(tx, order=order, + recipient=recipient, amount=amount, actor=actor, + request_id=f'{order["id"]}:{suffix}', now=now, + receipt_signer=app.receipt_signer, reference=f'order_{suffix}:{order["id"]}', + kind=kind)) + final = 'refunded' if refund_minor == total else 'settled' + await transition(tx, order, final, now=now, actor=actor, request_id=request_id, reason=reason) + refs = order['receipt_refs'] + [r['body']['transaction_id'] for r in receipts] + tx.execute('UPDATE store_orders SET settled_at=?,receipt_refs=? WHERE id=?', + (wire(now), canonical(refs).decode(), order['id']), write=True) + statement = {'order_id': order['id'], 'refund_minor': refund_minor, + 'release_minor': total-refund_minor, 'reason': reason, + 'decision_id': decision['id'] if decision else None, + 'policy': order['dispute_policy'], 'receipts': receipts, 'at': wire(now), + 'order_facts': {name: order[name] for name in ('payment_transaction_id', + 'payment_intent_digest', 'funded_at', 'delivered_at')}, + 'delivery_snapshot': delivery_snapshot(tx, order['id'])} + tx.execute('INSERT INTO order_settlements(order_id,decision_id,body) VALUES (?,?,?)', + (order['id'], statement['decision_id'], canonical(statement).decode()), write=True) + if decision: + tx.execute("UPDATE arbitration_cases SET state='executed' WHERE id=? AND state='decided'", + (decision['case_id'],), write=True) + order.update(receipt_refs=refs, settled_at=wire(now)) + return receipts + + +async def resolve_due(app, *, limit=100): + """Bounded worker tick, never called by a GET or by the ClearingEngine to buy. + + Time starts at persisted creation/funding, not worker retries. Undelivered + services/manual ciphertext refund; subjective complaints and expired panels + remain held. This function has no external side effect. + """ + from msg.plugins.orders import _row + now = app.clock() + changed = [] + async with app.metadata.transaction(write=True) as tx: + rows = tx.rows('''SELECT o.id,o.buyer FROM store_orders o + JOIN order_deadlines d ON d.order_id=o.id + WHERE o.state IN ('created','funded') AND d.expires_at<=? + ORDER BY d.expires_at,o.id LIMIT ?''', (wire(now),limit)) + for order_id, buyer in rows: + order = _row(tx, order_id, buyer) + terms = contract(tx, order_id) + policy = terms['policy']['policy'] + origin, seconds = ((order['created_at'], policy['funding_timeout_seconds']) + if order['state'] == 'created' else + (order['funded_at'], policy['delivery_timeout_seconds'])) + if now < parse_time(origin) + timedelta(seconds=seconds): + continue + if order['state'] == 'created': + await transition(tx, order, 'cancelled', now=now, actor=buyer, + request_id='timeout:'+order_id, reason='funding_timeout') + else: + await settle(app, tx, order, now=now, actor=buyer, + request_id='timeout:'+order_id, reason='delivery_timeout', + refund_minor=order['total_price_minor']) + changed.append(order_id) + return changed + + +# Published checkout contracts retain their original signed policy and journals. POLICY = freeze_json({ 'escrow_policy': 'escrow-v1', 'dispute_policy': 'dispute-v1', 'version': 1, 'reasons': ['buyer_cancel', 'buyer_accept'], @@ -54,11 +200,15 @@ def __init__(self, app): async def settle(self, ctx, request, tx, *, reason, order_id=None): # Import handlers' read-only projections, never their write entry points. - from msg.plugins.orders import _row, _subject from msg.plugins.delivery import _delivery, _verified_delivery + from msg.plugins.orders import _row, _subject buyer=_subject(ctx) require(order_id is None or reason == 'checkout_accept', 'escrow_decision_mismatch') order=_row(tx,order_id or request.arguments['order_id'],buyer) + require(tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (order['id'],)) is None, + 'order_contract_requires_v3') + require(tx.one('SELECT 1 FROM order_settlements WHERE order_id=?', (order['id'],)) is None, + 'order_already_resolved') require(order['buyer']==buyer, 'order_not_found') policy=validate_policy(order['escrow_policy'],order['dispute_policy']) require(reason in policy['reasons'], 'escrow_reason_unsupported') @@ -122,7 +272,7 @@ async def settle(self, ctx, request, tx, *, reason, order_id=None): # under the same serialized write transaction as the debit and journal. validate_decision(decision,body,self.app.receipt_signer.public_key, self.app.executor.clock()) - receipt=_post_transfer(tx,sender=order['escrow_subject'],recipient=recipient, + receipt=_post_escrow_transfer(tx,order=order,recipient=recipient, amount=order['total_price_minor'],actor=buyer,request_id=ledger_request_id, now=ctx.now,receipt_signer=self.app.receipt_signer, reference='order_'+outcome+':'+order['id'], diff --git a/src/msg/market/orders.py b/src/msg/market/orders.py new file mode 100644 index 00000000..308c2d9f --- /dev/null +++ b/src/msg/market/orders.py @@ -0,0 +1,218 @@ +"""New checkout contracts without changing published orders.buy@1/@2 semantics.""" +from __future__ import annotations + +from datetime import timedelta + +from msg.core.codec import canonical, digest, parse_time, wire +from msg.core.errors import require +from msg.core.models import HandlerOutput +from msg.market.escrow import settle, transition +from msg.market.policy import contract, snapshot +from msg.market.targets import ( + enqueue_notification, notification_view, save_target, target_for, validate_target, +) +from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _post_transfer +from msg.plugins.orders import _order_id, _row, _subject, _view, _viewer +from msg.plugins.schemas import IDENTIFIER, obj +from msg.plugins.store import _body, _listing, _package_row + +HASH = {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'} +ORDER = obj({'order_id': IDENTIFIER}, ('order_id',)) +INTENT = { + 'listing_id': IDENTIFIER, 'listing_revision': IDENTIFIER, + 'quantity': {'type': 'integer', 'minimum': 1, 'maximum': 10**9}, + 'currency_id': {'const': CURRENCY_ID}, + 'total_price_minor': {'type': 'integer', 'minimum': 1, 'maximum': MAX_MINOR}, + 'email': {'type': 'string', 'minLength': 3, 'maxLength': 254}, +} +INTENT_REQUIRED = ('listing_id','listing_revision','quantity','currency_id','total_price_minor') + + +def view(tx, order, viewer): + email = notification_view(tx, order) + projected = {**order, 'delivery_target': {**order['delivery_target'], 'email': email}} + result = _view(projected, viewer) + row = tx.one('SELECT digest FROM order_contracts WHERE order_id=?', (order['id'],)) + if row: + result['order_digest'] = row[0] + result['contract_version'] = 3 + locked = contract(tx, order['id']) + result['delivery_mode'] = locked['listing']['delivery_mode'] + result['policy_digest'] = locked['policy']['policy_digest'] + # Seller needs the *public encryption subkey*, not buyer's email. + if locked['recipient_key']: + result['recipient_key'] = locked['recipient_key'] + result['email_status'] = email['state'] + return result + + +async def create(app, tx, ctx, request): + buyer, args = _subject(ctx), request.arguments + listing = await _listing(app, ctx, request, tx, args['listing_id']) + body, revision = await _body(app, tx, listing) + require(body['mode'] == 'sale' and body['state'] == 'active' and + (body['expires_at'] is None or parse_time(body['expires_at']) > ctx.now), + 'listing_not_available') + require(args['listing_revision'] == listing.revision, 'listing_revision_conflict') + require(body['escrow_policy'] == 'escrow-v1', 'escrow_policy_unknown') + policy = snapshot(tx, body['dispute_policy']) + require(body['currency_id'] == args['currency_id'] == CURRENCY_ID, 'unsupported_currency') + total = body['price_minor'] * args['quantity'] + require(0 < total <= MAX_MINOR, 'money_overflow') + require(total == args['total_price_minor'], 'price_changed') + sold = tx.one('''SELECT COALESCE(SUM(quantity),0) FROM store_orders + WHERE listing_id=? AND state NOT IN ('cancelled','refunded')''', (listing.id,))[0] + require(sold + args['quantity'] <= body['quantity'], 'quantity_unavailable') + require(listing.owner != buyer, 'self_purchase_forbidden') + require(body['item_kind'] != 'secret' or body['delivery_mode'] == 'sealed_manual', + 'secret_requires_buyer_encryption') + package_id = package_revision = package_digest = None + if body['delivery_mode'] == 'managed_instant': + require(body['item_kind'] in {'file','bundle','text'}, 'order_delivery_mode_unsupported') + package = await _package_row(tx, body['package_ref']) + require(package is not None and package[1] == listing.id and package[3] == listing.owner and + package[5] == body['item_kind'] and package[10] == 'managed_instant', + 'package_not_available') + package_id, package_revision, package_digest = package[0], package[4], package[8] + recipient_key = None + if body['delivery_mode'] == 'sealed_manual': + row = tx.one('''SELECT key_id,recipient,public_key FROM encryption_subkeys + WHERE subject=? AND is_primary=1 AND retired_at IS NULL''', (buyer,)) + require(row is not None, 'delivery_recipient_key_required') + recipient_key = {'key_id': row[0], 'recipient': row[1], 'fingerprint': digest(row[2])} + target = await target_for(app, tx, buyer, args.get('email')) + order_id, now = _order_id(), wire(ctx.now) + escrow = 'esc_' + order_id[4:] + tx.execute("INSERT INTO ledger_accounts(id,kind,subject_id,source_id) VALUES (?,'order_escrow',NULL,?)", + (escrow, order_id), write=True) + tx.execute('''INSERT INTO store_orders + (id,buyer,seller,listing_id,listing_revision,package_id,package_revision,package_digest, + quantity,unit_price_minor,total_price_minor,currency_id,escrow_subject,escrow_policy, + dispute_policy,terms_digest,delivery_target,payment_intent_digest,payment_transaction_id, + state,created_at,funded_at,receipt_refs) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL,'created',?,NULL,'[]')''', + (order_id,buyer,listing.owner,listing.id,listing.revision,package_id,package_revision, + package_digest,args['quantity'],body['price_minor'],total,CURRENCY_ID,escrow, + body['escrow_policy'],body['dispute_policy'],digest(body['terms']),canonical(target).decode(), + request.payload_digest,now), write=True) + locked = {'order_id': order_id, 'version': 3, 'buyer': buyer, 'seller': listing.owner, + 'listing': body, 'listing_id': listing.id, 'escrow_subject': escrow, + 'listing_revision': listing.revision, + 'listing_digest': revision.content.digest, 'package_id': package_id, + 'package_revision': package_revision, 'package_digest': package_digest, + 'quantity': args['quantity'], 'total_price_minor': total, + 'terms_digest': digest(body['terms']), 'policy': policy, + 'recipient_key': recipient_key, 'handle_snapshot': target['handle_snapshot'], + 'buyer_principal': wire(ctx.principal), 'created_at': now} + tx.execute('INSERT INTO order_contracts(order_id,body,digest) VALUES (?,?,?)', + (order_id,canonical(locked).decode(),digest(locked)), write=True) + tx.execute('INSERT INTO order_deadlines(order_id,expires_at) VALUES (?,?)', + (order_id, wire(ctx.now+timedelta(seconds=policy['policy']['funding_timeout_seconds']))), write=True) + from msg.market.email import verification_message + order = _row(tx, order_id, buyer) + await verification_message(app, tx, ctx, request, order) + return order + + +async def fund(app, tx, ctx, request, order): + require(order['state'] == 'created', 'order_not_fundable') + locked = contract(tx, order['id']) + require(ctx.now < parse_time(order['created_at']) + timedelta( + seconds=locked['policy']['policy']['funding_timeout_seconds']), 'payment_intent_expired') + validate_target(tx, order) + receipt = _post_transfer(tx, sender=order['buyer'], recipient=order['escrow_subject'], + amount=order['total_price_minor'], actor=order['buyer'], request_id=request.request_id, + now=ctx.now, receipt_signer=app.receipt_signer, reference='order_fund:'+order['id']) + tx.execute('''UPDATE store_orders SET payment_transaction_id=?,payment_intent_digest=?, + funded_at=?,receipt_refs=? WHERE id=?''', (receipt['body']['transaction_id'], + request.payload_digest,wire(ctx.now),canonical([receipt['body']['transaction_id']]).decode(), + order['id']), write=True) + order.update(payment_transaction_id=receipt['body']['transaction_id'], + payment_intent_digest=request.payload_digest, funded_at=wire(ctx.now), + receipt_refs=[receipt['body']['transaction_id']]) + await transition(tx, order, 'funded', now=ctx.now, actor=order['buyer'], + request_id=request.request_id, reason='signed_payment_intent') + tx.execute('UPDATE order_deadlines SET expires_at=? WHERE order_id=?', + (wire(ctx.now+timedelta(seconds=locked['policy']['policy']['delivery_timeout_seconds'])), order['id']), write=True) + if locked['listing']['delivery_mode'] == 'managed_instant': + from msg.market.delivery import automatic + await automatic(app, tx, ctx, request, order) + return receipt + + +def install(app, op): + from msg.market.email import install as install_email + install_email(app, op) + + @op('orders.create', obj(INTENT, INTENT_REQUIRED), signature=True) + async def reserve(ctx, request, tx): + order = await create(app, tx, ctx, request) + return HandlerOutput(data={'order': view(tx, order, order['buyer'])}) + + @op('orders.buy', obj(INTENT, INTENT_REQUIRED), signature=True, version=3) + async def buy(ctx, request, tx): + order = await create(app, tx, ctx, request) + receipt = await fund(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], order['buyer']), order['buyer']), + 'payment': receipt}) + + @op('orders.fund', obj({'order_id': IDENTIFIER, 'order_digest': HASH, + 'total_price_minor': INTENT['total_price_minor'], 'currency_id': {'const': CURRENCY_ID}}, + ('order_id','order_digest','total_price_minor','currency_id')), signature=True) + async def pay(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + locked = contract(tx, order['id']) + require(digest(locked) == request.arguments['order_digest'] and + order['total_price_minor'] == request.arguments['total_price_minor'], + 'payment_intent_mismatch') + payment = await fund(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], buyer), buyer), + 'payment': payment}) + + @op('orders.cancel', ORDER, signature=True, version=2) + async def cancel(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + if order['state'] == 'created': + await transition(tx, order, 'cancelled', now=ctx.now, actor=buyer, + request_id=request.request_id, reason='buyer_cancelled') + elif order['state'] not in {'cancelled', 'refunded'}: + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_cancellable') + await settle(app, tx, order, now=ctx.now, actor=buyer, request_id=request.request_id, + reason='buyer_cancelled', refund_minor=order['total_price_minor']) + return HandlerOutput(data={'order': view(tx, order, buyer)}) + + @op('orders.email_bind', obj({'order_id': IDENTIFIER, 'email': INTENT['email']}, + ('order_id','email')), signature=True) + async def email_bind(ctx, request, tx): + buyer = _subject(ctx) + order = _row(tx, request.arguments['order_id'], buyer) + require(order['buyer'] == buyer, 'order_not_found') + contract(tx, order['id']) + old = order['delivery_target'].get('email', {}) + require(order['state'] in {'created','funded','delivered','settled'}, 'order_closed') + require(order['delivered_at'] is None or + (old.get('endpoint_id') is None and + old.get('address_snapshot') == request.arguments['email']), + 'delivery_target_locked') + target = await target_for(app, tx, buyer, request.arguments['email']) + target['handle_snapshot'] = order['delivery_target']['handle_snapshot'] + require(target['email'].get('endpoint_id') is not None, 'email_not_verified') + save_target(tx, order, target) + validate_target(tx, order, email=True) + if order['delivered_at']: + await enqueue_notification(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, order, buyer)}) + + @op('orders.contract', ORDER, effect='read') + async def get_contract(ctx, request, tx): + viewer = _viewer(ctx) + order = _row(tx, request.arguments['order_id'], viewer) + locked = contract(tx, order['id']) + # Neither endpoint nor credential ceilings are part of seller projections. + return HandlerOutput(data={'order': view(tx, order, viewer), 'contract': { + k:v for k,v in locked.items() if k not in {'buyer_principal','handle_snapshot'}}}) diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py new file mode 100644 index 00000000..434ab6e4 --- /dev/null +++ b/src/msg/market/policy.py @@ -0,0 +1,132 @@ +"""Small, immutable policies. No expressions, callbacks or model discretion.""" +from __future__ import annotations + +from msg.core.codec import canonical, digest, loads +from msg.core.errors import require + +DEFAULT_POLICY = { + 'id': 'dispute-v1', 'version': 1, + 'funding_timeout_seconds': 900, 'delivery_timeout_seconds': 86400, + 'case_timeout_seconds': 604800, 'decision_lifetime_seconds': 86400, + 'panel_size': 3, 'quorum': 2, 'candidates': [], + 'allow_appeal': False, 'appeal_seconds': 3600, + 'selection': 'sha256-order-policy-round-subject-v1', + 'member_invalidation': 'hold-no-replacement-v1', + 'rules': 'objective-faults-v1', +} + + +def validate(policy): + require(isinstance(policy, dict) and set(policy) == set(DEFAULT_POLICY), + 'arbitration_policy_invalid') + require(isinstance(policy['id'], str) and 1 <= len(policy['id']) <= 80 and + type(policy['version']) is int and policy['version'] == 1 and policy['rules'] == 'objective-faults-v1' and + policy['selection'] == DEFAULT_POLICY['selection'] and + policy['member_invalidation'] == DEFAULT_POLICY['member_invalidation'], + 'arbitration_policy_invalid') + for name, maximum in (('funding_timeout_seconds', 86400), + ('delivery_timeout_seconds', 2592000), + ('case_timeout_seconds', 2592000), + ('decision_lifetime_seconds', 604800), + ('appeal_seconds', 86400)): + require(type(policy[name]) is int and 60 <= policy[name] <= maximum, + 'arbitration_policy_invalid', name) + size, quorum, candidates = policy['panel_size'], policy['quorum'], policy['candidates'] + require(type(size) is int and 1 <= size <= 9 and type(quorum) is int and + size // 2 < quorum <= size and type(policy['allow_appeal']) is bool and + isinstance(candidates, list) and len(candidates) <= 100 and + all(isinstance(s, str) and s.startswith('u_') and s != 'u_root' + for s in candidates) and len(candidates) == len(set(candidates)), + 'arbitration_policy_invalid') + require(not policy['allow_appeal'] or + policy['decision_lifetime_seconds'] > policy['appeal_seconds'], + 'arbitration_policy_invalid') + return policy + + +def load_policy(tx, policy_id): + row = tx.one('SELECT body,digest FROM arbitration_policies WHERE id=?', (policy_id,)) + if row: + policy = loads(row[0]) + require(digest(policy) == row[1], 'arbitration_policy_corrupt') + else: + require(policy_id == DEFAULT_POLICY['id'], 'arbitration_policy_unknown') + policy = loads(canonical(DEFAULT_POLICY)) + return validate(policy) + + +def snapshot(tx, policy_id): + policy = load_policy(tx, policy_id) + # Freeze the grant epoch, not merely the subject: revoke+regrant cannot revive + # a vote or silently replace a member in an existing order. + epochs = {} + for candidate in policy['candidates']: + row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (candidate,)) + if row and row[1]: + epochs[candidate] = row[0] + return {'policy': policy, 'policy_digest': digest(policy), 'candidate_epochs': epochs} + + +def contract(tx, order_id): + row = tx.one('SELECT body,digest FROM order_contracts WHERE order_id=?', (order_id,)) + require(row is not None, 'order_contract_required') + body = loads(row[0]) + require(digest(body) == row[1] and body['order_id'] == order_id, + 'order_contract_corrupt') + validate_projection(tx, body) + return body + + +def delivery_snapshot(tx, order_id): + """Commit delivery identity and prepared bytes, excluding subsequent buyer ACK.""" + from msg.plugins.delivery import _delivery + delivery = _delivery(tx, order_id) + return digest(None if delivery is None else { + key: value for key, value in delivery.items() + if key not in {'state', 'claimed_at', 'receipt'}}) + + +def validate_projection(tx, locked): + """A restored mutable projection cannot replace the immutable checkout facts.""" + require(locked.get('version') == 3, 'order_contract_version') + listing = locked['listing'] + expected = {name: locked[name] for name in ( + 'buyer', 'seller', 'listing_id', 'listing_revision', 'package_id', + 'package_revision', 'package_digest', 'quantity', 'total_price_minor', + 'escrow_subject', 'terms_digest', 'created_at')} + expected.update(unit_price_minor=listing['price_minor'], + currency_id=listing['currency_id'], escrow_policy=listing['escrow_policy'], + dispute_policy=listing['dispute_policy']) + mutable = ('state', 'receipt_refs', 'settled_at', 'payment_transaction_id', + 'payment_intent_digest', 'funded_at', 'delivered_at') + columns = (*expected, *mutable) + row = tx.one('SELECT ' + ','.join(columns) + ' FROM store_orders WHERE id=?', + (locked['order_id'],)) + require(row is not None, 'order_contract_mismatch') + order = dict(zip(columns, row)) + require(all(order[name] == value for name, value in expected.items()), + 'order_contract_mismatch') + if order['payment_transaction_id'] is not None: + payment = tx.one('''SELECT debit_account,credit_account,amount_minor,currency_id, + reference,committed_at FROM money_ledger WHERE id=?''', + (order['payment_transaction_id'],)) + require(payment == (locked['buyer'], locked['escrow_subject'], + locked['total_price_minor'], listing['currency_id'], + 'order_fund:' + locked['order_id'], order['funded_at']), 'order_payment_mismatch') + fact = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (locked['order_id'],)) + if fact is None: + require(order['state'] not in {'settled', 'refunded'}, 'order_settlement_missing') + return + fact = loads(fact[0]) + receipts = fact['receipts'] + references = [order['payment_transaction_id']] + [r['body']['transaction_id'] for r in receipts] + require(fact['refund_minor'] + fact['release_minor'] == locked['total_price_minor'] and + order['state'] == ('refunded' if fact['release_minor'] == 0 else 'settled') and + order['settled_at'] == fact['at'] and loads(order['receipt_refs']) == references and + all(order[name] == value for name, value in fact['order_facts'].items()) and + delivery_snapshot(tx, locked['order_id']) == fact['delivery_snapshot'], + 'order_settlement_mismatch') + for receipt in receipts: + row = tx.one('SELECT receipt FROM money_ledger WHERE id=?', + (receipt['body']['transaction_id'],)) + require(row is not None and loads(row[0]) == receipt, 'order_settlement_mismatch') diff --git a/src/msg/market/rationale.py b/src/msg/market/rationale.py new file mode 100644 index 00000000..291da1d2 --- /dev/null +++ b/src/msg/market/rationale.py @@ -0,0 +1,91 @@ +"""Pinned, case-private reasons are part of a signed arbitration allocation. + +This is a reference to an ordinary immutable File revision, not executable policy. +The case grants parties/panel access to that one blob, never the author's folder. +""" +from __future__ import annotations + +from msg.core.codec import canonical, decode, digest, loads, parse_time, wire +from msg.core.errors import Failure, require +from msg.core.models import BlobRef, HandlerOutput, ResourceRef +from msg.market.delivery import verify_blob +from msg.plugins.common import check_access +from msg.plugins.orders import _subject +from msg.plugins.schemas import IDENTIFIER, obj + +PINNED_REF = obj({'id': IDENTIFIER, 'revision': IDENTIFIER}, ('id', 'revision')) +MAX_REASON_BYTES = 65536 + + +def evidence_id(case, ref): + return 'rationale_' + digest({'case_id': case['id'], 'round': case['round'], + 'ref': wire(ref)})[7:] + + +def record(tx, case, proposal): + """Check immutable bindings without reinterpreting an old grant or policy.""" + ref = proposal['rationale_ref'] + row = tx.one('SELECT author,visibility,body FROM arbitration_evidence WHERE id=? AND case_id=?', + (evidence_id(case, ref), case['id'])) + require(row is not None, 'decision_rationale_missing') + body = loads(row[2]) + require(row[0] in case['panels'][str(case['round'])] and row[1] == 'parties' and + body['kind'] == 'rationale' and body['round'] == case['round'] and + body['source'] == ref and body['blob']['digest'] == proposal['rationale_digest'], + 'decision_rationale_mismatch') + return body + + +async def verified(app, tx, case, proposal): + body = record(tx, case, proposal) + try: + await verify_blob(app, decode(BlobRef, body['blob'])) + except Failure as exc: + if exc.code == 'package_missing': + raise Failure('decision_rationale_missing') from None + if exc.code == 'package_digest_mismatch': + raise Failure('decision_rationale_mismatch') from None + raise + + +def install(app, op): + @op('orders.dispute_rationale', obj({'case_id': IDENTIFIER, 'ref': PINNED_REF}, + ('case_id', 'ref')), signature=True) + async def deposit(ctx, request, tx): + from msg.market.arbitration import owned_case, panel_valid + actor = _subject(ctx) + case = await owned_case(tx, ctx, request.arguments['case_id']) + require(actor in case['panels'][str(case['round'])] and await panel_valid(tx, case), + 'arbitrator_required') + require(case['state'] == 'open' and + ctx.now < parse_time(case['round_deadlines'][str(case['round'])]), + 'case_evidence_closed') + ref = decode(ResourceRef, request.arguments['ref']) + resource = await tx.resource(ref.id) + require(resource.owner == actor and resource.type in {'file', 'attachment'} and + resource.state == 'active', 'rationale_not_owned') + await check_access(app, ctx, request, tx, ref.id, 'read') + blob = (await tx.revision(ref)).content + require(0 < blob.size <= MAX_REASON_BYTES and + blob.media_type in {'text/plain', 'text/markdown'}, 'rationale_text_required') + await verify_blob(app, blob) + raw = await app.contents.read_bytes(blob, limit=MAX_REASON_BYTES) + try: + require(bool(raw.decode('utf-8').strip()), 'rationale_text_required') + except UnicodeDecodeError: + raise Failure('rationale_text_required') from None + eid = evidence_id(case, wire(ref)) + previous = tx.one('SELECT body FROM arbitration_evidence WHERE id=?', (eid,)) + if previous is None: + require(tx.one('SELECT COUNT(*) FROM arbitration_evidence WHERE case_id=? AND author=?', + (case['id'], actor))[0] < 64, 'case_evidence_limit') + await app.contents.pin(blob, eid) + body = {'kind': 'rationale', 'blob': wire(blob), 'source': wire(ref), + 'round': case['round'], 'at': wire(ctx.now), + 'author_request_digest': request.payload_digest} + tx.execute('INSERT INTO arbitration_evidence(id,case_id,author,visibility,body) ' + "VALUES (?,?,?,'parties',?)", + (eid, case['id'], actor, canonical(body).decode()), write=True) + binding = {'rationale_ref': wire(ref), 'rationale_digest': blob.digest} + record(tx, case, binding) + return HandlerOutput(data={**binding, 'evidence_id': eid}) diff --git a/src/msg/market/references.py b/src/msg/market/references.py new file mode 100644 index 00000000..8980f222 --- /dev/null +++ b/src/msg/market/references.py @@ -0,0 +1,19 @@ +"""SQL-authoritative market content roots, shared by backup and collection. + +Order IDs, digests and pins never confer read access. Pins protect a blob before +SQL commits; this inventory protects it after the originating file is purged. +""" +from msg.core.codec import loads + + +def content_references(execute): + for table, column in (('store_packages', 'payload_refs'), + ('store_deliveries', 'payload_refs')): + for (raw,) in execute(f'SELECT {column} FROM {table} ORDER BY id'): + for ref in loads(raw): + yield ref['blob'] + # Legacy archives can be inspected before the new schema is installed. + exists = next(iter(execute("SELECT to_regclass('arbitration_evidence')")))[0] + if exists is not None: + for (raw,) in execute('SELECT body FROM arbitration_evidence ORDER BY id'): + yield loads(raw)['blob'] diff --git a/src/msg/market/targets.py b/src/msg/market/targets.py new file mode 100644 index 00000000..1833d0aa --- /dev/null +++ b/src/msg/market/targets.py @@ -0,0 +1,157 @@ +"""Buyer-owned endpoint binding shared by checkout, delivery and SMTP worker.""" +from __future__ import annotations + +from dataclasses import replace + +from msg.core.codec import canonical, decode, digest, loads +from msg.core.errors import Failure, require +from msg.core.models import EffectJob +from msg.market.delivery_targets import account_email, mail_enabled, pickup_url, validate_address +from msg.market.policy import contract +from msg.plugins.common import new_id + + +def _email(tx, subject): + generation, settings = account_email(tx, subject) + if settings is None: + return None, None + endpoint = 'em_' + digest((subject, settings.address, settings.verified_at, generation))[7:39] + return settings, endpoint + + +async def target_for(app, tx, buyer, email_address=None): + handle = (await tx.resource(buyer)).name + target = {'subject_id': buyer, 'channel': 'site', 'handle_snapshot': handle, + 'email': {'state': 'disabled'}} + if email_address is None: + return target + validate_address(email_address) + email, endpoint = _email(tx, buyer) + verified = (email is not None and email.subject_id == buyer and + email.address == email_address and email.verified_at is not None) + target['email'] = {'state': 'pending' if mail_enabled(app) else 'disabled', + 'subject_id': buyer, 'address_snapshot': email_address, + 'endpoint_id': endpoint if verified else None} + # Verification is a separate, order-information-free external effect. + return target + + +def save_target(tx, order, target): + tx.execute('UPDATE store_orders SET delivery_target=? WHERE id=?', + (canonical(target).decode(), order['id']), write=True) + order['delivery_target'] = target + + +def validate_target(tx, order, *, email=False, encryption=False): + target = order['delivery_target'] + require(target.get('subject_id') == order['buyer'] and target.get('channel') == 'site', + 'delivery_recipient_mismatch') + if encryption: + locked = contract(tx, order['id']).get('recipient_key') + row = tx.one('SELECT subject,recipient,retired_at FROM encryption_subkeys WHERE key_id=?', + (locked['key_id'],)) if locked else None + require(row is not None and row[0] == order['buyer'] and row[1] == locked['recipient'] + and row[2] is None, 'delivery_recipient_mismatch') + if email: + chosen = target.get('email', {}) + require(not chosen.get('notification_disabled'), 'delivery_recipient_mismatch') + endpoint_id = chosen.get('endpoint_id') or '' + if endpoint_id.startswith('dep_'): + row = tx.one('SELECT subject,address,active FROM delivery_email_endpoints WHERE id=?', + (endpoint_id,)) + require(row is not None and row[2] and + row[0] == chosen.get('subject_id') == order['buyer'] and + row[1] == chosen.get('address_snapshot'), 'delivery_recipient_mismatch') + validate_address(row[1]) + return row[1] + current, endpoint = _email(tx, order['buyer']) + require(chosen.get('subject_id') == order['buyer'] and current is not None and + current.subject_id == order['buyer'] and current.verified_at is not None and + chosen.get('endpoint_id') == endpoint and + chosen.get('address_snapshot') == current.address, + 'delivery_recipient_mismatch') + validate_address(current.address) + return current.address + return None + + +async def enqueue_notification(app, tx, ctx, request, order): + chosen = order['delivery_target'].get('email', {}) + if (not mail_enabled(app) or not chosen.get('endpoint_id') or + chosen.get('notification_disabled')): + return + try: + validate_target(tx, order, email=True) + except Failure as exc: + if exc.code != 'delivery_recipient_mismatch': + raise + save_target(tx, order, {**order['delivery_target'], 'email': { + **chosen, 'state': 'pending', 'error': exc.code}}) + return + # Only the buyer can bind/queue the auxiliary channel, even when the seller + # uploads ciphertext later. Checkout's signed principal is persisted privately. + locked = contract(tx, order['id']) + from msg.core.models import Principal + principal = decode(Principal, locked['buyer_principal']) + dedupe = 'market-mail:'+order['id']+':'+chosen['endpoint_id'] + if tx.one('SELECT 1 FROM jobs WHERE dedupe=?', (dedupe,)): + return + job = EffectJob(id=new_id('job'), event_id='delivery:'+order['id'], kind='market_mail', + dedupe_key=dedupe, principal=principal, + operation='orders.buy', arguments={'order_id': order['id'], 'endpoint_id': chosen['endpoint_id']}, + state='pending', attempts=0, next_attempt_at=ctx.now, lease_until=None) + await tx.enqueue(job) + target = {**order['delivery_target'], 'email': {**chosen, 'state': 'queued'}} + save_target(tx, order, target) + + +async def render_notification(app, tx, job): + from msg.plugins.delivery import _delivery + from msg.plugins.orders import _row + order = _row(tx, job.arguments['order_id'], job.principal.subject) + require(order['buyer'] == job.principal.subject and job.principal.actor == order['buyer'], + 'delivery_recipient_mismatch') + delivery = _delivery(tx, order['id']) + require(order['state'] in {'delivered','accepted','settled'} and delivery is not None and + delivery['recipient_subject'] == order['buyer'], 'delivery_not_available') + require(order['delivery_target'].get('email', {}).get('endpoint_id') == job.arguments['endpoint_id'], + 'delivery_recipient_mismatch') + locked = contract(tx, order['id']) + address = validate_target(tx, order, email=True, + encryption=locked['listing']['delivery_mode'] == 'sealed_manual') + # A stable site URL is not a capability. GET always requires fresh authentication. + link = pickup_url(app, order['id']) + body = {'order_id': order['id'], 'handle': locked['handle_snapshot'], 'pickup': link} + return replace(job, kind='mail', arguments={'recipient': address, + 'recipient_subject': order['buyer'], 'subject': 'msg order notification', + 'text': canonical(body).decode()}) + + +def notification_view(tx, order): + """Project current job facts; never persist SMTP state into the order.""" + chosen = dict(order['delivery_target'].get('email', {'state': 'disabled'})) + if chosen.get('notification_disabled'): + return {**chosen, 'state': 'disabled'} + endpoint = chosen.get('endpoint_id') + if not endpoint: + return chosen + dedupe = 'market-mail:' + order['id'] + ':' + endpoint + row = tx.one('SELECT body FROM jobs WHERE dedupe=?', (dedupe,)) + chosen.pop('last_result', None) + if row is None: + # Old cached state is not evidence that an external delivery completed. + return {**chosen, 'state': 'disabled' if chosen.get('state') == 'disabled' else 'pending'} + job = decode(EffectJob, loads(row[0])) + require(job.kind == 'market_mail' and job.dedupe_key == dedupe and + job.principal.actor == job.principal.subject == order['buyer'] and + job.arguments.get('order_id') == order['id'] and + job.arguments.get('endpoint_id') == endpoint, + 'delivery_recipient_mismatch') + result = tx.setting('job_status:' + job.id, {}) + code = result.get('code') + state = ('disabled' if code == 'mail_disabled' else + 'smtp_accepted' if job.state == 'done' and code == 'sent' else + 'delivered_unknown' if job.state in {'done', 'uncertain'} else + 'failed' if job.state == 'failed' else + 'pending' if code is not None else 'queued') + return {**chosen, 'state': state, **({'last_result': code} if code is not None else {})} diff --git a/src/msg/plugins/delivery.py b/src/msg/plugins/delivery.py index 00226345..3658777d 100644 --- a/src/msg/plugins/delivery.py +++ b/src/msg/plugins/delivery.py @@ -159,6 +159,9 @@ async def get(ctx, request, tx): require(buyer is not None and ctx.principal.actor == buyer, 'order_not_found') order = _buyer_order(tx, request.arguments['order_id'], buyer) + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (order['id'],)): + from msg.market.delivery import read + return await read(app, tx, ctx, request) delivery = _delivery(tx, order['id']) require(delivery is not None, 'delivery_not_found') await _verified_delivery(app, tx, order, delivery) @@ -177,6 +180,9 @@ async def get(ctx, request, tx): 'delivery_digest': {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'}}, ('order_id', 'delivery_digest')), signature=True) async def accept(ctx, request, tx): + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (request.arguments['order_id'],)): + from msg.market.delivery import accept as market_accept + return await market_accept(app, tx, ctx, request) receipt,decision,delivery = await EscrowEngine(app).settle( ctx,request,tx,reason='buyer_accept') return HandlerOutput(data={'order_id':request.arguments['order_id'],'state':'settled', @@ -214,4 +220,6 @@ async def notify(ctx, request, tx): enabled=request.arguments.get('enabled', True)) return HandlerOutput(data={'order_id': order['id'], 'notification': status}) + from msg.market.delivery import install as install_market + install_market(app, op) finish() diff --git a/src/msg/plugins/money.py b/src/msg/plugins/money.py index 059d16df..b589e881 100644 --- a/src/msg/plugins/money.py +++ b/src/msg/plugins/money.py @@ -51,10 +51,14 @@ def _supply(tx): def _post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer'): + receipt_signer, reference=None, kind='transfer', escrow_authority=None): """Called only within the executor's serialized write transaction.""" require(kind in {'transfer', 'refund'}, 'invalid_money_kind') amount = _amount(amount) + account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) + if account and account[0] == 'order_escrow': + from msg.market.escrow import _ESCROW_WRITE + require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') require(sender != recipient, 'self_transfer_forbidden') require(sender not in {ROOT_SUBJECT,'@root'}, 'root_local_only') require(_balance(tx, sender) >= amount, 'insufficient_funds') diff --git a/src/msg/plugins/orders.py b/src/msg/plugins/orders.py index d635fa1a..3c4d0594 100644 --- a/src/msg/plugins/orders.py +++ b/src/msg/plugins/orders.py @@ -209,8 +209,8 @@ async def cancel(ctx, request, tx): @op('orders.get', obj({'order_id': IDENTIFIER}, ('order_id',)), effect='read') async def get(ctx, request, tx): viewer = _viewer(ctx) - return HandlerOutput(data={'order': _view(_row(tx, request.arguments['order_id'], - viewer), viewer)}) + from msg.market.orders import view + return HandlerOutput(data={'order': view(tx, _row(tx, request.arguments['order_id'], viewer), viewer)}) @op('orders.list', obj({'role': {'enum': ['buy', 'sell']}, 'status': {'enum': ['open', 'completed', 'disputed']}, @@ -227,7 +227,7 @@ async def list_orders(ctx, request, tx): where = ('buyer=?' if role == 'buy' else 'seller=?') values = [viewer] if status: - where += (' AND state IN (\'funded\',\'delivered\',\'accepted\')' + where += (' AND state IN (\'created\',\'funded\',\'delivered\',\'accepted\')' if status == 'open' else ' AND state IN (\'settled\',\'cancelled\',\'refunded\')' if status == 'completed' else @@ -243,8 +243,11 @@ async def list_orders(ctx, request, tx): async def payment(ctx, request, tx): viewer = _viewer(ctx) row = _row(tx, request.arguments['order_id'], viewer) + if tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (row['id'],)): + from msg.market.policy import contract + contract(tx, row['id']) result = {'order_id': row['id'], - 'status': 'refunded' if row['state'] == 'refunded' else 'funded', + 'status': _view(row, viewer)['payment_status'], 'amount_minor': row['total_price_minor'], 'currency_id': row['currency_id']} if viewer == row['buyer']: @@ -254,9 +257,13 @@ async def payment(ctx, request, tx): (transaction_id,)) require(ledger is not None, 'order_not_found') receipts.append(loads(ledger[0])) - result['receipt'] = receipts[0] + result['receipt'] = receipts[0] if receipts else None result['receipts'] = receipts result['escrow_balance_minor'] = _balance(tx, row['escrow_subject']) return HandlerOutput(data={'payment': result}) + from msg.market.orders import install as install_market + install_market(app, op) + from msg.market.arbitration import install as install_arbitration + install_arbitration(app, op) finish() diff --git a/src/msg/plugins/store.py b/src/msg/plugins/store.py index 9113ae9c..2aaacbbb 100644 --- a/src/msg/plugins/store.py +++ b/src/msg/plugins/store.py @@ -190,7 +190,7 @@ async def package_deposit(ctx, request, tx): require(resource.revision == request.arguments['listing_revision'], 'listing_revision_conflict') require(body['delivery_mode'] == 'managed_instant' and - body['item_kind'] in {'file','bundle'} and body['state'] != 'closed', + body['item_kind'] in {'file','bundle','text'} and body['state'] != 'closed', 'package_mode_unsupported') manifest = request.arguments['manifest'] require(len(canonical(manifest)) <= MAX_MANIFEST_BYTES, 'manifest_too_large') @@ -206,6 +206,8 @@ async def package_deposit(ctx, request, tx): require(await direct_ancestor(tx, rid) is None, 'payload_private_conversation') await check_access(app, ctx, request, tx, rid, 'read') revision=await tx.revision(ResourceRef(id=rid,revision=ref['revision'])) + if body['item_kind'] == 'text': + require(revision.content.media_type.startswith('text/'), 'text_payload_required') blobs.append(revision.content) total+=revision.content.size require(total <= 1024**4, 'package_too_large') diff --git a/src/msg/security/authentication.py b/src/msg/security/authentication.py index 1cbfe7b5..3c2281ee 100644 --- a/src/msg/security/authentication.py +++ b/src/msg/security/authentication.py @@ -10,6 +10,7 @@ from msg.core.models import Principal,SignatureProof,TokenProof from msg.core.requests import payload_fields,signing_bytes from msg.security.crypto import key_id,subject_id,verify +from msg.security.quarantine import active as quarantine_active CUSTODIAL_SIGNED_WRITES=frozenset({ @@ -30,6 +31,9 @@ async def authenticate(self,request,session, *, entry): require(session.setting('active_root_certificate',self.certificates.root_certificate.resource_id)== self.certificates.root_certificate.resource_id,'service_restart_required') spec=self.registry.operation(request.operation,request.contract_version) + if quarantine_active(session): + require(spec.effect=='read','writes_paused') + raise Failure('recovery_quarantined') require(request.protocol_version==1 and request.target_service==self.service,'wrong_service') require(request.payload_digest==digest(payload_fields(request)),'payload_digest_mismatch') now=self.clock() diff --git a/src/msg/security/authorization.py b/src/msg/security/authorization.py index 8a4f48d9..dcfb104b 100644 --- a/src/msg/security/authorization.py +++ b/src/msg/security/authorization.py @@ -5,6 +5,7 @@ from msg.core.codec import loads,parse_time,wire from msg.core.models import ResourceRef from msg.security.sharing_policy import share_target_error +from msg.security.quarantine import active as quarantine_active, require_live_authority from msg.security.policy import allows,grant_covers,scope_contains,CERTGATE _WRITE_CHECKS={'write','create','remove','chmod','chgrp','chown','manage','certgate','purge','tool_use'} @@ -18,6 +19,7 @@ def __init__(self,registry,certificates): self.registry,self.certificates=registry,certificates async def grants(self,principal,session): + require_live_authority(session) grants=[] for cid in principal.certificates: certificate=await self.certificates.validate(cid,session) @@ -35,6 +37,7 @@ async def has(self,principal,capability,operation,resource,session): for g in await self.grants(principal,session)]) async def _ceiling(self,principal,operation,resource,session): + require_live_authority(session) if principal.method=='anonymous': return allowed=any([operation in g.operations and await scope_contains(g.scope,resource,session) @@ -142,6 +145,8 @@ async def share_link_read_allowed(self,grantor,credential_id,resource,chain,now, The original signed owner's credential is a revocation boundary. A link cannot borrow that principal for other reads or outlive its current scope. """ + if quarantine_active(session): + return False if resource.owner!=grantor or share_target_error(self.registry,resource,chain,session): return False memberships={m.organization_id for m in await session.memberships(grantor)} @@ -168,6 +173,7 @@ async def require_base(self,principal,operation,resource,session): async def require(self,context,request,checks,session): principal=context.principal + require_live_authority(session) if principal.subject==ROOT_SUBJECT or principal.actor==ROOT_SUBJECT: require(context.entry=='local_admin' and principal.method=='local','local_only') memberships={m.organization_id for m in await session.memberships(principal.subject)} if principal.subject else set() @@ -211,7 +217,6 @@ async def require(self,context,request,checks,session): (ancestor.id,principal.subject)) if ban is not None and (ban[0] is None or parse_time(ban[0])>context.now): require(False,'topic_banned') - break direct=None for ancestor in chain: direct=session.one('SELECT participant_a,participant_b,state,resource_id FROM dm_conversations WHERE resource_id=?', diff --git a/src/msg/security/custody_history.py b/src/msg/security/custody_history.py new file mode 100644 index 00000000..39297716 --- /dev/null +++ b/src/msg/security/custody_history.py @@ -0,0 +1,136 @@ +"""Read-only evidence for a bounded custody migration, never retirement authority. + +A valid ACK is the client's signed declaration that it decrypted one mapped copy. +It cannot prove that no external ciphertext, recipient or recoverable backup exists. +""" +from __future__ import annotations + +from msg.core.codec import canonical, decode, unb64 +from msg.core.errors import Failure +from msg.core.models import Signature +from msg.security.age_keys import encryption_key_id, public_from_recipient +from msg.security.crypto import key_id, verify + + +def _sha256(value): + return (isinstance(value, str) and len(value) == 71 and + value.startswith('sha256:') and + all(char in '0123456789abcdef' for char in value[7:])) + + +def _entry(value): + return (isinstance(value, dict) and set(value) == {'id', 'revision', 'ciphertext_digest'} + and all(isinstance(value[key], str) and value[key] for key in ('id', 'revision')) + and _sha256(value['ciphertext_digest'])) + + +def history_evidence(subject_id, challenge_id, challenge, details, current): + """Reverify persisted ACKs, not merely their presence or cached success flags. + + Legacy mappings bind the keys through the immutable challenge. New mappings + additionally carry the exact old/new key IDs; a conflicting binding is denied. + Unknown or malformed records stay pending and are not repaired by this read. + """ + frozen = details.get('age_inventory') + mappings = details.get('rewrap_mappings', {}) + acks = details.get('rewrap_acks', {}) + result = {'verification_scope': 'enumerated_keystore_age_revisions', + 'external_coverage': 'unknown', 'inventory_changed': True, + 'mapped_count': len(mappings) if isinstance(mappings, dict) else 0, + 'acked_count': len(acks) if isinstance(acks, dict) else 0, + 'verified_acked_count': 0, 'known_ciphertexts_migrated': False, + 'historical_revisions_require_migration': True} + try: + if not (isinstance(frozen, list) and all(_entry(item) for item in frozen) + and isinstance(mappings, dict) and isinstance(acks, dict) + and isinstance(current, list) and all(_entry(item) for item in current)): + return result + sources = {item['revision']: item for item in frozen} + if len(sources) != len(frozen): + return result + public = unb64(challenge['public_key'], limit=32) + if (challenge['subject_id'] != subject_id or challenge['challenge_id'] != challenge_id + or key_id(public) != details['new_identity_key_id'] + or encryption_key_id(public_from_recipient(challenge['encryption_recipient'])) + != details['new_encryption_key_id']): + return result + copies = [] + for revision, mapping in mappings.items(): + if not (isinstance(mapping, dict) and mapping.get('old') == sources.get(revision) + and revision in sources and _entry(mapping.get('new')) + and mapping['new']['revision'] != revision + and mapping.get('recipient') == challenge['encryption_recipient'] + and mapping.get('old_encryption_key_id', details['old_encryption_key_id']) + == details['old_encryption_key_id'] + and mapping.get('new_encryption_key_id', details['new_encryption_key_id']) + == details['new_encryption_key_id']): + return result + copies.append(mapping['new']) + if len({item['revision'] for item in copies}) != len(copies): + return result + expected = sorted(frozen + copies, key=lambda item: (item['id'], item['revision'])) + result['inventory_changed'] = current != expected + verified = 0 + for revision, ack in acks.items(): + mapping = mappings.get(revision) + if not (mapping is not None and isinstance(ack, dict) + and ack.get('old_revision') == revision + and ack.get('new_revision') == mapping['new']['revision'] + and ack.get('ciphertext_digest') == mapping['new']['ciphertext_digest'] + and _sha256(ack.get('plaintext_digest')) + and isinstance(ack.get('request_id'), str) and ack['request_id']): + continue + signed = {'subject_id': subject_id, 'challenge_id': challenge_id, + 'old': mapping['old'], 'new': mapping['new'], + 'plaintext_digest': ack['plaintext_digest'], 'request_id': ack['request_id']} + try: + verify(public, canonical(signed), decode(Signature, ack['signature']), + purpose='custodial-rewrap-ack') + except (Failure, KeyError, TypeError, ValueError): + continue + verified += 1 + result['verified_acked_count'] = verified + complete = (set(mappings) == set(acks) == set(sources) + and verified == len(sources) and not result['inventory_changed']) + result['known_ciphertexts_migrated'] = complete + result['historical_revisions_require_migration'] = not complete + except (Failure, KeyError, TypeError, ValueError): + # Persisted damaged state is not successful evidence, nor a reason to + # expose ciphertext/secret details through an exception message. + pass + return result + + +async def retirement_evidence(tx, subject_id, details, *, history_recoverable): + """Separate observable online retirement from unproven backup destruction.""" + new_signing = tx.one('SELECT key_id FROM identity_keys WHERE subject=? AND is_primary=1', + (subject_id,)) + new_encryption = tx.one('SELECT key_id FROM encryption_subkeys WHERE subject=? AND is_primary=1', + (subject_id,)) + switched = (new_signing is not None and new_encryption is not None + and new_signing[0] == details.get('new_identity_key_id') + and new_encryption[0] == details.get('new_encryption_key_id')) + vault = tx.one('''SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext, + destroyed_at,signing_key_id,encryption_key_id FROM custodial_vault WHERE subject=?''', + (subject_id,)) + destroyed = (vault is not None and vault[0] == 'destroyed' and vault[5] is not None + and all(value is None for value in vault[1:5]) + and vault[6] == details.get('old_identity_key_id') + and vault[7] == details.get('old_encryption_key_id')) + old_signing = tx.one('SELECT retired_at,is_primary FROM identity_keys WHERE subject=? AND key_id=?', + (subject_id, details.get('old_identity_key_id'))) + old_encryption = tx.one('SELECT retired_at,is_primary FROM encryption_subkeys WHERE subject=? AND key_id=?', + (subject_id, details.get('old_encryption_key_id'))) + try: + credential = await tx.credential(details.get('old_identity_key_id')) + signing_revoked = credential.subject_id == subject_id and credential.revoked_at is not None + except Failure: + signing_revoked = False + online = (switched and destroyed and signing_revoked and + all(row is not None and row[0] is not None and not row[1] + for row in (old_signing, old_encryption))) + return {'identity_switched': switched, 'history_recoverable': history_recoverable, + 'online_retired': online, 'backup_retired': False, 'server_key_retired': False, + 'backup_retirement_evidence': 'unavailable', + 'completion_status': ('pending_backup_retirement' if online and history_recoverable + else 'pending_history' if switched else 'pending_identity_switch')} diff --git a/src/msg/security/quarantine.py b/src/msg/security/quarantine.py new file mode 100644 index 00000000..85d12aa2 --- /dev/null +++ b/src/msg/security/quarantine.py @@ -0,0 +1,19 @@ +"""A restored snapshot is not current authority until a local recovery is proven. + +The gate is a database fact, not a deletable config marker. No network operation +may clear it. Absence alone means an ordinary installation; malformed, false or +unknown values fail closed. This is containment, not a revocation replay proof. +""" +from msg.core.errors import require + +SETTING = 'recovery_quarantine' + + +def active(session): + return session.one('SELECT value FROM settings WHERE key=?', (SETTING,)) is not None + + +def require_live_authority(session): + # Even anonymous access to formerly public content may revive an ACL that + # was revoked after the snapshot. Only the transport health probe remains. + require(not active(session), 'recovery_quarantined') diff --git a/src/msg/security/rotation_journal.py b/src/msg/security/rotation_journal.py new file mode 100644 index 00000000..6e31ba2c --- /dev/null +++ b/src/msg/security/rotation_journal.py @@ -0,0 +1,60 @@ +"""Validate a locally approved root rotation without loading a network service.""" +from msg.constants import ROOT_SUBJECT, ONLINE_CA +from msg.core.codec import canonical, decode, digest, unb64 +from msg.core.errors import require +from msg.core.models import Certificate, CertificateRequest, Signature +from msg.security.certificates import certificate_body, verify_csr +from msg.security.crypto import Ed25519Signer, open_private_key, verify + +_FIELDS = {'version', 'old_certificate', 'new_trust', 'new_envelope', 'online_csr', + 'previous_envelope_digest', 'statement', 'new_signature', 'old_signature'} + + +def authorization(journal): + """Public, signed commitments; never copy the encrypted root into an audit.""" + return {'version': journal['version'], 'statement': journal['statement'], + 'old_certificate_digest': digest(journal['old_certificate']), + 'new_trust_digest': digest(journal['new_trust']), + 'new_envelope_digest': digest(journal['new_envelope']), + 'online_csr_digest': digest(journal['online_csr']), + 'previous_envelope_digest': journal['previous_envelope_digest']} + + +def validate(journal, *, pin, service, online_public): + require(isinstance(journal, dict) and journal.get('version') == 2, + 'rotation_journal_upgrade_required') + require(set(journal) == _FIELDS, 'invalid_rotation_journal') + signer = Ed25519Signer.from_bytes(open_private_key(journal['new_envelope'], pin)) + trust = journal['new_trust'] + require(set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, + 'invalid_trust_anchor') + require(signer.public_key == unb64(trust['public_key'], limit=32), 'root_key_mismatch') + proof = authorization(journal) + verify(signer.public_key, canonical(proof), decode(Signature, journal['new_signature']), + purpose='root-rotation') + old = decode(Certificate, journal['old_certificate']) + new = decode(Certificate, trust['certificate']) + statement = journal['statement'] + require(statement['old_certificate'] == old.resource_id and + statement['new_certificate'] == new.resource_id and + statement['new_fingerprint'] == digest(signer.public_key) and + type(statement['previous_key_proved']) is bool and + statement['previous_key_proved'] == (journal['old_signature'] is not None), + 'rotation_statement_mismatch') + require(old.resource_id != new.resource_id and old.key_id != new.key_id, + 'rotation_keys_must_change') + require(new.key_id == signer.key_id and new.subject_id == new.issuer_id == ROOT_SUBJECT and + new.parent_certificate_id is None and new.kind == 'ca' and + new.issuance is not None and not new.authority_sources and + old.subject_id == ROOT_SUBJECT and + new.target_service == old.target_service == service, 'rotation_service_mismatch') + verify(signer.public_key, canonical(certificate_body(new)), new.signature, purpose='certificate') + csr = decode(CertificateRequest, journal['online_csr']) + verify_csr(csr) + require(csr.public_key == online_public and csr.subject_id == csr.applicant == ONLINE_CA and + csr.requested_issuer == ROOT_SUBJECT and csr.target_service == service and + csr.kind == 'ca' and csr.delegation_depth == 0 and not csr.authority_sources and + csr.issuance is not None and csr.issuance.max_child_ca_depth == 0 and + len(csr.grants) == 1 and csr.grants[0].capability == 'cert.issue', + 'rotation_online_csr_mismatch') + return signer, old, new, csr, proof diff --git a/src/msg/storage/ledger_migration.py b/src/msg/storage/ledger_migration.py index a6729948..d01d4178 100644 --- a/src/msg/storage/ledger_migration.py +++ b/src/msg/storage/ledger_migration.py @@ -111,7 +111,7 @@ def _validate_identity_references(conn, escrow): 'from_subject', 'to_subject', 'publisher', 'buyer', 'seller', 'parent', 'source_id', 'target_id', 'participant_a', 'participant_b', 'initiator', 'blocker', 'blocked', - 'invited_by', 'granted_by'} + 'invited_by', 'granted_by', 'arbitrator', 'party', 'author'} columns = [(table, column) for table, column in conn.execute(''' SELECT table_name,column_name FROM information_schema.columns WHERE table_schema=current_schema()''') @@ -224,6 +224,22 @@ def _install_subject_account_trigger(conn): 'custodial_upgrades': ('subject', 'subject_id'), 'legacy_directive_versions': ('subject',), 'sync_checkpoints': ('subject',), + # Active market owners/panel authority are not financial account references. + # Settlement receipts retain historical debit/credit IDs intentionally. + 'order_contracts': ('buyer', 'seller', 'buyer_principal.actor', + 'buyer_principal.subject', 'buyer_principal.certificates.*', + 'buyer_principal.ceiling.*.scope.resource_id', + 'policy.policy.candidates.*'), + 'order_settlements': (), + 'order_transitions': ('actor',), + 'delivery_envelopes': ('recipient_subject',), + 'arbitration_policies': ('candidates.*',), + 'arbitration_cases': ('buyer', 'seller', 'opened_by', 'policy.policy.candidates.*', + 'panels.0.*', 'panels.1.*'), + 'arbitration_votes': ('arbitrator', 'principal.actor', 'principal.subject', + 'principal.certificates.*', 'principal.ceiling.*.scope.resource_id'), + 'arbitration_decisions': ('voters.*',), + 'arbitration_evidence': ('source.id',), # Signed settlement parties are identities; order/account digests are not. 'order_escrow_decisions': ('actor', 'recipient'), } diff --git a/src/msg/storage/postgres.py b/src/msg/storage/postgres.py index c983e0dd..63b6f528 100644 --- a/src/msg/storage/postgres.py +++ b/src/msg/storage/postgres.py @@ -199,6 +199,61 @@ channel TEXT NOT NULL CHECK(channel='site'), state TEXT NOT NULL CHECK(state IN ('prepared','claimed')), prepared_at TEXT NOT NULL, claimed_at TEXT, receipt TEXT); +CREATE TABLE IF NOT EXISTS order_deadlines ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), expires_at TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS order_deadlines_due ON order_deadlines(expires_at,order_id); +-- Market v3 keeps published v1/v2 signed contracts intact. Creation reserves stock without money. +ALTER TABLE store_orders ALTER COLUMN payment_transaction_id DROP NOT NULL; +ALTER TABLE store_orders ALTER COLUMN funded_at DROP NOT NULL; +ALTER TABLE store_orders DROP CONSTRAINT IF EXISTS store_orders_state_check; +ALTER TABLE store_orders ADD CONSTRAINT store_orders_state_check CHECK(state IN + ('created','funded','delivered','accepted','settled','cancelled','refunded','disputed')); +CREATE TABLE IF NOT EXISTS delivery_email_endpoints ( + id TEXT PRIMARY KEY, subject TEXT NOT NULL REFERENCES identities(id), + address TEXT NOT NULL, verified_at TEXT NOT NULL, active BOOLEAN NOT NULL DEFAULT TRUE); +CREATE TABLE IF NOT EXISTS delivery_email_challenges ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL REFERENCES store_orders(id), + subject TEXT NOT NULL REFERENCES identities(id), address TEXT NOT NULL, + verifier TEXT NOT NULL, expires_at TEXT NOT NULL, consumed BOOLEAN NOT NULL DEFAULT FALSE); +CREATE TABLE IF NOT EXISTS order_contracts ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), body TEXT NOT NULL, digest TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS order_settlements ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), decision_id TEXT UNIQUE, body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS order_transitions ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL REFERENCES store_orders(id), body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS delivery_envelopes ( + order_id TEXT PRIMARY KEY REFERENCES store_orders(id), body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitration_policies ( + id TEXT PRIMARY KEY, body TEXT NOT NULL, digest TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitrator_roles ( + subject TEXT PRIMARY KEY REFERENCES identities(id), epoch TEXT NOT NULL, active BOOLEAN NOT NULL); +CREATE TABLE IF NOT EXISTS arbitrator_conflicts ( + arbitrator TEXT NOT NULL REFERENCES identities(id), party TEXT NOT NULL REFERENCES identities(id), + PRIMARY KEY(arbitrator,party)); +CREATE TABLE IF NOT EXISTS arbitration_cases ( + id TEXT PRIMARY KEY, order_id TEXT NOT NULL UNIQUE REFERENCES store_orders(id), + round INTEGER NOT NULL DEFAULT 0, state TEXT NOT NULL, deadline TEXT NOT NULL, body TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS arbitration_votes ( + case_id TEXT NOT NULL REFERENCES arbitration_cases(id), round INTEGER NOT NULL, + arbitrator TEXT NOT NULL REFERENCES identities(id), body TEXT NOT NULL, + PRIMARY KEY(case_id,round,arbitrator)); +CREATE TABLE IF NOT EXISTS arbitration_decisions ( + id TEXT PRIMARY KEY, case_id TEXT NOT NULL REFERENCES arbitration_cases(id), + round INTEGER NOT NULL, body TEXT NOT NULL, revoked BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE(case_id,round)); +CREATE TABLE IF NOT EXISTS arbitration_evidence ( + id TEXT PRIMARY KEY, case_id TEXT NOT NULL REFERENCES arbitration_cases(id), + author TEXT NOT NULL REFERENCES identities(id), visibility TEXT NOT NULL, body TEXT NOT NULL); +DO $$ DECLARE tbl TEXT; BEGIN + FOREACH tbl IN ARRAY ARRAY['order_contracts','order_settlements','order_transitions', + 'arbitration_policies','arbitration_votes'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname=tbl||'_append_only' + AND tgrelid=tbl::regclass) THEN + EXECUTE format('CREATE TRIGGER %I BEFORE UPDATE OR DELETE ON %I FOR EACH ROW + EXECUTE FUNCTION msg_money_ledger_append_only()', tbl||'_append_only',tbl); + END IF; + END LOOP; +END $$; CREATE TABLE IF NOT EXISTS order_escrow_decisions ( order_id TEXT PRIMARY KEY REFERENCES store_orders(id), id TEXT NOT NULL UNIQUE, body TEXT NOT NULL, signature TEXT NOT NULL, diff --git a/src/msg/storage/sqlite.py b/src/msg/storage/sqlite.py index 452b3275..1acc061a 100644 --- a/src/msg/storage/sqlite.py +++ b/src/msg/storage/sqlite.py @@ -378,10 +378,16 @@ async def certificate_revoked(self, id): async def memberships(self, subject): if subject is None: return () - memberships=tuple(decode(Membership, loads(row[0])) for row in - self.rows("SELECT body FROM memberships WHERE subject=? ORDER BY org", (subject,))) - active=tuple(member for member in memberships if member.status=='active' - and member.organization_id!='g_public') + rows=self.rows("""SELECT m.org,m.body FROM memberships m + JOIN resources r ON r.id=m.org + WHERE m.subject=? AND r.type='organization' AND r.state='active' + ORDER BY m.org""",(subject,)) + memberships=((org,decode(Membership,loads(body))) for org,body in rows) + # A stale membership row must not revive an archived/missing organization + # or borrow an organization/subject identifier from a mismatched body. + active=tuple(member for org,member in memberships if member.status=='active' + and member.organization_id==org and member.subject_id==subject + and org!='g_public') row=self.one("SELECT body FROM identities WHERE id=? AND kind='subject'",(subject,)) if row is None: return active diff --git a/src/msg/transports/http.py b/src/msg/transports/http.py index 2307d403..e7bbbccb 100644 --- a/src/msg/transports/http.py +++ b/src/msg/transports/http.py @@ -9,6 +9,7 @@ from __future__ import annotations from starlette.requests import Request +from starlette.responses import JSONResponse, Response from msg.core.errors import Failure from msg.transports.http_routes import ( @@ -32,6 +33,20 @@ def __init__(self, app, *, service): self.service = service async def __call__(self, scope, receive, send): + executor = getattr(self.service, 'executor', None) + if scope['type'] == 'http' and executor is not None and executor.recovery_drill_active(): + # Public ACLs in an old snapshot can also have been revoked. Do not + # serve business content before authority replay and local promotion. + health = (scope.get('raw_path', b'') == b'/healthz' and + scope['method'] in {'GET', 'HEAD'} and not scope.get('query_string')) + data = ({'status':'recovery_quarantined','ready':False, + 'writes_enabled':False,'outbound_enabled':False} + if health else {'error':{'code':'recovery_quarantined','retryable':False}}) + response = (Response(status_code=503, headers=BASE_HEADERS) + if scope['method'] == 'HEAD' else + JSONResponse(data, status_code=503, headers=BASE_HEADERS)) + await response(scope, receive, send) + return if scope['type'] == 'http' and scope['method'] == 'GET': raw = scope.get('raw_path') or scope['path'].encode('utf-8') maximum = self.service.settings.server.limits.max_path_bytes diff --git a/src/msg/workers/effects.py b/src/msg/workers/effects.py index 69b5f68c..ec43dbe1 100644 --- a/src/msg/workers/effects.py +++ b/src/msg/workers/effects.py @@ -16,6 +16,7 @@ from msg.constants import ROOT_SUBJECT from msg.core.codec import decode, wire from msg.core.errors import Failure, require +from msg.security.quarantine import active as quarantine_active from msg.core.models import ( CapabilityGrant, EffectJob, EmailSettings, ExecutionContext, Principal, ResourceRef, ) @@ -115,6 +116,8 @@ def __init__(self, app, *, tool_runner=None, mail_sender=None, webhook_sender=No async def _claim(self): async with self.app.metadata.transaction(write=True) as tx: + if quarantine_active(tx): + return None, False # Orphaned external work cannot be assumed not to have executed. for (raw_id,) in tx.execute("SELECT id FROM jobs WHERE state='running' ORDER BY next_at,id"): job = await tx.job(raw_id) @@ -267,6 +270,18 @@ async def _mail(self, job): require(state in {'sent', 'uncertain'}, 'invalid_delivery_result') await self._finish(job, 'done' if state == 'sent' else 'uncertain', state) + async def _market_mail(self, job): + from msg.market.targets import mail_enabled, render_notification + from msg.market.email import render_verification + require(mail_enabled(self.app), 'mail_disabled') + async with self.app.metadata.transaction(write=False) as tx: + await current_principal(self.app, job.principal, tx) + outgoing = await (render_verification(self.app,tx,job) if job.kind=='market_email_verify' + else render_notification(self.app,tx,job)) + state = await self.mail_sender.send(outgoing) + require(state in {'sent', 'uncertain'}, 'invalid_delivery_result') + await self._finish(job, 'done' if state == 'sent' else 'uncertain', state) + async def _webhook(self, job): from msg.core.codec import canonical,decode,loads from msg.core.models import Event @@ -340,6 +355,13 @@ async def _webhook(self, job): async def run_once(self): if self.app.executor.recovery_drill_active(): return False + if 'orders' in self.app.settings.server.plugins: + from msg.market.escrow import resolve_due + if await resolve_due(self.app): + return True + from msg.market.arbitration import resolve_cases + if await resolve_cases(self.app): + return True job, execute = await self._claim() if job is None: return False @@ -350,6 +372,8 @@ async def run_once(self): self.app.settings.server.staging_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='effect-', dir=self.app.settings.server.staging_dir) as temp: await self._tool(job, Path(temp)) + elif job.kind in {'market_mail','market_email_verify'}: + await self._market_mail(job) elif job.kind == 'mail': await self._mail(job) elif job.kind == 'webhook': @@ -375,7 +399,7 @@ async def run_once(self): except Failure as exc: # Known pre-execution rejection is a definite failure. External calls # surface uncertain explicitly rather than inventing an exactly-once promise. - if job.kind=='mail' and exc.retryable and exc.code=='mail_connection_failed': + if job.kind in {'mail','market_mail','market_email_verify'} and exc.retryable and exc.code=='mail_connection_failed': await self._retry_mail(job,exc.code) return True await self._finish(job, 'uncertain' if exc.code in {'external_uncertain','job_lease_lost'} else 'failed', exc.code) diff --git a/src/msg/workers/maintenance.py b/src/msg/workers/maintenance.py index 4b708c9e..37569493 100644 --- a/src/msg/workers/maintenance.py +++ b/src/msg/workers/maintenance.py @@ -190,6 +190,9 @@ async def _collect(app,tx, *, grace_seconds=3600): # In-flight chunks and sealed uploads are roots until the transfer expires. for raw, in tx.execute('SELECT c.body FROM chunks c JOIN transfers t ON t.id=c.transfer_id'): live.add(loads(raw)['content']['digest'][7:]) + from msg.market.references import content_references + for content in content_references(tx.execute): + live.add(content['digest'][7:]) recent=time.time()-grace_seconds removed=0 # Explicit pins are live roots even before a Revision/Transfer pointer is @@ -246,6 +249,9 @@ async def _collect(app,tx, *, grace_seconds=3600): async def run_maintenance(app,action, *, scheduled=False,principal=None): require(action in {'cleanup_expired','deliver_due_todos','rebuild_search','collect_garbage'},'unknown_maintenance_action') async with app.metadata.transaction(write=True) as tx: + from msg.security.quarantine import active as quarantine_active + if quarantine_active(tx): + return {'skipped':'recovery_quarantine'} if not scheduled: require(principal is not None,'maintenance_authority_required') from msg.workers.effects import current_principal diff --git a/tests/issue_78_80/__init__.py b/tests/issue_78_80/__init__.py new file mode 100644 index 00000000..b53e996a --- /dev/null +++ b/tests/issue_78_80/__init__.py @@ -0,0 +1 @@ +"""Keep this suite's conftest separate from the shared integration fixtures.""" diff --git a/tests/issue_78_80/test_read_tree_transports.py b/tests/issue_78_80/test_read_tree_transports.py index 61299f8b..2c92e851 100644 --- a/tests/issue_78_80/test_read_tree_transports.py +++ b/tests/issue_78_80/test_read_tree_transports.py @@ -12,7 +12,7 @@ from msg.transports.client import HTTPTransport,PathGETTransport,MCPHTTPTransport from msg.transports.dictionary import build_dictionary,read_query_path_document from msg.transports.read_tree_path import decode_read_tree_path -from test_read_query_tree import tree,query +from .test_read_query_tree import tree,query PATH='/_r/q/3/r/r_tree/n/1/f/i/x/children/n/1/f/i/x/children/n/1/f/i/up/1/up/1' diff --git a/tests/issue_78_80/test_tree_query_ref.py b/tests/issue_78_80/test_tree_query_ref.py index fa267c83..de0f6651 100644 --- a/tests/issue_78_80/test_tree_query_ref.py +++ b/tests/issue_78_80/test_tree_query_ref.py @@ -7,8 +7,8 @@ from msg.core.models import ResourceRef from msg.transports.http import create_app from msg.transports.client import HTTPTransport -from test_read_query_tree import tree,query -from test_read_tree_transports import business_state +from .test_read_query_tree import tree,query +from .test_read_tree_transports import business_state async def seal_query(h,args): diff --git a/tests/test_authorization_matrix.py b/tests/test_authorization_matrix.py new file mode 100644 index 00000000..1c6d7d17 --- /dev/null +++ b/tests/test_authorization_matrix.py @@ -0,0 +1,142 @@ +"""One source table exercised through the actual transports and CLI dispatcher.""" +from datetime import timedelta + +import httpx +import pytest + +from msg import cli +from msg.client import ClientState, MsgClient +from msg.core.codec import canonical, loads, wire +from msg.core.requests import request_for +from msg.transports.client import HTTPTransport, PathGETTransport, GraphQLTransport, MCPHTTPTransport +from msg.transports.http import create_app +from test_authorization import approve, scoped +from test_authorization_sources import private_post, grant +from test_service import NOW, call, register + + +SOURCES = ('owner','mode','inheritance','group-member','group-maintainer','group-owner', + 'certificate','share-user','share-group') +ADAPTERS = (HTTPTransport, PathGETTransport, GraphQLTransport, MCPHTTPTransport) + + +def ok(result): + assert result.status == 'ok', wire(result) + return result + + +@pytest.mark.asyncio +@pytest.mark.parametrize('source', SOURCES) +async def test_live_source_loss_across_transports_cli_and_projections(installed,tmp_path,monkeypatch,capsys,source): + app,root = installed + owner = await register(app,'matrix-owner') + reader = await register(app,'matrix-reader') + writer = reader if source == 'owner' else owner + rid,revision = await private_post(app,writer) + certs = () + group = None + + async def invoke(operation,args,identity=owner,**kw): + return ok(await call(app,operation,args,key=identity[0],subject=identity[1],**kw)) + + async def change(operation,args,identity=owner,**kw): + async with app.metadata.transaction(write=False) as tx: + generation = (await tx.resource(args['id'])).generation + return await invoke(operation,args,identity,expected=((args['id'],generation),),**kw) + + if source in {'mode','inheritance'}: + await change('content.chmod',{'id':rid,'mode':'0644'}) + if source == 'inheritance': + folder = (await invoke('content.topic_create',{'parent':'/main','name':'inherited'})).resources[0].id + # A live parent link determines traversal even for an old revision. + async with app.metadata.transaction(write=True) as tx: + from dataclasses import replace + resource = await tx.resource(rid) + await tx.replace(replace(resource,parent=folder,generation=resource.generation+1),resource.generation) + if source.startswith('group-') or source == 'share-group': + creator = reader if source == 'group-owner' else owner + invited = owner if source == 'group-owner' else reader + group = (await invoke('group.create',{'name':'matrix-group'},creator)).resources[0].id + await invoke('group.invite',{'group':group,'subject':invited[1]},creator) + await invoke('group.join',{'group':group},invited) + if source == 'group-maintainer': + await invoke('group.role_set',{'group':group,'subject':reader[1],'role':'maintainer'}) + if source == 'share-group': + await grant(app,owner,rid,group,kind='group') + else: + await change('content.chgrp',{'id':rid,'group':group}) + await change('content.chmod',{'id':rid,'mode':'0640'}) + elif source == 'share-user': + gid = await grant(app,owner,rid,reader[1]) + elif source == 'certificate': + ops = ('discovery.get@1','discovery.search@1','communication.sync@1', + 'communication.inbox@1','communication.outbox@1') + cert = await approve(app,root,reader[1],reader[0],( + scoped(app,'resource.read_override',rid,ops),)) + certs = (cert.resource_id,) + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + adapters = [kind(app.settings.service_url,http=http) for kind in ADAPTERS] + state = ClientState(tmp_path/'client',server=app.settings.service_url) + state.save_signer(reader[0]) + state.data.update(subject_id=reader[1],certificates=list(certs));state._save() + monkeypatch.setitem(cli.TRANSPORTS,'http',lambda server:HTTPTransport(server,http=http)) + monkeypatch.setattr(cli,'MsgClient',lambda saved,adapter:MsgClient(saved,adapter,clock=lambda:NOW)) + + async def matrix(expected): + for args in ({'id':rid},{'id':rid,'view':'history'},{'id':rid,'revision':revision}): + packet = request_for('discovery.get',args,app.settings.service_url, + signer=reader[0],subject=reader[1],certificates=certs,source='msg', + expires_at=NOW+timedelta(seconds=60)) + for adapter in adapters: + result = await adapter.call(packet) + assert result.status == expected,(source,adapter.name,args,wire(result)) + parsed = cli.parser().parse_args(['--config-dir',str(state.directory),'--server', + app.settings.service_url,'call','discovery.get',canonical({'id':rid}).decode()]) + result = await cli.run(parsed) + rendered = loads(capsys.readouterr().out.strip()) + assert (result == 0) == (expected == 'ok'),rendered + assert rendered['status'] == expected,rendered + + await matrix('ok') + # Queue an actual reference. Notification projections must not retain + # it after access is lost, independently of payload/history reads. + await invoke('communication.send',{'recipient':reader[1], + 'resource':{'id':rid,'revision':revision}},writer) + if source == 'owner': + authority = await approve(app,root,reader[1],reader[0],( + scoped(app,'resource.chown',rid,('content.chown@1',)),)) + await change('content.chown',{'id':rid,'owner':owner[1]},reader,certs=(authority.resource_id,)) + elif source == 'mode': + await change('content.chmod',{'id':rid,'mode':'0600'}) + elif source == 'inheritance': + await change('content.chmod',{'id':folder,'mode':'0700'}) + elif source == 'group-owner': + await invoke('group.owner_transfer',{'group':group,'subject':owner[1]},reader) + await matrix('ok') # Losing a role does not remove active membership. + await invoke('group.leave',{'group':group},reader) + elif source.startswith('group-') or source == 'share-group': + await invoke('group.leave',{'group':group},reader) + elif source == 'share-user': + await invoke('sharing.revoke',{'grant_id':gid},contract_version=2) + else: + authority = await approve(app,root,owner[1],owner[0],( + scoped(app,'cert.revoke',cert.resource_id,('cert.revoke@1',)),)) + await invoke('cert.revoke',{'id':cert.resource_id,'reason':'matrix revocation'},certs=(authority.resource_id,)) + async with app.metadata.transaction(write=False) as tx: + before = tuple(tx.one('SELECT COUNT(*) FROM '+table)[0] for table in ('events','jobs','audit','revisions')) + await matrix('error') + # Drop a revoked certificate before testing ordinary projections: the + # revoked envelope itself has already been rejected by every adapter. + for operation,args in (('discovery.search',{'query':'source-matrix-private'}), + ('communication.sync',{}),('communication.inbox',{}),('communication.outbox',{})): + packet = request_for(operation,args,app.settings.service_url,signer=reader[0],subject=reader[1], + source='msg',expires_at=NOW+timedelta(seconds=60)) + for adapter in adapters: + result = ok(await adapter.call(packet)) + assert 'source-matrix-private' not in canonical(result.data).decode(),(source,operation,wire(result)) + if operation != 'communication.sync': + assert rid not in canonical(result.data).decode(),(source,operation,wire(result)) + async with app.metadata.transaction(write=False) as tx: + assert before == tuple(tx.one('SELECT COUNT(*) FROM '+table)[0] for table in ('events','jobs','audit','revisions')) diff --git a/tests/test_authorization_recovery_boundaries.py b/tests/test_authorization_recovery_boundaries.py new file mode 100644 index 00000000..f65a2fba --- /dev/null +++ b/tests/test_authorization_recovery_boundaries.py @@ -0,0 +1,128 @@ +"""Real restore and in-flight revocation never turn a stored reference into access.""" +import asyncio +from datetime import timedelta + +import httpx +import pytest + +from msg.admin.backups import backup, restore +from msg.application import Application +from msg.config import load_settings +from msg.core.codec import b64, canonical, wire +from msg.core.errors import Failure +from msg.core.requests import request_for +from msg.transports.client import HTTPTransport, PathGETTransport, GraphQLTransport, MCPHTTPTransport +from msg.transports.http import create_app +from msg.workers.effects import EffectWorker +from test_authorization_sources import private_post, grant +from test_share_grants_v2 import invoke, ok +from test_share_links import proof +from test_service import NOW, register + + +@pytest.mark.asyncio +async def test_shared_post_does_not_open_attachment_and_concurrent_revocation_is_final(installed): + app, _ = installed + owner = await register(app, 'boundary-owner') + reader = await register(app, 'boundary-reader') + rid, revision = await private_post(app, owner) + source = ok(await invoke(app, owner, 'content.file_put', { + 'parent': '/@boundary-owner/files', 'name': 'secret.txt', + 'data': b64(b'attachment-secret'), 'media_type': 'text/plain'})) + async with app.metadata.transaction(write=False) as tx: + generation = (await tx.resource(rid)).generation + attached = ok(await invoke(app, owner, 'content.attach', { + 'post': rid, 'source': wire(source.resources[0])}, expected=((rid, generation),))) + attachment = attached.data['attachment']['id'] + gid = await grant(app, owner, rid, reader[1]) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + adapters = [kind(app.settings.service_url, http=http) for kind in + (HTTPTransport, PathGETTransport, GraphQLTransport, MCPHTTPTransport)] + def packet(operation, args): + return request_for(operation, args, app.settings.service_url, + signer=reader[0], subject=reader[1], source='msg', + expires_at=NOW + timedelta(seconds=60)) + for adapter in adapters: + ok(await adapter.call(packet('discovery.get', {'id': rid}))) + hidden = await adapter.call(packet('discovery.get', {'id': attachment})) + assert hidden.status == 'error', wire(hidden) + links = ok(await adapter.call(packet('discovery.links', {'id': rid, 'rel': 'f'}))) + assert not links.data['items'], wire(links) + # Reads may serialize before or after revoke, but every call started + # after the mutation completes must reject, even on reused clients. + results = await asyncio.gather( + *(adapter.call(packet('discovery.get', {'id': rid})) for adapter in adapters), + invoke(app, owner, 'sharing.revoke', {'grant_id': gid}, version=2)) + ok(results[-1]) + for adapter in adapters: + for args in ({'id': rid}, {'id': rid, 'revision': revision}): + denied = await adapter.call(packet('discovery.get', args)) + assert denied.error.code == 'permission_denied', wire(denied) + + +@pytest.mark.asyncio +async def test_share_link_every_supported_transport_and_explicit_get_rejection(installed): + app, _ = installed + owner = await register(app, 'link-matrix-owner') + rid, _ = await private_post(app, owner) + async with app.metadata.transaction(write=True) as tx: + tx.set_setting('share_links_enabled', True) + token, verifier = proof() + created = ok(await invoke(app, owner, 'sharing.link_create', { + 'resource': rid, 'verifier': verifier, + 'expires_at': wire(NOW + timedelta(hours=1))})) + lid = created.data['link']['id'] + packet = request_for('sharing.link_read', {'link_id': lid, 'token': token}, + app.settings.service_url, expires_at=NOW + timedelta(seconds=60)) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + adapters = [kind(app.settings.service_url, http=http) for kind in + (HTTPTransport, GraphQLTransport, MCPHTTPTransport)] + for adapter in adapters: + result = ok(await adapter.call(packet)) + assert set(result.data) == {'id', 'revision', 'media_type', 'data'} + with pytest.raises(Failure): + await PathGETTransport(app.settings.service_url, http=http).call(packet) + ok(await invoke(app, owner, 'sharing.link_revoke', {'link_id': lid})) + for adapter in adapters: + result = await adapter.call(packet) + assert result.error.code == 'share_link_unavailable', wire(result) + + +@pytest.mark.asyncio +async def test_actual_postgres_restore_preserves_revocation_and_filters_notifications(installed, tmp_path, pg_dsn): + app, _ = installed + owner = await register(app, 'restored-owner') + reader = await register(app, 'restored-reader') + rid, revision = await private_post(app, owner) + gid = await grant(app, owner, rid, reader[1]) + ok(await invoke(app, owner, 'communication.send', { + 'recipient': reader[1], 'resource': {'id': rid, 'revision': revision}})) + ok(await invoke(app, reader, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': gid}, version=2)) + archive = tmp_path / 'revoked.zip' + await backup(app, archive) + config = tmp_path / 'restored-etc' + result = restore(archive, config, tmp_path / 'restored-data', postgres_dsn=pg_dsn) + assert result['outbound'] == 'disabled_recovery_drill' + restored = Application(load_settings(config), clock=lambda: NOW) + await restored.load() + try: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(restored)), + base_url=restored.settings.service_url) as http: + for kind in (HTTPTransport, PathGETTransport, GraphQLTransport, MCPHTTPTransport): + adapter = kind(restored.settings.service_url, http=http) + for args in ({'id': rid}, {'id': rid, 'view': 'history'}, {'id': rid, 'revision': revision}): + packet = request_for('discovery.get', args, restored.settings.service_url, + signer=reader[0], subject=reader[1], expires_at=NOW + timedelta(seconds=60)) + denied = await adapter.call(packet) + assert denied.error.code == 'permission_denied', wire(denied) + for operation in ('communication.inbox', 'communication.outbox', 'communication.sync'): + projected = ok(await invoke(restored, reader, operation, {})) + assert 'source-matrix-private' not in canonical(projected.data).decode() + if operation != 'communication.sync': + assert rid not in canonical(projected.data).decode() + assert await EffectWorker(restored).run_once() is False + finally: + await restored.close() diff --git a/tests/test_authorization_sources.py b/tests/test_authorization_sources.py new file mode 100644 index 00000000..3e3a98a3 --- /dev/null +++ b/tests/test_authorization_sources.py @@ -0,0 +1,168 @@ +"""Live source matrix: credentials are ceilings; independent grants are alternatives. + +These regressions also simulate stale/corrupt metadata restored from a backup. +The live authorizer must not assume that issuance-time validation ran recently. +""" +from dataclasses import replace +from datetime import timedelta + +import httpx +import pytest + +from msg.core.codec import canonical, wire +from msg.core.requests import request_for +from msg.transports.client import HTTPTransport, PathGETTransport, MCPHTTPTransport +from msg.transports.http import create_app +from test_service import NOW, call, register +from test_share_grants_v2 import invoke, ok + + +async def private_post(app, owner): + post = ok(await invoke(app, owner, 'content.post_create', + {'parent': '/main', 'body': 'source-matrix-private'})) + rid = post.resources[0].id + ok(await invoke(app, owner, 'content.chmod', {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),))) + return rid, post.resources[0].revision + + +async def grant(app, owner, rid, grantee, *, kind='user', parent=None, reshare=False): + args = {'resource': rid, 'grantee': grantee, 'grantee_kind': kind, + 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=1)), + 'allow_reshare': reshare} + if parent: + args['source_grant_id'] = parent + return ok(await invoke(app, owner, 'sharing.grant', args, version=2)).data['grant']['id'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('source', ['mode', 'share']) +async def test_archived_organization_cannot_authorize_stale_active_membership(installed, source): + app, _ = installed + owner = await register(app, 'matrix-owner') + member = await register(app, 'matrix-member') + group = ok(await invoke(app, owner, 'group.create', {'name': 'matrix-org'})).resources[0].id + ok(await invoke(app, owner, 'group.invite', {'group': group, 'subject': member[1]})) + ok(await invoke(app, member, 'group.join', {'group': group})) + rid, _ = await private_post(app, owner) + if source == 'share': + await grant(app, owner, rid, group, kind='group') + else: + async with app.metadata.transaction(write=False) as tx: + generation = (await tx.resource(rid)).generation + changed = ok(await invoke(app, owner, 'content.chgrp', {'id': rid, 'group': group}, + expected=((rid, generation),))) + ok(await invoke(app, owner, 'content.chmod', {'id': rid, 'mode': '0640'}, + expected=((rid, changed.data['generation']),))) + ok(await invoke(app, member, 'discovery.get', {'id': rid})) + async with app.metadata.transaction(write=True) as tx: + organization = await tx.resource(group) + # A restored membership row is not authority for a retired organization. + await tx.replace(replace(organization, state='archived', + generation=organization.generation + 1), organization.generation) + result = await invoke(app, member, 'discovery.get', {'id': rid}) + assert result.error and result.error.code == 'permission_denied', wire(result) + # An independent explicit user grant still works; source failure is not a veto. + await grant(app, owner, rid, member[1]) + ok(await invoke(app, member, 'discovery.get', {'id': rid})) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('corruption', ['future_operation', 'future_constraint', 'parent_expiry']) +async def test_restored_share_sources_cannot_silently_expand_contract(installed, corruption): + app, _ = installed + owner = await register(app, 'restore-owner') + middle = await register(app, 'restore-middle') + reader = await register(app, 'restore-reader') + rid, _ = await private_post(app, owner) + parent = await grant(app, owner, rid, middle[1], reshare=True) + child = await grant(app, middle, rid, reader[1], parent=parent) + ok(await invoke(app, reader, 'discovery.get', {'id': rid})) + async with app.metadata.transaction(write=True) as tx: + if corruption == 'future_operation': + tx.execute('UPDATE share_grants_v2 SET operations=? WHERE id=?', + ('["read","write"]', child), write=True) + elif corruption == 'future_constraint': + tx.execute('UPDATE share_grants_v2 SET constraints=? WHERE id=?', + ('{"future_permission":true}', child), write=True) + else: + tx.execute('UPDATE share_grants_v2 SET expires_at=? WHERE id=?', + (wire(NOW + timedelta(hours=1)), parent), write=True) + result = await invoke(app, reader, 'discovery.get', {'id': rid}) + assert result.error and result.error.code == 'permission_denied', wire(result) + + +@pytest.mark.asyncio +async def test_existing_share_source_stops_at_new_preview_boundary(installed): + app, _ = installed + owner = await register(app, 'preview-matrix-owner') + reader = await register(app, 'preview-matrix-reader') + rid, _ = await private_post(app, owner) + source = await grant(app, owner, rid, reader[1], reshare=True) + async with app.metadata.transaction(write=True) as tx: + tx.set_setting('hosting_preview_file:' + rid, {'owner': owner[1]}) + async with app.metadata.transaction(write=False) as tx: + resource = await tx.resource(rid) + assert not await app.authorizer.share_source_active( + resource, source, reader[1], NOW, tx, reshare=True) + denied = await invoke(app, reader, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' + + +@pytest.mark.asyncio +async def test_topic_ban_cannot_be_bypassed_by_a_preexisting_nested_topic(installed): + app, _ = installed + owner = await register(app, 'ban-matrix-owner') + writer = await register(app, 'ban-matrix-writer') + parent = ok(await invoke(app, owner, 'content.topic_create', + {'parent': '/main', 'name': 'ban-matrix'})).resources[0].id + child = ok(await invoke(app, writer, 'content.topic_create', + {'parent': parent, 'name': 'nested'})).resources[0].id + ok(await invoke(app, owner, 'content.topic_ban', + {'id': parent, 'subject_id': writer[1]})) + result = await invoke(app, writer, 'content.post_create', + {'parent': child, 'body': 'must not cross ancestor ban'}) + assert result.error and result.error.code == 'topic_banned', wire(result) + + +@pytest.mark.asyncio +async def test_share_revocation_matrix_covers_current_history_transports_search_sync(installed): + app, _ = installed + owner = await register(app, 'transport-matrix-owner') + reader = await register(app, 'transport-matrix-reader') + rid, revision = await private_post(app, owner) + gid = await grant(app, owner, rid, reader[1]) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + transports = [kind(app.settings.service_url, http=http) for kind in + (HTTPTransport, PathGETTransport, MCPHTTPTransport)] + async def matrix(expected): + for args in ({'id': rid}, {'id': rid, 'view': 'history'}, + {'id': rid, 'revision': revision}): + packet = request_for('discovery.get', args, app.settings.service_url, + signer=reader[0], subject=reader[1], source='msg', + expires_at=NOW + timedelta(seconds=60)) + for transport in transports: + result = await transport.call(packet) + assert result.status == expected, (transport.name, args, wire(result)) + response = await http.post('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/_r/graphql', json={ + 'query': 'query($p: JSON!) { call(packet: $p) }', + 'variables': {'p': wire(packet)}}) + assert response.json()['data']['call']['status'] == expected, response.text + await matrix('ok') + sync = ok(await invoke(app, reader, 'communication.sync', {})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': gid}, version=2)) + async with app.metadata.transaction(write=False) as tx: + before = (tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0]) + await matrix('error') + search = ok(await invoke(app, reader, 'discovery.search', {'query': 'source-matrix-private'})) + assert rid not in canonical(search.data).decode() + current = ok(await invoke(app, reader, 'communication.sync', {})) + assert all(item.get('kind') == 'revoked' or item.get('ref', {}).get('id') != rid + for item in current.data['items']) + async with app.metadata.transaction(write=False) as tx: + assert before == (tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0]) diff --git a/tests/test_backup_daemon.py b/tests/test_backup_daemon.py index bf716938..30dcdbb6 100644 --- a/tests/test_backup_daemon.py +++ b/tests/test_backup_daemon.py @@ -22,7 +22,11 @@ async def test_consistent_backup_restores_content_but_never_root_private_key(ins restored=Application(load_settings(tmp_path/'restored-etc'),clock=lambda:NOW) await restored.load() read=await call(restored,'discovery.get',{'id':posted.resources[0].id}) - assert read.status=='ok' and read.data['content']=='survives recovery',wire(read) + assert read.status=='error' and read.error.code=='recovery_quarantined',wire(read) + # Inspect offline bytes without treating pre-replay snapshot ACLs as current. + async with restored.metadata.transaction(write=False) as tx: + revision=await tx.revision(posted.resources[0]) + assert await restored.contents.read_bytes(revision.content)==b'survives recovery' assert await restored.contents.read_bytes(binary)==b'\x00\xffbackup' assert not (tmp_path/'restored-etc'/'root').exists() await restored.close() diff --git a/tests/test_backup_drill.py b/tests/test_backup_drill.py index 6a74fcdb..731e7338 100644 --- a/tests/test_backup_drill.py +++ b/tests/test_backup_drill.py @@ -3,6 +3,8 @@ import asyncio import threading import zipfile +import httpx +import subprocess import psycopg import pytest @@ -15,6 +17,8 @@ from msg.core.models import EffectJob,Principal from msg.storage.git import LFSObjectStore from msg.workers.effects import EffectWorker +from msg.workers.maintenance import run_maintenance +from msg.transports.http import create_app from test_service import NOW,call, register @@ -51,7 +55,7 @@ def paused_dump(destination): @pytest.mark.asyncio -async def test_restore_preserves_shared_lfs_hardlinks_and_disables_outbound(installed, tmp_path, pg_dsn): +async def test_restore_preserves_shared_lfs_hardlinks_and_disables_outbound(installed, tmp_path, pg_dsn, monkeypatch): app,_=installed key,subject,_=await register(app,'backup-lfs') created=await call(app,'git.create',{'parent':'/@backup-lfs','name':'source.git'}, @@ -80,6 +84,7 @@ async def test_restore_preserves_shared_lfs_hardlinks_and_disables_outbound(inst restored_data=tmp_path/'drill-data' result=restore(archive,restored_config,restored_data,postgres_dsn=pg_dsn) assert result['outbound']=='disabled_recovery_drill' + assert result['revocation_replay']=='required' and result['promotion']=='blocked' settings=load_settings(restored_config) restored_repo=settings.server.repositories_dir/(rid+'.git') linked=restored_repo/'lfs'/'objects'/oid[:2]/oid[2:4]/oid @@ -93,8 +98,27 @@ async def test_restore_preserves_shared_lfs_hardlinks_and_disables_outbound(inst assert settings.server.valkey_url is None assert settings.server.mail is None assert not (restored_config/'root').exists() - restored=Application(settings,clock=lambda:NOW) - await restored.load() + # A copied/deleted config marker must not resurrect snapshot authority. + (restored_config/'recovery-drill.json').unlink() + async def forbidden_sync(*args,**kwargs): + raise AssertionError('loading a quarantine rewrote release resources') + with monkeypatch.context() as patched: + patched.setattr('msg.bootstrap.sync_system_sources',forbidden_sync) + restored=Application(settings,clock=lambda:NOW) + await restored.load() + anonymous=await call(restored,'discovery.get',{'id':rid}) + assert anonymous.status=='error' and anonymous.error.code=='recovery_quarantined' + authenticated=await call(restored,'discovery.get',{'id':rid},key=key,subject=subject) + assert authenticated.status=='error' and authenticated.error.code=='recovery_quarantined' + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(restored)), + base_url='http://testserver') as http: + health=await http.get('/healthz') + assert health.status_code==503 and health.json()['status']=='recovery_quarantined' + assert health.json()['ready'] is False and health.json()['outbound_enabled'] is False + assert (await http.get('/main')).status_code==503 + assert (await http.head('/healthz')).content==b'' + assert await run_maintenance(restored,'deliver_due_todos',scheduled=True)=={ + 'skipped':'recovery_quarantine'} rejected=await call(restored,'content.post_create', {'parent':'/main','body':'should not publish'}, key=key,subject=subject) @@ -111,8 +135,12 @@ async def send(self,*args,**kwargs): worker=EffectWorker(restored,mail_sender=sender,webhook_sender=sender, tool_runner=sender) assert await worker.run_once() is False + assert await worker._claim()==(None,False) # Database gate, independent of process cache. assert sender.calls==0 async with restored.metadata.transaction(write=False) as tx: + quarantine=tx.setting('recovery_quarantine') + assert quarantine['authority']=='health_only' + assert quarantine['revocation_replay']=='required' runtime=tx.setting('runtime_config') assert runtime['accept_writes'] is False assert runtime['cleanup_enabled'] is False @@ -150,3 +178,26 @@ async def test_restore_rejects_lfs_digest_conflict_before_touching_pg(installed, restore(altered,tmp_path/'bad-etc',tmp_path/'bad-data',postgres_dsn=pg_dsn) assert not (tmp_path/'bad-etc').exists() assert not (tmp_path/'bad-data').exists() + + +@pytest.mark.asyncio +async def test_interrupted_restore_rolls_back_dump_and_gate_together(installed,tmp_path,pg_dsn,monkeypatch): + from msg.admin import restore_database + app,_=installed + archive=tmp_path/'atomic.zip' + await backup(app,archive) + original=restore_database.subprocess.run + def fail_inside_import(argv,**kwargs): + if argv[0]=='psql': + from pathlib import Path + script=Path(argv[argv.index('--file')+1]) + with script.open('a') as stream: + stream.write('\nSELECT 1/0;\n') + return original(argv,**kwargs) + monkeypatch.setattr(restore_database.subprocess,'run',fail_inside_import) + with pytest.raises(subprocess.CalledProcessError): + restore(archive,tmp_path/'failed-etc',tmp_path/'failed-data',postgres_dsn=pg_dsn) + with psycopg.connect(pg_dsn) as connection: + assert connection.execute("SELECT 1 FROM information_schema.tables " + "WHERE table_schema=current_schema() LIMIT 1").fetchone() is None + assert not (tmp_path/'failed-etc').exists() and not (tmp_path/'failed-data').exists() diff --git a/tests/test_backup_policy.py b/tests/test_backup_policy.py index 1c1b2e47..dca227d9 100644 --- a/tests/test_backup_policy.py +++ b/tests/test_backup_policy.py @@ -7,6 +7,7 @@ from msg.application import Application from msg.config import load_settings from msg.core.codec import wire +from msg.plugins.money import _balance, _supply from test_service import NOW, call, register @@ -42,11 +43,12 @@ async def test_real_backup_restore_preserves_money_and_credential_policy(install await restored.load() try: current = await call(restored, 'money.state', {}) - assert current.status == 'ok', wire(current) - assert current.data == original.data + assert current.status == 'error' and current.error.code == 'recovery_quarantined', wire(current) balance = await call(restored, 'money.balance', {}, key=key, subject=subject) - assert balance.status == 'ok' and balance.data == original_balance.data, wire(balance) + assert balance.status == 'error' and balance.error.code == 'recovery_quarantined', wire(balance) async with restored.metadata.transaction(write=False) as tx: + assert _supply(tx) == original.data['total_supply_minor'] + assert _balance(tx, subject) == original_balance.data['balance_minor'] runtime = tx.setting('runtime_config') assert runtime['accept_writes'] is False and runtime['cleanup_enabled'] is False finally: diff --git a/tests/test_ci_shards.py b/tests/test_ci_shards.py index bbd7eab2..41564600 100644 --- a/tests/test_ci_shards.py +++ b/tests/test_ci_shards.py @@ -1,17 +1,16 @@ -"""The CI gate must reject missing, duplicate, changed or failing evidence.""" +"""The CI gate must reject missing, duplicate, changed, skipped or failing evidence.""" import importlib.util import json import os import subprocess import sys -from pathlib import Path import xml.etree.ElementTree as ET +from pathlib import Path import pytest - -spec = importlib.util.spec_from_file_location('ci_shards', - Path(__file__).resolve().parents[1] / 'scripts' / 'ci_shards.py') +SCRIPT = Path(__file__).resolve().parents[1] / 'scripts' / 'ci_shards.py' +spec = importlib.util.spec_from_file_location('ci_shards', SCRIPT) ci = importlib.util.module_from_spec(spec) spec.loader.exec_module(ci) NODES = [f'tests/sub/test_contract.py::test_case[{i}]' for i in range(100)] @@ -24,8 +23,8 @@ def write_evidence(directory, count=4): suite = ET.Element('testsuite') for node in record['selected']: case = ET.SubElement(suite, 'testcase', name=node) - properties = ET.SubElement(case, 'properties') - ET.SubElement(properties, 'property', name='msg.nodeid', value=node) + props = ET.SubElement(case, 'properties') + ET.SubElement(props, 'property', name='msg.nodeid', value=node) ET.ElementTree(suite).write(directory / f'tests-{index}.xml') @@ -55,7 +54,8 @@ def test_valid_complete_evidence_is_accepted(tmp_path): @pytest.mark.parametrize('damage', ['missing_manifest', 'missing_report', 'duplicate_shard', - 'missing_test', 'failure', 'error', 'different_suite']) + 'missing_test', 'failure', 'error', 'skipped', 'different_suite', 'wrong_id', 'duplicate_id', + 'missing_id', 'ambiguous_id']) def test_gate_rejects_incomplete_or_inconsistent_evidence(tmp_path, damage): write_evidence(tmp_path) manifest = tmp_path / 'shard-1.json' @@ -70,9 +70,19 @@ def test_gate_rejects_incomplete_or_inconsistent_evidence(tmp_path, damage): manifest.write_text(json.dumps(ci.evidence(NODES + ['new_test'], 1, 4))) else: tree = ET.parse(report) - case = tree.getroot().find('testcase') + cases = tree.getroot().findall('testcase') + case = cases[0] + prop = case.find('./properties/property') if damage == 'missing_test': tree.getroot().remove(case) + elif damage == 'wrong_id': + prop.set('value', 'tests/test_other.py::not_executed') + elif damage == 'duplicate_id': + prop.set('value', cases[1].find('./properties/property').get('value')) + elif damage == 'missing_id': + case.remove(case.find('properties')) + elif damage == 'ambiguous_id': + ET.SubElement(case.find('properties'), 'property', name='msg.nodeid', value='other') else: ET.SubElement(case, damage) tree.write(report) @@ -80,6 +90,20 @@ def test_gate_rejects_incomplete_or_inconsistent_evidence(tmp_path, damage): ci.verify(tmp_path, 4) +def test_actual_pytest_shards_emit_verifiable_node_ids(tmp_path): + tests = tmp_path / 'tests' + tests.mkdir() + (tmp_path / 'pytest.ini').write_text('[pytest]\n') + (tests / 'test_probe.py').write_text( + 'import pytest\n@pytest.mark.parametrize("value", [1, 2, 3, 4])\n' + 'def test_parameter(value):\n assert value > 0\n') + for index in range(2): + result = subprocess.run([sys.executable, str(SCRIPT), 'run', str(index), '2'], + cwd=tmp_path, capture_output=True, text=True, timeout=30, check=False) + assert result.returncode == 0, result.stdout + result.stderr + assert ci.verify(tmp_path / 'artifacts', 2) == 4 + + @pytest.mark.parametrize('damage', ['renamed_node', 'duplicate_node', 'missing_identity', 'duplicate_identity']) def test_equal_test_counts_do_not_hide_wrong_execution_identities(tmp_path, damage): write_evidence(tmp_path) diff --git a/tests/test_custody_exit_evidence.py b/tests/test_custody_exit_evidence.py new file mode 100644 index 00000000..e963430a --- /dev/null +++ b/tests/test_custody_exit_evidence.py @@ -0,0 +1,104 @@ +"""Verify real age copies before ACK, and distinguish online from backup retirement.""" +import os +import shutil + +import pytest + +from msg.client_recovery import _age +from msg.core.codec import b64, canonical, digest, loads, unb64, wire +from msg.security.age_keys import generate_age_key +from msg.security.crypto import Ed25519Signer +from test_custodial_rewrap import _trusted_age_entry +from test_custodial_upgrade import begin, finish +from test_service import call + + +@pytest.mark.asyncio +async def test_empty_upgrade_still_reports_backup_retirement_pending(installed): + app, _ = installed + created = await call(app, 'identity.custodial_create', {'handle':'exit-state', + 'nonce':b64(os.urandom(32)), 'recovery_secret':b64(os.urandom(32))}, contract_version=2) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + signer = Ed25519Signer.generate() + age_key, recipient = generate_age_key() + challenge = await begin(app, subject, token, signer, recipient) + done = await finish(app, subject, token, signer, age_key, challenge.data) + assert done.status == 'ok' and done.data['status'] == 'completed' + result = await call(app, 'identity.custodial_upgrade_inventory', + {'challenge_id':challenge.data['challenge_id']}, key=signer, subject=subject) + assert result.status == 'ok', wire(result) + retirement = result.data['retirement'] + assert retirement['identity_switched'] + assert retirement['online_signing_key_deleted'] and retirement['online_encryption_key_deleted'] + assert result.data['phase'] == 'online_key_retired' + assert result.data['history_recoverable'] + assert result.data['verification_scope'] == 'enumerated_and_client_verified_revisions_only' + assert retirement['backup_retired'] is False and retirement['server_key_retired'] is False + + +@pytest.mark.asyncio +async def test_changed_persisted_ack_is_not_still_successful_evidence(installed, tmp_path): + if shutil.which('age') is None: + pytest.skip('age CLI is unavailable') + app, _ = installed + created = await call(app, 'identity.custodial_create', {'handle':'ack-corruption', + 'nonce':b64(os.urandom(32)), 'recovery_secret':b64(os.urandom(32))}, contract_version=2) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + plaintext = b'client-verified historical content\n' + old_cipher = _age('--encrypt', '--recipient', created.data['encryption_recipient'], input_data=plaintext) + rid, revision = await _trusted_age_entry(app, subject, old_cipher) + signer = Ed25519Signer.generate() + age_key, recipient = generate_age_key() + challenge = await begin(app, subject, token, signer, recipient) + cid = challenge.data['challenge_id'] + await finish(app, subject, token, signer, age_key, challenge.data) + mapped = await call(app, 'identity.custodial_rewrap_revision', { + 'challenge_id':cid, 'ciphertext_ref':{'id':rid, 'revision':revision}, + 'old_encryption_key_id':created.data['encryption_key_id'], 'new_recipient':recipient}, + subject=subject, token=token) + assert mapped.status == 'ok', wire(mapped) + mapping = mapped.data['mapping'] + assert mapping['old_key_id'] == created.data['encryption_key_id'] + assert mapping['new_key_id'] != mapping['old_key_id'] + fetched = await call(app, 'identity.custodial_migration_get', + {'challenge_id':cid, 'old_revision':revision}, subject=subject, token=token) + assert fetched.status == 'ok', wire(fetched) + async with app.metadata.transaction(write=False) as tx: + from msg.core.models import ResourceRef + new = await tx.revision(ResourceRef(id=mapping['new']['id'], revision=mapping['new']['revision'])) + identity = tmp_path/'age.key' + identity.write_text(age_key+'\n'); identity.chmod(0o600) + ciphertext = await app.contents.read_bytes(new.content) + assert _age('--decrypt', '--identity', str(identity), input_data=ciphertext) == plaintext + inventory = await call(app, 'identity.custodial_upgrade_inventory', {'challenge_id':cid}, + subject=subject, token=token) + assert inventory.status == 'ok', wire(inventory) + request_id = 'verified-ack' + from msg.security.custodial_migration import ack_statement, keyed_source, keyed_target + async with app.metadata.transaction(write=False) as tx: + details = loads(tx.one('SELECT body FROM custodial_upgrades WHERE id=?', (cid,))[0]) + signed = ack_statement(subject, cid, inventory.data['inventory_digest'], + keyed_source(mapping['old'], details), keyed_target(mapping, details), + digest(plaintext), request_id, method='rewrap') + ack = await call(app, 'identity.custodial_rewrap_ack', { + 'challenge_id':cid, 'old_revision':revision, 'new_revision':mapping['new']['revision'], + 'ciphertext_digest':mapping['new']['ciphertext_digest'], 'plaintext_digest':digest(plaintext), + 'inventory_digest':inventory.data['inventory_digest'], 'method':'rewrap', + 'decryption_ack':wire(signer.sign(canonical(signed), purpose='custodial-history-ack-v1'))}, + subject=subject, token=token, rid=request_id, contract_version=2) + assert ack.status == 'ok', wire(ack) + inventory = await call(app, 'identity.custodial_upgrade_inventory', {'challenge_id':cid}, + subject=subject, token=token) + assert inventory.data['history_recoverable'] + assert revision in inventory.data['verified_revisions'] + async with app.metadata.transaction(write=True) as tx: + details = loads(tx.one('SELECT body FROM custodial_upgrades WHERE id=?', (cid,))[0]) + details['rewrap_acks'][revision]['plaintext_digest'] = digest(b'changed after signature') + tx.execute('UPDATE custodial_upgrades SET body=? WHERE id=?', + (canonical(details).decode(), cid), write=True) + damaged = await call(app, 'identity.custodial_upgrade_inventory', {'challenge_id':cid}, + subject=subject, token=token) + assert damaged.status == 'error' + assert damaged.error is not None diff --git a/tests/test_market_arbitration.py b/tests/test_market_arbitration.py new file mode 100644 index 00000000..9f396704 --- /dev/null +++ b/tests/test_market_arbitration.py @@ -0,0 +1,227 @@ +"""Real signed panel votes; no fake principal or direct settlement mutation.""" +import asyncio +from copy import deepcopy +from datetime import timedelta + +import pytest +from test_market_lifecycle import buy, market +from test_service import NOW, call, register + +from msg.admin.market import apply_market +from msg.core.codec import b64, canonical, digest, unb64, wire +from msg.core.errors import Failure +from msg.market.policy import DEFAULT_POLICY +from msg.plugins.money import _balance + + +async def configured(installed, *, panel_size=3, count=4, appeal=False): + app, root = installed + members = {} + for i in range(count): + key, uid, _ = await register(app, f'arbitrator-{i}') + members[uid] = key + await apply_market(app, root, action='grant', operator='isolated-fixture', subject=uid) + policy = deepcopy(DEFAULT_POLICY) + policy.update(id='test-panel-v1', candidates=list(members), panel_size=panel_size, + quorum=panel_size//2+1, allow_appeal=appeal) + await apply_market(app, root, action='publish', operator='isolated-fixture', policy=policy) + sk, seller, bk, buyer, listing, _ = await market(app, root, mode='service', kind='service', + policy=policy['id']) + purchase = await buy(app, bk, buyer, listing) + assert purchase.status == 'ok', wire(purchase) + opened = await call(app, 'orders.dispute_open', { + 'order_id':purchase.data['order']['id'], 'reason':'quality'}, key=bk, subject=buyer) + assert opened.status == 'ok', wire(opened) + return app, root, members, sk, seller, bk, buyer, purchase.data['order'], opened.data + + +async def details(app, key, subject, case_id): + result = await call(app,'orders.dispute_get',{'case_id':case_id},key=key,subject=subject) + assert result.status == 'ok', wire(result) + return result.data + + +async def reasoned_base(app, key, subject, case_id, *, text='The signed allocation follows the case evidence.'): + data = await details(app, key, subject, case_id) + async with app.metadata.transaction(write=False) as tx: + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'", (subject,))[0] + from uuid import uuid4 + source = await call(app, 'content.file_put', {'parent':parent, + 'name':'reason-'+uuid4().hex+'.txt', 'media_type':'text/plain', 'data':b64(text.encode())}, + key=key, subject=subject) + assert source.status == 'ok', wire(source) + pinned = await call(app, 'orders.dispute_rationale', + {'case_id':case_id, 'ref':wire(source.resources[0])}, key=key, subject=subject) + assert pinned.status == 'ok', wire(pinned) + return {**data['proposal_base'], **{name:pinned.data[name] + for name in ('rationale_ref', 'rationale_digest')}} + + +async def vote(app, key, subject, base, *, refund=2_000_000, outcome='split'): + proposal = {**base, 'refund_minor':refund, 'outcome':outcome} + return await call(app,'orders.dispute_vote',{'proposal':proposal, + 'signature':wire(key.sign(canonical(proposal),purpose='arbitration-decision'))}, + key=key,subject=subject) + + +@pytest.mark.asyncio +async def test_fixed_panel_split_quorum_and_concurrent_execution(installed): + app, _root, members, _sk, seller, bk, buyer, order, opened = await configured(installed) + data = await details(app,bk,buyer,opened['case_id']) + panel = list(opened['panel']) + expected = sorted(members,key=lambda uid:(digest({'version':1,'order_id':order['id'], + 'policy_digest':opened['policy_digest'],'round':0,'subject':uid}),uid))[:3] + assert panel == expected and data['panel_valid'] + base = await reasoned_base(app,members[panel[0]],panel[0],opened['case_id']) + # The second signed statement by the same member cannot create a new vote. + first = await vote(app,members[panel[0]],panel[0],base) + assert first.status=='ok' and first.data['decision'] is None + conflict = await vote(app,members[panel[0]],panel[0],base,refund=0,outcome='release') + assert conflict.error.code=='conflicting_vote' + second = await vote(app,members[panel[1]],panel[1],base) + assert second.status=='ok', wire(second) + decision = second.data['decision'] + assert decision and decision['refund_minor']==2_000_000 + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,order['escrow_subject'] if 'escrow_subject' in order else + tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0])==5_000_000 + assert _balance(tx,seller)==0 # A vote is never a payment. + args={'case_id':opened['case_id'],'decision_id':decision['id']} + results=await asyncio.gather(*[call(app,'orders.dispute_execute',args,key=bk, + subject=buyer,rid=f'execution-{i}') for i in range(3)]) + assert all(r.status=='ok' for r in results), [wire(r) for r in results] + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,buyer)==17_000_000 and _balance(tx,seller)==3_000_000 + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?',(order['id'],))[0]==1 + assert tx.one('SELECT COUNT(*) FROM money_ledger WHERE reference LIKE ?', + ('order_%:'+order['id'],))[0]==3 # fund + split's two legs + + +@pytest.mark.asyncio +async def test_role_isolated_evidence_and_summary_has_no_private_fields(installed): + app, root, members, sk, seller, bk, buyer, order, opened = await configured(installed) + cid = opened['case_id'] + statement=await call(app,'orders.dispute_statement',{'case_id':cid, + 'visibility':'panel','text':'Only the panel may see the buyer testimony.'},key=bk,subject=buyer) + assert statement.status=='ok',wire(statement) + eid=statement.data['evidence_id'] + seller_view=await details(app,sk,seller,cid) + assert not seller_view['evidence'] + denied=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=sk,subject=seller) + missing=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':'unknown'},key=sk,subject=seller) + assert denied.error.code==missing.error.code=='evidence_not_found' + arb=opened['panel'][0] + allowed=await call(app,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=members[arb],subject=arb) + assert unb64(allowed.data['data'])==b'Only the panel may see the buyer testimony.' + outsider=next(uid for uid in members if uid not in opened['panel']) + for case_id in (cid,'case_unknown'): + hidden=await call(app,'orders.dispute_get',{'case_id':case_id},key=members[outsider],subject=outsider) + assert hidden.error.code=='case_not_found' + public=await call(app,'orders.dispute_summary',{'case_id':cid}) + assert public.status=='ok' + assert set(public.data)=={'case_id','state','policy_digest','outcome'} + text=canonical(public.data).decode() + assert all(private not in text for private in (buyer,seller,order['id'],eid,'testimony')) + await apply_market(app,root,action='revoke',operator='test',subject=arb) + after=await call(app,'orders.dispute_get',{'case_id':cid},key=members[arb],subject=arb) + assert after.error.code=='case_not_found' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('invalidation',['revoke','regrant','conflict','decision_revoke','expire']) +async def test_invalidated_decision_never_releases_funds(installed,invalidation): + app, root, members, _sk, seller, bk, buyer, order, opened = await configured( + installed,panel_size=1,count=2) + cid, arb=opened['case_id'],opened['panel'][0] + base=await reasoned_base(app,members[arb],arb,cid) + result=await vote(app,members[arb],arb,base,refund=0,outcome='release') + decision=result.data['decision'] + if invalidation in {'revoke','regrant'}: + await apply_market(app,root,action='revoke',operator='test',subject=arb) + if invalidation=='regrant': + await apply_market(app,root,action='grant',operator='test',subject=arb) + elif invalidation=='conflict': + declared=await call(app,'orders.arbitrator_conflict',{'party':buyer},key=members[arb],subject=arb) + assert declared.status=='ok' + elif invalidation=='decision_revoke': + revoked=await call(app,'orders.dispute_revoke',{'case_id':cid,'decision_id':decision['id']},key=members[arb],subject=arb) + assert revoked.status=='ok' + else: + app.clock=lambda:NOW+timedelta(days=10) + from msg.market.arbitration import resolve_cases + assert await resolve_cases(app)==[cid] + if invalidation!='expire': + failed=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':decision['id']},key=bk,subject=buyer) + assert failed.status=='error',wire(failed) + async with app.metadata.transaction(write=False) as tx: + escrow=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0] + assert _balance(tx,escrow)==5_000_000 and _balance(tx,seller)==0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 + + +@pytest.mark.asyncio +async def test_one_appeal_disjoint_panel_and_old_decision_is_unusable(installed): + app, _root, members, sk, seller, bk, buyer, _order, opened=await configured( + installed,panel_size=1,count=2,appeal=True) + cid,arb=opened['case_id'],opened['panel'][0] + base=await reasoned_base(app,members[arb],arb,cid) + result=await vote(app,members[arb],arb,base,refund=0,outcome='release') + old=result.data['decision']['id'] + premature=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':old},key=bk,subject=buyer) + assert premature.error.code=='appeal_window_open' + appealed=await call(app,'orders.dispute_appeal',{'case_id':cid},key=bk,subject=buyer) + assert appealed.status=='ok',wire(appealed) + assert set(appealed.data['panel']).isdisjoint(opened['panel']) + again=await call(app,'orders.dispute_appeal',{'case_id':cid},key=sk,subject=seller) + assert again.error.code=='appeal_not_available' + old_attempt=await call(app,'orders.dispute_execute',{'case_id':cid,'decision_id':old},key=bk,subject=buyer) + assert old_attempt.error.code=='decision_not_current' + reviewer=appealed.data['panel'][0] + current=await reasoned_base(app,members[reviewer],reviewer,cid) + appeal_vote=await vote(app,members[reviewer],reviewer,current,refund=5_000_000,outcome='refund') + assert appeal_vote.status=='ok',wire(appeal_vote) + final=await call(app,'orders.dispute_execute',{'case_id':cid, + 'decision_id':appeal_vote.data['decision']['id']},key=bk,subject=buyer) + assert final.status=='ok',wire(final) + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,buyer)==20_000_000 and _balance(tx,seller)==0 + + +@pytest.mark.asyncio +async def test_split_rollback_restores_escrow_decision_and_retry(installed,monkeypatch): + app, _root, members, _sk, _seller, bk, buyer, order, opened=await configured(installed,panel_size=1,count=1) + cid,arb=opened['case_id'],opened['panel'][0] + base=await reasoned_base(app,members[arb],arb,cid) + result=await vote(app,members[arb],arb,base) + args={'case_id':cid,'decision_id':result.data['decision']['id']} + from msg.market import escrow + original=escrow._post_transfer + def fail_after_first_leg(*args,**kwargs): + original(*args,**kwargs) + raise Failure('forced_split_rollback') + monkeypatch.setattr(escrow,'_post_transfer',fail_after_first_leg) + failed=await call(app,'orders.dispute_execute',args,key=bk,subject=buyer) + assert failed.error.code=='forced_split_rollback' + async with app.metadata.transaction(write=False) as tx: + account=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0] + assert _balance(tx,account)==5_000_000 and _balance(tx,buyer)==15_000_000 + assert tx.one('SELECT state FROM arbitration_cases WHERE id=?',(cid,))[0]=='decided' + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 + monkeypatch.setattr(escrow,'_post_transfer',original) + retried=await call(app,'orders.dispute_execute',args,key=bk,subject=buyer) + assert retried.status=='ok',wire(retried) + + +@pytest.mark.asyncio +async def test_default_no_panel_holds_without_choosing_new_candidates(installed): + app,root=installed + _sk,seller,bk,buyer,listing,_=await market(app,root,mode='service',kind='service') + order=(await buy(app,bk,buyer,listing)).data['order'] + opened=await call(app,'orders.dispute_open',{'order_id':order['id'],'reason':'quality'},key=bk,subject=buyer) + assert opened.status=='ok' and opened.data['state']=='held' and not opened.data['panel'] + _key,uid,_=await register(app,'late-arbitrator') + await apply_market(app,root,action='grant',operator='test',subject=uid) + data=await details(app,bk,buyer,opened.data['case_id']) + assert not data['case']['panels']['0'] and not data['panel_valid'] + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,seller)==0 and tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 diff --git a/tests/test_market_delivery.py b/tests/test_market_delivery.py new file mode 100644 index 00000000..f26647e3 --- /dev/null +++ b/tests/test_market_delivery.py @@ -0,0 +1,228 @@ +"""Site authority, recipient ownership, SMTP minimisation and resumable payloads.""" +import asyncio +import subprocess +from dataclasses import replace + +import pytest +from test_market_lifecycle import buy, market +from test_service import call + +from msg.core.codec import b64, canonical, digest, loads, unb64, wire +from msg.core.errors import Failure +from msg.core.models import MailConfig +from msg.plugins.money import _balance +from msg.workers.effects import EffectWorker + + +class Sender: + def __init__(self, result='sent'): + self.messages = [] + self.result = result + + async def send(self, job): + self.messages.append(dict(job.arguments)) + if isinstance(self.result, Exception): + raise self.result + return self.result + + +def enable_mail(app): + config = MailConfig(enabled=True,host='localhost',port=2525,tls='starttls', + sender='test@example.invalid',credential_file=None) + app.settings = replace(app.settings,server=replace(app.settings.server,mail=config)) + + +async def verified_address(app,key,subject,address): + # Test the real signed account-email flow; don't manufacture verification rows. + result=await call(app,'identity.email_set',{'address':address},key=key,subject=subject) + assert result.status=='ok',wire(result) + async with app.metadata.transaction(write=False) as tx: + raw=tx.one("SELECT body FROM jobs WHERE kind='mail' AND body::jsonb->'principal'->>'subject'=? ORDER BY id DESC",(subject,)) + if raw is None: # Jobs index actor/subject in the canonical Principal, not columns. + raise AssertionError('verification job missing') + job=loads(raw[0]) + token=job['arguments']['text'].rsplit(': ',1)[1] + result=await call(app,'identity.email_verify',{'token':token},key=key,subject=subject) + assert result.status=='ok',wire(result) + + +async def drain(app,sender): + worker=EffectWorker(app,mail_sender=sender) + for _ in range(100): + if not await worker.run_once(): + return + raise AssertionError('worker did not converge') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('smtp', ['sent','uncertain','connection_failed','disabled']) +async def test_verified_checkout_mail_is_minimal_and_not_claimed(installed,smtp): + app,root=installed + enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root) + await verified_address(app,bk,buyer,'buyer@example.invalid') + if smtp=='disabled': + app.settings=replace(app.settings,server=replace(app.settings.server,mail=None)) + result=await buy(app,bk,buyer,listing,email='buyer@example.invalid') + assert result.status=='ok' and result.data['order']['state']=='settled',wire(result) + order=result.data['order']; oid=order['id'] + sender=Sender(Failure('mail_connection_failed',retryable=True) if smtp=='connection_failed' else smtp) + await drain(app,sender) + expected={'sent':'smtp_accepted','uncertain':'delivered_unknown', + 'connection_failed':'pending','disabled':'disabled'}[smtp] + got=await call(app,'orders.get',{'order_id':oid},key=bk,subject=buyer) + assert got.data['order']['email_status']==expected,wire(got) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?',(oid,))==('prepared',None) + assert _balance(tx,buyer)==15_000_000 and _balance(tx,seller)==5_000_000 + if smtp=='disabled': + assert not sender.messages + else: + assert len(sender.messages)==1 + mail=sender.messages[0] + body=loads(mail['text']) + assert body=={'order_id':oid,'handle':'@order-buyer', + 'pickup':app.settings.service_url+'/_orders/'+oid+'/_delivery'} + assert mail['recipient']=='buyer@example.invalid' + assert all(secret not in mail['text'] for secret in + ('delivery-ok','store selftest','token','buyer@example.invalid',seller)) + assert set(mail)=={'recipient','recipient_subject','subject','text'} + seller_view=await call(app,'orders.get',{'order_id':oid},key=sk,subject=seller) + assert 'buyer@example.invalid' not in canonical(seller_view).decode() + + +@pytest.mark.asyncio +async def test_unverified_checkout_has_no_order_information_until_owner_verifies(installed): + app,root=installed; enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root) + bought=await buy(app,bk,buyer,listing,email='new@example.invalid') + assert bought.status=='ok' and bought.data['order']['state']=='settled',wire(bought) + oid=bought.data['order']['id']; sender=Sender() + await drain(app,sender) + assert len(sender.messages)==1 + body=loads(sender.messages[0]['text']) + assert set(body)=={'challenge_id','token','purpose'} + assert all(value not in sender.messages[0]['text'] for value in + (oid,buyer,seller,'order-buyer','delivery-ok','store selftest','pickup')) + args={'order_id':oid,'challenge_id':body['challenge_id'],'token':body['token']} + wrong=await call(app,'orders.email_verify',args,key=sk,subject=seller) + assert wrong.error.code=='order_not_found' + confirmed=await call(app,'orders.email_verify',args,key=bk,subject=buyer,rid='verify-order') + assert confirmed.status=='ok',wire(confirmed) + repeat=await call(app,'orders.email_verify',args,key=bk,subject=buyer,rid='verify-order') + assert repeat.replayed + other=await call(app,'orders.email_verify',args,key=bk,subject=buyer) + assert other.error.code=='email_challenge_invalid' + await drain(app,sender) + assert len(sender.messages)==2 and loads(sender.messages[1]['text'])['order_id']==oid + # Order verification does not overwrite the subject's account email preferences. + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT 1 FROM emails WHERE subject=?',(buyer,)) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('change',['disable','address','wrong_owner']) +async def test_send_time_revalidation_prevents_stale_or_cross_subject_delivery(installed,change): + app,root=installed; enable_mail(app) + _sk,seller,bk,buyer,listing,_=await market(app,root) + await verified_address(app,bk,buyer,'buyer@example.invalid') + bought=await buy(app,bk,buyer,listing,email='buyer@example.invalid'); oid=bought.data['order']['id'] + if change=='disable': + result=await call(app,'orders.email_disable',{'order_id':oid},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + elif change=='address': + result=await call(app,'identity.email_set',{'address':'replacement@example.invalid'},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + else: + async with app.metadata.transaction(write=True) as tx: + row=tx.one('SELECT delivery_target FROM store_orders WHERE id=?',(oid,)) + target=loads(row[0]); target['email']['subject_id']=seller + tx.execute('UPDATE store_orders SET delivery_target=? WHERE id=?',(canonical(target).decode(),oid),write=True) + sender=Sender(); await drain(app,sender) + assert all(oid not in message['text'] for message in sender.messages) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state FROM store_orders WHERE id=?',(oid,))[0]=='settled' + assert _balance(tx,seller)==5_000_000 + + +@pytest.mark.asyncio +async def test_sealed_secret_requires_buyer_key_and_explicit_acceptance(installed,monkeypatch,tmp_path): + import msg.security.age_keys as ages + original=ages.generate_age_key; pairs=[] + def generate(): + pair=original(); pairs.append(pair); return pair + monkeypatch.setattr(ages,'generate_age_key',generate) + app,root=installed; enable_mail(app) + sk,seller,bk,buyer,listing,_=await market(app,root,mode='sealed_manual',kind='secret') + private,recipient=pairs[-1] + bought=await buy(app,bk,buyer,listing,email='new@example.invalid') + assert bought.status=='ok',wire(bought) + order=bought.data['order']; oid=order['id'] + encrypted=await asyncio.to_thread(subprocess.run, ['age','-r',recipient], + input=b'private-secret-123',capture_output=True,check=True) + ciphertext=encrypted.stdout + private_file=tmp_path/'identity'; private_file.write_text(private+'\n'); private_file.chmod(0o600) + decrypted=await asyncio.to_thread(subprocess.run, ['age','-d','-i',str(private_file)], + input=ciphertext,capture_output=True,check=True) + assert decrypted.stdout==b'private-secret-123' + uploaded=await call(app,'content.file_put',{'parent':'/@order-seller/files','name':'encrypted.age', + 'data':b64(ciphertext),'media_type':'application/age'},key=sk,subject=seller) + assert uploaded.status=='ok',wire(uploaded) + args={'order_id':oid,'payload_ref':wire(uploaded.resources[0]),'payload_digest':digest(ciphertext), + 'recipient_key_id':order['recipient_key']['key_id']} + wrong=await call(app,'delivery.submit',{**args,'recipient_key_id':'other-key'},key=sk,subject=seller) + assert wrong.error.code=='delivery_recipient_mismatch' + sent=await call(app,'delivery.submit',args,key=sk,subject=seller) + assert sent.status=='ok' and sent.data['state']=='delivered',wire(sent) + delivered=await call(app,'delivery.get',{'order_id':oid},key=bk,subject=buyer,contract_version=2) + assert unb64(delivered.data['delivery']['payloads'][0]['data'])==ciphertext + sender=Sender(); await drain(app,sender) + assert all('private-secret-123' not in m['text'] for m in sender.messages) + async with app.metadata.transaction(write=False) as tx: + assert _balance(tx,seller)==0 + accepted=await call(app,'delivery.accept',{'order_id':oid,'delivery_digest':sent.data['delivery_digest']}, + key=bk,subject=buyer,contract_version=2) + assert accepted.status=='ok' and accepted.data['state']=='settled',wire(accepted) + + +@pytest.mark.asyncio +async def test_large_service_transfer_has_buyer_acl_and_never_implies_acceptance(installed): + app,root=installed + sk,seller,bk,buyer,listing,_=await market(app,root,mode='service',kind='service') + body=bytes(range(256))*8193 # Larger than 2 MiB; no oversized JSON result. + opened=await call(app,'transfer.open',{'direction':'upload','size':len(body),'digest':digest(body), + 'requested_part_bytes':65536},key=sk,subject=seller) + assert opened.status=='ok',wire(opened) + tid=opened.data['transfer_id']; part=opened.data['part_bytes'] + for offset in range(0,len(body),part): + chunk=body[offset:offset+part] + result=await call(app,'transfer.part_put',{'transfer_id':tid,'offset':offset, + 'data':b64(chunk),'digest':digest(chunk)},key=sk,subject=seller) + assert result.status=='ok',wire(result) + sealed=await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(body),'final_digest':digest(body)}, + key=sk,subject=seller) + assert sealed.status=='ok',wire(sealed) + bought=await buy(app,bk,buyer,listing); oid=bought.data['order']['id'] + submitted=await call(app,'delivery.submit',{'order_id':oid,'payload_ref':wire(sealed.output), + 'payload_digest':digest(body)},key=sk,subject=seller) + assert submitted.status=='ok',wire(submitted) + before=await call(app,'delivery.get',{'order_id':oid},key=bk,subject=buyer,contract_version=2) + assert before.status=='ok' and 'data' not in before.data['delivery']['payloads'][0],wire(before) + args=before.data['delivery']['payloads'][0]['transfer']['arguments'] + exported=await call(app,'delivery.transfer_open',args,key=bk,subject=buyer) + assert exported.status=='ok',wire(exported) + transfer=exported.data['transfer_id']; output=bytearray(); part=exported.data['part_bytes'] + for offset in range(0,len(body),part): + result=await call(app,'transfer.part_get',{'transfer_id':transfer,'offset':offset, + 'length':min(part,len(body)-offset)},key=bk,subject=buyer) + assert result.status=='ok',wire(result) + output.extend(unb64(result.data['data'])) + assert bytes(output)==body + wrong=await call(app,'transfer.part_get',{'transfer_id':transfer,'offset':0,'length':1},key=sk,subject=seller) + assert wrong.error.code=='transfer_owner_required' + async with app.metadata.transaction(write=False) as tx: + resource=await tx.resource(exported.resources[0].id) + assert resource.owner==buyer and resource.mode==0o600 + assert _balance(tx,seller)==0 + assert tx.one('SELECT state FROM store_deliveries WHERE order_id=?',(oid,))[0]=='prepared' + assert tx.one('SELECT state FROM store_orders WHERE id=?',(oid,))[0]=='delivered' diff --git a/tests/test_market_integration.py b/tests/test_market_integration.py new file mode 100644 index 00000000..3a8984dc --- /dev/null +++ b/tests/test_market_integration.py @@ -0,0 +1,106 @@ +"""Published checkout compatibility, restored snapshot integrity and disabled mail.""" +from dataclasses import replace + +import pytest +from test_market_delivery import Sender, drain, enable_mail, verified_address +from test_market_lifecycle import buy, market +from test_orders import _intent +from test_service import call + +from msg.core.codec import digest, wire +from msg.plugins.delivery import _delivery + + +@pytest.mark.asyncio +@pytest.mark.parametrize('field,value', [ + ('seller', None), ('listing_id', None), ('listing_revision', 'r_mixed'), + ('package_revision', 'r_mixed'), ('terms_digest', digest('mixed terms')), + ('escrow_policy', 'escrow-instant-v1'), ('dispute_policy', 'mixed-policy'), + ('created_at', '2026-01-01T00:00:00Z'), ('unit_price_minor', 1), ('quantity', 2), +]) +async def test_funding_rejects_mixed_immutable_checkout_projection(installed, field, value): + app, root = installed + _, _, key, buyer, listing, _ = await market(app, root, mode='service', kind='service') + created = await call(app, 'orders.create', _intent(listing), key=key, subject=buyer) + assert created.status == 'ok', wire(created) + order = created.data['order'] + async with app.metadata.transaction(write=True) as tx: + tx.execute(f'UPDATE store_orders SET {field}=? WHERE id=?', + (buyer if value is None else value, order['id']), write=True) + counts = {table: tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('money_ledger', 'order_transitions', 'events', 'jobs')} + result = await call(app, 'orders.fund', {'order_id': order['id'], + 'order_digest': order['order_digest'], 'total_price_minor': 5_000_000, + 'currency_id': 'primary'}, key=key, subject=buyer) + assert result.status == 'error', wire(result) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state FROM store_orders WHERE id=?', (order['id'],)) == ('created',) + for table, count in counts.items(): + assert tx.one(f'SELECT COUNT(*) FROM {table}')[0] == count + + +@pytest.mark.asyncio +@pytest.mark.parametrize('changed', ['terms', 'payment_refs', 'settled_at', 'delivery_id', 'prepared_at']) +async def test_claim_rejects_mixed_settled_snapshot(installed, changed): + app, root = installed + _, _, key, buyer, listing, _ = await market(app, root) + bought = await buy(app, key, buyer, listing) + assert bought.status == 'ok', wire(bought) + oid = bought.data['order']['id'] + async with app.metadata.transaction(write=True) as tx: + if changed == 'terms': + tx.execute('UPDATE store_orders SET terms_digest=? WHERE id=?', + (digest('mixed terms'), oid), write=True) + elif changed == 'payment_refs': + tx.execute("UPDATE store_orders SET receipt_refs='[]' WHERE id=?", (oid,), write=True) + elif changed == 'settled_at': + tx.execute('UPDATE store_orders SET settled_at=? WHERE id=?', + ('2026-01-01T00:00:00Z', oid), write=True) + elif changed == 'prepared_at': + tx.execute('UPDATE store_deliveries SET prepared_at=? WHERE order_id=?', + ('2026-01-01T00:00:00Z', oid), write=True) + else: + from msg.market.delivery import body_for + from msg.plugins.orders import _row + d = _delivery(tx, oid) + identifier = 'dlv_other_restored_delivery' + body = body_for(_row(tx, oid, buyer), identifier, d['kind'], d['manifest'], d['payload_refs']) + tx.execute('UPDATE store_deliveries SET id=?,delivery_digest=? WHERE order_id=?', + (identifier, digest(body), oid), write=True) + delivery = _delivery(tx, oid) + counts = {table: tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('money_ledger', 'order_settlements', 'events', 'jobs')} + result = await call(app, 'delivery.accept', {'order_id': oid, + 'delivery_digest': delivery['delivery_digest']}, key=key, subject=buyer, contract_version=2) + assert result.status == 'error', wire(result) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?', (oid,)) == ('prepared', None) + for table, count in counts.items(): + assert tx.one(f'SELECT COUNT(*) FROM {table}')[0] == count + + +@pytest.mark.asyncio +@pytest.mark.parametrize('when', ['before_checkout', 'before_worker']) +async def test_explicitly_disabled_mail_never_calls_sender(installed, when): + app, root = installed + enable_mail(app) + _, seller, key, buyer, listing, _ = await market(app, root) + await verified_address(app, key, buyer, 'buyer@example.invalid') + def disable(): + app.settings = replace(app.settings, server=replace(app.settings.server, + mail=replace(app.settings.server.mail, enabled=False))) + if when == 'before_checkout': + disable() + result = await buy(app, key, buyer, listing, email='buyer@example.invalid') + assert result.status == 'ok' and result.data['order']['state'] == 'settled', wire(result) + oid = result.data['order']['id'] + disable() + sender = Sender() + await drain(app, sender) + assert sender.messages == [] + result = await call(app, 'orders.get', {'order_id': oid}, key=key, subject=buyer) + assert result.data['order']['email_status'] == 'disabled', wire(result) + async with app.metadata.transaction(write=False) as tx: + from msg.plugins.money import _balance + assert _balance(tx, buyer) == 15_000_000 and _balance(tx, seller) == 5_000_000 + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?', (oid,)) == ('prepared', None) diff --git a/tests/test_market_lease_deadlines.py b/tests/test_market_lease_deadlines.py new file mode 100644 index 00000000..b6633e91 --- /dev/null +++ b/tests/test_market_lease_deadlines.py @@ -0,0 +1,56 @@ +"""An expired notification lease must converge in both job and order projections.""" +from datetime import timedelta + +import pytest +from test_market_delivery import Sender, drain, enable_mail, verified_address +from test_market_lifecycle import buy, market +from test_service import call + +from msg.core.codec import wire +from msg.plugins.money import _balance +from msg.workers.effects import EffectWorker + + +@pytest.mark.asyncio +@pytest.mark.parametrize('transition', ['finish', 'retry']) +@pytest.mark.parametrize('disabled', [False, True]) +async def test_expired_market_attempt_preserves_money_and_notification_truth(installed, transition, disabled): + app, root = installed + enable_mail(app) + _sk, seller, bk, buyer, listing, _ = await market(app, root) + await verified_address(app, bk, buyer, 'buyer@example.invalid') + sender = Sender() + await drain(app, sender) + sender.messages.clear() + bought = await buy(app, bk, buyer, listing, email='buyer@example.invalid') + assert bought.status == 'ok', wire(bought) + oid = bought.data['order']['id'] + worker = EffectWorker(app, mail_sender=sender, lease_seconds=1) + job, execute = await worker._claim() + assert execute and job.kind == 'market_mail' and job.arguments['order_id'] == oid + now = app.clock() + if disabled: + result = await call(app, 'orders.email_disable', {'order_id':oid}, key=bk, subject=buyer) + assert result.status == 'ok', wire(result) + app.clock = lambda: now + timedelta(seconds=1) + # Exercise completion/retry before any sweeper observes the exact deadline. + if transition == 'finish': + await worker._finish(job, 'done', 'sent') + else: + await worker._retry_mail(job, 'mail_connection_failed') + got = await call(app, 'orders.get', {'order_id':oid}, key=bk, subject=buyer) + assert got.status == 'ok', wire(got) + assert got.data['order']['email_status'] == ('disabled' if disabled else 'delivered_unknown') + assert got.data['order']['state'] == 'settled' + async with app.metadata.transaction(write=False) as tx: + assert (await tx.job(job.id)).state == 'uncertain' + assert tx.setting('job_status:' + job.id) == {'code':'expired_execution_lease'} + assert _balance(tx,buyer) == 15_000_000 and _balance(tx,seller) == 5_000_000 + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?',(oid,)) == ('prepared',None) + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?',(oid,))[0] == 1 + assert await worker._claim() == (None,False) + await worker._finish(job, 'done', 'sent') + await worker._retry_mail(job, 'mail_connection_failed') + after = await call(app, 'orders.get', {'order_id':oid}, key=bk, subject=buyer) + assert after.data['order']['email_status'] == got.data['order']['email_status'] + assert not sender.messages diff --git a/tests/test_market_lifecycle.py b/tests/test_market_lifecycle.py new file mode 100644 index 00000000..72ddf50b --- /dev/null +++ b/tests/test_market_lifecycle.py @@ -0,0 +1,141 @@ +"""The v3 market contract: real signatures, immutable snapshots and conserved money.""" +import asyncio +from datetime import timedelta + +import pytest +from test_orders import _intent, _sale +from test_service import NOW, call, register + +from msg.admin.money import apply_money +from msg.core.codec import unb64, wire + + +async def market(app, root, *, mode='managed_instant', kind='bundle', quantity=2, + policy='dispute-v1'): + seller_key, seller, _ = await register(app, 'order-seller') + buyer_key, buyer, _ = await register(app, 'order-buyer') + if mode == 'managed_instant': + listing, package = await _sale(app, seller_key, seller, quantity=quantity) + else: + created = await call(app, 'store.listing_create', { + 'name': 'sealed sale', 'item_kind': kind, 'price_minor': 5_000_000, + 'currency_id': 'primary', 'quantity': quantity, 'delivery_mode': mode, + 'escrow_policy': 'escrow-v1', 'dispute_policy': policy, + 'terms': 'Delivery within one day; manual acceptance.'}, + key=seller_key, subject=seller) + assert created.status == 'ok', wire(created) + active = await call(app, 'store.listing_update', { + 'id': created.resources[0].id, 'state': 'active'}, + key=seller_key, subject=seller, + expected=((created.resources[0].id, created.data['generation']),)) + assert active.status == 'ok', wire(active) + listing, package = active.data['listing'], None + await apply_money(app, root, action='mint', operator='test-console', amount_minor=20_000_000) + await apply_money(app, root, action='transfer', operator='test-console', + subject_id=buyer, amount_minor=20_000_000) + return seller_key, seller, buyer_key, buyer, listing, package + + +async def buy(app, key, buyer, listing, **extra): + return await call(app, 'orders.buy', {**_intent(listing), **extra}, + key=key, subject=buyer, contract_version=3) + + +@pytest.mark.asyncio +async def test_instant_delivery_settlement_and_claim_are_distinct(installed): + app, root = installed + _sk, seller, bk, buyer, listing, package = await market(app, root) + args = _intent(listing) + result = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=3, rid='instant-once') + assert result.status == 'ok', wire(result) + order = result.data['order'] + assert order['state'] == 'settled' + assert order['package_digest'] == package['digest'] + replay = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=3, rid='instant-once') + assert replay.replayed and replay.data == result.data + delivery = await call(app, 'delivery.get', {'order_id': order['id']}, + key=bk, subject=buyer, contract_version=2) + assert delivery.status == 'ok', wire(delivery) + data = delivery.data['delivery'] + assert data['state'] == 'prepared' and data['claimed_at'] is None + assert data['manifest']['text'] == 'msg.lmm.best store selftest' + assert unb64(data['payloads'][0]['data']) == b'delivery-ok\n' + async with app.metadata.transaction(write=False) as tx: + escrow = tx.one('SELECT escrow_subject FROM store_orders WHERE id=?', (order['id'],))[0] + from msg.plugins.money import _balance + assert _balance(tx, buyer) == 15_000_000 + assert _balance(tx, seller) == 5_000_000 + assert _balance(tx, escrow) == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?', (order['id'],))[0] == 1 + count = tx.one('SELECT COUNT(*) FROM money_ledger')[0] + for _ in range(2): + claimed = await call(app, 'delivery.accept', {'order_id': order['id'], + 'delivery_digest': data['delivery_digest']}, key=bk, subject=buyer, contract_version=2) + assert claimed.status == 'ok', wire(claimed) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM money_ledger')[0] == count + + +@pytest.mark.asyncio +async def test_created_payment_snapshot_and_cancel_before_fund(installed): + app, root = installed + sk, seller, bk, buyer, listing, _ = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + assert created.status == 'ok', wire(created) + order = created.data['order'] + assert order['state'] == 'created' and order['funded_at'] is None + changed = await call(app, 'store.listing_update', { + 'id': listing['listing_id'], 'price_minor': 7_000_000, 'terms': 'Later terms'}, + key=sk, subject=seller, expected=((listing['listing_id'], 2),)) + assert changed.status == 'ok', wire(changed) + funded = await call(app, 'orders.fund', {'order_id': order['id'], + 'order_digest': order['order_digest'], 'total_price_minor': 5_000_000, + 'currency_id': 'primary'}, key=bk, subject=buyer) + assert funded.status == 'ok' and funded.data['order']['state'] == 'settled', wire(funded) + assert funded.data['order']['listing_revision'] == listing['listing_revision'] + assert funded.data['order']['terms_digest'] == listing['terms_digest'] + other = await call(app, 'orders.create', _intent(changed.data['listing']), key=bk, subject=buyer) + cancelled = await call(app, 'orders.cancel', {'order_id': other.data['order']['id']}, + key=bk, subject=buyer, contract_version=2) + assert cancelled.status == 'ok' and cancelled.data['order']['state'] == 'cancelled' + + +@pytest.mark.asyncio +async def test_objective_missing_package_refunds_atomically(installed): + app, root = installed + _sk, seller, bk, buyer, listing, package = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + order = created.data['order'] + async with app.metadata.transaction(write=True) as tx: + tx.execute('DELETE FROM store_packages WHERE id=?', (package['id'],), write=True) + funded = await call(app, 'orders.fund', {'order_id': order['id'], + 'order_digest': order['order_digest'], 'total_price_minor': 5_000_000, + 'currency_id': 'primary'}, key=bk, subject=buyer) + assert funded.status == 'ok' and funded.data['order']['state'] == 'refunded', wire(funded) + async with app.metadata.transaction(write=False) as tx: + from msg.plugins.money import _balance + assert _balance(tx, buyer) == 20_000_000 and _balance(tx, seller) == 0 + assert tx.one('SELECT COUNT(*) FROM store_deliveries')[0] == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0] == 1 + + +@pytest.mark.asyncio +async def test_last_quantity_and_timeout_do_not_double_spend(installed): + app, root = installed + _sk, seller, bk, buyer, listing, _ = await market(app, root, mode='service', kind='service', quantity=1) + results = await asyncio.gather(*(buy(app, bk, buyer, listing) for _ in range(2))) + assert sum(r.status == 'ok' for r in results) == 1 + assert [r.error.code for r in results if r.error] == ['quantity_unavailable'] + order = next(r.data['order'] for r in results if r.status == 'ok') + assert order['state'] == 'funded' + app.clock = lambda: NOW + timedelta(days=2) + from msg.market.escrow import resolve_due + await resolve_due(app) + await resolve_due(app) + async with app.metadata.transaction(write=False) as tx: + from msg.plugins.money import _balance + assert tx.one('SELECT state FROM store_orders WHERE id=?', (order['id'],))[0] == 'refunded' + assert _balance(tx, buyer) == 20_000_000 and _balance(tx, seller) == 0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0] == 1 diff --git a/tests/test_market_notification_reuse.py b/tests/test_market_notification_reuse.py new file mode 100644 index 00000000..f0c62e4b --- /dev/null +++ b/tests/test_market_notification_reuse.py @@ -0,0 +1,193 @@ +"""Shared notification boundaries, with job facts separate from goods/settlement.""" +from copy import deepcopy +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest + +from msg.core.codec import canonical, wire +from msg.core.errors import Failure +from msg.core.models import EffectJob, Principal +from msg.market import targets +from msg.market import delivery_targets + +NOW = datetime(2026, 9, 27, tzinfo=UTC) +BUYER, ORDER, ENDPOINT = 'u_buyer', 'ord_notification', 'em_current' + + +def app_for(origin='https://msg.example.invalid', enabled=True): + return SimpleNamespace(settings=SimpleNamespace(service_url=origin, + server=SimpleNamespace(mail=SimpleNamespace(enabled=enabled)))) + + +class ReadSession: + """Unit-only read adapter; integration cases below use the real application.""" + def __init__(self, job=None, code=None): + self.job = job + self.code = code + self.reads = [] + + async def resource(self, subject): + assert subject == BUYER + return SimpleNamespace(name='@buyer') + + def one(self, sql, parameters=()): + self.reads.append((sql, parameters)) + if 'FROM jobs' in sql: + assert parameters == ('market-mail:' + ORDER + ':' + ENDPOINT,) + return (canonical(self.job).decode(),) if self.job else None + return None + + def setting(self, name, default=None): + assert name == 'job_status:' + self.job.id + return {'code': self.code} if self.code is not None else default + + +def order_for(*, state='queued', disabled=False): + return {'id': ORDER, 'buyer': BUYER, + 'delivery_target': {'subject_id': BUYER, 'channel': 'site', + 'email': {'subject_id': BUYER, 'endpoint_id': ENDPOINT, + 'address_snapshot': 'buyer@example.invalid', 'state': state, + **({'notification_disabled': True} if disabled else {})}}} + + +def job_for(state, *, attempts=1, subject=BUYER, endpoint=ENDPOINT): + principal = Principal(actor=subject, subject=subject, credential_id=None, + method='signature', certificates=(), ceiling=()) + return EffectJob(id='job_notice', event_id='delivery:' + ORDER, kind='market_mail', + dedupe_key='market-mail:' + ORDER + ':' + endpoint, principal=principal, + operation='orders.buy', arguments={'order_id': ORDER, 'endpoint_id': endpoint}, + state=state, attempts=attempts, next_attempt_at=NOW, + lease_until=NOW + timedelta(seconds=120) if state == 'running' else None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('codepoint', list(range(32)) + [127]) +async def test_unit_checkout_rejects_control_characters_like_legacy(codepoint): + address = 'buyer' + chr(codepoint) + '@example.invalid' + with pytest.raises(Failure) as legacy: + delivery_targets.validate_address(address) + assert legacy.value.code == 'invalid_email' + with pytest.raises(Failure) as current: + await targets.target_for(app_for(), ReadSession(), BUYER, address) + assert current.value.code == 'invalid_email' + + +@pytest.mark.parametrize('state,code,expected', [ + ('pending', None, 'queued'), ('running', None, 'queued'), + ('pending', 'mail_connection_failed', 'pending'), + ('done', 'sent', 'smtp_accepted'), ('done', 'mail_disabled', 'disabled'), + ('failed', 'credential_expired', 'failed'), + ('uncertain', 'expired_execution_lease', 'delivered_unknown'), +]) +def test_unit_projection_reads_job_not_stale_order_state(state, code, expected): + order = order_for(state='smtp_accepted') + before = deepcopy(order) + tx = ReadSession(job_for(state), code) + view = targets.notification_view(tx, order) + assert view['state'] == expected + assert view['endpoint_id'] == ENDPOINT + assert order == before # A GET must not rewrite the order's stored projection. + + +def test_unit_disabled_endpoint_wins_without_reactivating_old_job(): + tx = ReadSession(job_for('done'), 'sent') + assert targets.notification_view(tx, order_for(disabled=True))['state'] == 'disabled' + assert tx.reads == [] + + +def test_unit_missing_job_cannot_claim_sent_from_cached_order(): + assert targets.notification_view(ReadSession(), order_for(state='smtp_accepted'))['state'] == 'pending' + + +@pytest.mark.parametrize('change', ['subject', 'endpoint', 'order', 'kind']) +def test_unit_projection_rejects_cross_order_or_cross_subject_job(change): + job = job_for('done') + if change == 'subject': + job = job_for('done', subject='u_other') + elif change == 'endpoint': + job = job_for('done', endpoint='em_other') + elif change == 'order': + job = replace(job, arguments={**job.arguments, 'order_id': 'ord_other'}) + else: + job = replace(job, kind='market_email_verify') + with pytest.raises(Failure) as caught: + targets.notification_view(ReadSession(job, 'sent'), order_for()) + assert caught.value.code == 'delivery_recipient_mismatch' + + +@pytest.mark.parametrize('origin', [ + 'javascript:alert(1)', 'https://name:secret@example.invalid', + 'https://msg.example.invalid?token=secret', 'https://msg.example.invalid#secret', + 'https://', 'https://[broken', 'https://msg.example.invalid\r\nBcc: other', + 'https://msg.example.invalid\x00', 'https://msg.example.invalid\\@other.invalid', +]) +def test_unit_pickup_origin_fails_closed_without_echo(origin): + with pytest.raises(Failure) as caught: + delivery_targets.pickup_url(app_for(origin), ORDER) + assert caught.value.code == 'invalid_delivery_origin' + assert 'secret' not in str(caught.value) + + +@pytest.mark.parametrize('origin', ['https://msg.example.invalid', + 'http://localhost:8099', 'https://msg.example.invalid/prefix/']) +def test_unit_pickup_link_preserves_plain_authenticated_route(origin): + assert delivery_targets.pickup_url(app_for(origin), ORDER) == ( + origin.rstrip('/') + '/_orders/' + ORDER + '/_delivery') + + +@pytest.mark.asyncio +async def test_integration_market_completion_only_changes_job_facts(installed): + from test_market_delivery import Sender, drain, enable_mail, verified_address + from test_market_lifecycle import buy, market + from test_service import call + + app, root = installed + enable_mail(app) + _sk, _seller, key, buyer, listing, _ = await market(app, root) + await verified_address(app, key, buyer, 'buyer@example.invalid') + sender = Sender() + await drain(app, sender) + sender.messages.clear() + bought = await buy(app, key, buyer, listing, email='buyer@example.invalid') + assert bought.status == 'ok', wire(bought) + oid = bought.data['order']['id'] + tables = ('store_orders', 'store_deliveries', 'money_ledger', 'order_contracts') + async with app.metadata.transaction(write=False) as tx: + before = {t: list(tx.execute('SELECT * FROM ' + t + ' ORDER BY 1')) for t in tables} + await drain(app, sender) + assert len(sender.messages) == 1 + got = await call(app, 'orders.get', {'order_id': oid}, key=key, subject=buyer) + assert got.status == 'ok', wire(got) + assert got.data['order']['email_status'] == 'smtp_accepted' + assert got.data['order']['delivery_target']['email']['state'] == 'smtp_accepted' + async with app.metadata.transaction(write=False) as tx: + assert {t: list(tx.execute('SELECT * FROM ' + t + ' ORDER BY 1')) for t in tables} == before + assert tx.one('SELECT state,claimed_at FROM store_deliveries WHERE order_id=?', (oid,)) == ('prepared', None) + + +@pytest.mark.asyncio +async def test_integration_lifecycle_email_uses_same_origin_guard(installed): + from test_market_delivery import Sender, drain, enable_mail, verified_address + from test_market_lifecycle import buy, market + from msg.workers.effects import EffectWorker + + app, root = installed + enable_mail(app) + _sk, _seller, key, buyer, listing, _ = await market(app, root) + await verified_address(app, key, buyer, 'buyer@example.invalid') + sender = Sender() + await drain(app, sender) + sender.messages.clear() + bought = await buy(app, key, buyer, listing, email='buyer@example.invalid') + assert bought.status == 'ok', wire(bought) + worker = EffectWorker(app, mail_sender=sender) + job, execute = await worker._claim() + assert execute and job.kind == 'market_mail' + app.settings = replace(app.settings, service_url='https://secret@example.invalid') + async with app.metadata.transaction(write=False) as tx: + with pytest.raises(Failure) as caught: + await targets.render_notification(app, tx, job) + assert caught.value.code == 'invalid_delivery_origin' + assert sender.messages == [] diff --git a/tests/test_market_paths.py b/tests/test_market_paths.py new file mode 100644 index 00000000..6229d171 --- /dev/null +++ b/tests/test_market_paths.py @@ -0,0 +1,89 @@ +"""Published pickup/payment URLs remain private read-only lifecycle projections.""" +import httpx +import pytest +from test_market_delivery import Sender, drain, enable_mail, verified_address +from test_market_lifecycle import buy, market +from test_order_paths import _headers +from test_orders import _intent +from test_service import call, register + +from msg.core.codec import loads, unb64, wire +from msg.transports.http import create_app + + +async def snapshot(app): + async with app.metadata.transaction(write=False) as tx: + return {table: tx.rows(f'SELECT * FROM {table} ORDER BY 1') for table in ( + 'store_orders', 'store_deliveries', 'money_ledger', 'events', 'audit')} + + +@pytest.mark.asyncio +async def test_emailed_pickup_url_reads_new_delivery_without_claiming(installed): + app, root = installed + enable_mail(app) + sk, seller, bk, buyer, listing, _ = await market(app, root) + ok, outsider, _ = await register(app, 'order-outsider') + await verified_address(app, bk, buyer, 'buyer@example.invalid') + bought = await buy(app, bk, buyer, listing, email='buyer@example.invalid') + assert bought.status == 'ok', wire(bought) + oid = bought.data['order']['id'] + sender = Sender() + await drain(app, sender) + assert len(sender.messages) == 1 + message = loads(sender.messages[0]['text']) + assert message['order_id'] == oid + before = await snapshot(app) + args = {'order_id': oid} + headers = _headers(app, 'delivery.get', args, bk, buyer) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + response = await http.get(message['pickup'], headers=headers) + assert response.status_code == 200, response.text + delivery = response.json()['delivery'] + assert delivery['order_id'] == oid and delivery['state'] == 'prepared' + assert delivery['claimed_at'] is None + assert unb64(delivery['payloads'][0]['data']) == b'delivery-ok\n' + alias = f'/@order-buyer/orders/{oid}/_delivery' + assert (await http.get(alias, headers=headers)).json() == response.json() + head = await http.head(message['pickup'], headers=headers) + assert head.status_code == 200 and head.headers['etag'] == response.headers['etag'] + for key, subject in ((None, None), (sk, seller), (ok, outsider)): + auth = {} if key is None else _headers(app, 'delivery.get', args, key, subject) + denied = await http.get(message['pickup'], headers=auth) + missing_id = 'ord_' + 'z' * 32 + missing_auth = {} if key is None else _headers(app, 'delivery.get', + {'order_id': missing_id}, key, subject) + missing = await http.get(f'/_orders/{missing_id}/_delivery', headers=missing_auth) + assert denied.status_code in {403, 404} and denied.status_code == missing.status_code + assert await snapshot(app) == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize('cancelled', [False, True]) +async def test_unfunded_payment_url_has_no_fabricated_receipt_or_payment(installed, cancelled): + app, root = installed + sk, seller, bk, buyer, listing, _ = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer) + assert created.status == 'ok', wire(created) + oid = created.data['order']['id'] + if cancelled: + result = await call(app, 'orders.cancel', {'order_id': oid}, key=bk, + subject=buyer, contract_version=2) + assert result.status == 'ok', wire(result) + before = await snapshot(app) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + for key, subject in ((bk, buyer), (sk, seller)): + response = await http.get(f'/_orders/{oid}/_payment', headers=_headers( + app, 'orders.payment', {'order_id': oid}, key, subject)) + assert response.status_code == 200, response.text + payment = response.json()['payment'] + assert payment['status'] == 'pending' + assert payment['amount_minor'] == 5_000_000 + assert payment['currency_id'] == 'primary' + if subject == buyer: + assert payment['receipt'] is None and payment['receipts'] == [] + assert payment['escrow_balance_minor'] == 0 + else: + assert 'receipts' not in payment and 'escrow_balance_minor' not in payment + assert await snapshot(app) == before diff --git a/tests/test_market_rationale.py b/tests/test_market_rationale.py new file mode 100644 index 00000000..21e0c6be --- /dev/null +++ b/tests/test_market_rationale.py @@ -0,0 +1,173 @@ +"""The signed decision must commit to one immutable, case-private explanation.""" +import pytest +from test_market_arbitration import configured, details, vote +from test_service import call + +from msg.core.codec import b64, canonical, digest, unb64, wire +from msg.plugins.money import _balance + + +async def unchanged_money(app, order, buyer, seller): + async with app.metadata.transaction(write=False) as tx: + account=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(order['id'],))[0] + assert _balance(tx,account)==5_000_000 + assert _balance(tx,buyer)==15_000_000 and _balance(tx,seller)==0 + assert tx.one('SELECT COUNT(*) FROM order_settlements')[0]==0 + + +async def source_file(app, key, subject, raw, *, media='text/plain'): + from uuid import uuid4 + async with app.metadata.transaction(write=False) as tx: + parent=tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(subject,))[0] + result=await call(app,'content.file_put',{'parent':parent,'name':'reason-'+uuid4().hex+'.txt', + 'data':b64(raw),'media_type':media},key=key,subject=subject) + assert result.status=='ok',wire(result) + return result + + +async def bound(app, members, opened, raw=b'Only the specified work was delivered.'): + member=opened['panel'][0] + source=await source_file(app,members[member],member,raw) + args={'case_id':opened['case_id'],'ref':wire(source.resources[0])} + result=await call(app,'orders.dispute_rationale',args,key=members[member],subject=member) + assert result.status=='ok',wire(result) + base=dict((await details(app,members[member],member,opened['case_id']))['proposal_base']) + base.update({k:result.data[k] for k in ('rationale_ref','rationale_digest')}) + return source,result.data,base + + +@pytest.mark.asyncio +async def test_signed_allocation_without_rationale_is_rejected(installed): + app,_root,members,_sk,seller,bk,buyer,order,opened=await configured(installed,panel_size=1,count=1) + member=opened['panel'][0] + base=(await details(app,bk,buyer,opened['case_id']))['proposal_base'] + result=await vote(app,members[member],member,base) + assert result.status=='error', 'An allocation without an immutable rationale must never form a decision' + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM arbitration_votes')[0]==0 + assert tx.one('SELECT COUNT(*) FROM arbitration_decisions')[0]==0 + await unchanged_money(app,order,buyer,seller) + + +@pytest.mark.asyncio +async def test_rationale_is_private_pinned_and_survives_source_edit(installed): + app,_root,members,sk,seller,bk,buyer,_order,opened=await configured(installed,panel_size=1,count=2) + original=b'Only the specified work was delivered.' + source,binding,base=await bound(app,members,opened,original) + cid,member=opened['case_id'],opened['panel'][0] + # Repeated explicit deposits do not duplicate evidence or alter the pinned fact. + repeated=await call(app,'orders.dispute_rationale',{'case_id':cid, + 'ref':wire(source.resources[0])},key=members[member],subject=member) + assert repeated.status=='ok' and repeated.data==binding,wire(repeated) + changed=await call(app,'content.text_patch',{'id':source.resources[0].id, + 'base_revision':source.resources[0].revision,'exact':original.decode(), + 'replacement':'A later draft is not the signed rationale.'},key=members[member],subject=member, + expected=((source.resources[0].id,source.data['generation']),)) + assert changed.status=='ok',wire(changed) + args={'case_id':cid,'evidence_id':binding['evidence_id']} + for key,subject in ((bk,buyer),(sk,seller)): + view=await call(app,'orders.dispute_evidence_get',args,key=key,subject=subject) + assert view.status=='ok' and unb64(view.data['data'])==original,wire(view) + forbidden=await call(app,'discovery.get',{'id':source.resources[0].id},key=key,subject=subject) + assert forbidden.status=='error' # The case does not share the author's file tree. + outsider=next(uid for uid in members if uid!=member) + denied=await call(app,'orders.dispute_evidence_get',args,key=members[outsider],subject=outsider) + missing=await call(app,'orders.dispute_evidence_get',{'case_id':'unknown', + 'evidence_id':binding['evidence_id']},key=members[outsider],subject=outsider) + assert denied.error.code==missing.error.code=='case_not_found' + public=await call(app,'orders.dispute_summary',{'case_id':cid}) + assert original.decode() not in str(wire(public)) + assert binding['rationale_digest'] not in str(wire(public)) + cast=await vote(app,members[member],member,base) + assert cast.status=='ok',wire(cast) + decision=cast.data['decision'] + assert decision['rationale_ref']==wire(source.resources[0]) + assert decision['rationale_digest']==digest(original) + executed=await call(app,'orders.dispute_execute',{'case_id':cid, + 'decision_id':decision['id']},key=bk,subject=buyer) + assert executed.status=='ok',wire(executed) + + +@pytest.mark.asyncio +async def test_valid_signature_cannot_be_reused_with_another_rationale(installed): + app,_root,members,_sk,seller,_bk,buyer,order,opened=await configured(installed,panel_size=1,count=1) + member=opened['panel'][0] + _source,_binding,first=await bound(app,members,opened,b'First proposed explanation.') + _source,_binding,second=await bound(app,members,opened,b'A different explanation.') + original={**first,'refund_minor':2_000_000,'outcome':'split'} + altered={**second,'refund_minor':2_000_000,'outcome':'split'} + signature=members[member].sign(canonical(original),purpose='arbitration-decision') + result=await call(app,'orders.dispute_vote',{'proposal':altered,'signature':wire(signature)}, + key=members[member],subject=member) + assert result.error.code=='invalid_signature',wire(result) + await unchanged_money(app,order,buyer,seller) + + +@pytest.mark.asyncio +async def test_rationale_from_another_case_does_not_authorize_a_vote(installed): + app,_root,members,_sk,seller,bk,buyer,order,opened=await configured(installed,panel_size=1,count=1) + member=opened['panel'][0] + _source,_binding,first=await bound(app,members,opened) + # The frozen listing snapshot supplies the same advertised checkout terms. + async with app.metadata.transaction(write=False) as tx: + from msg.market.policy import contract + locked=contract(tx,order['id']) + purchased=await call(app,'orders.buy',{'listing_id':order['listing_id'], + 'listing_revision':order['listing_revision'],'quantity':1, + 'currency_id':'primary','total_price_minor':locked['total_price_minor']}, + key=bk,subject=buyer,contract_version=3) + assert purchased.status=='ok',wire(purchased) + other=await call(app,'orders.dispute_open',{'order_id':purchased.data['order']['id'], + 'reason':'quality'},key=bk,subject=buyer) + assert other.status=='ok',wire(other) + base=dict((await details(app,bk,buyer,other.data['case_id']))['proposal_base']) + base.update({k:first[k] for k in ('rationale_ref','rationale_digest')}) + result=await vote(app,members[member],member,base) + assert result.error.code=='decision_rationale_missing',wire(result) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM arbitration_votes')[0]==0 + assert _balance(tx,seller)==0 and _balance(tx,buyer)==10_000_000 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mutation',['missing_bytes','wrong_digest']) +async def test_unavailable_or_mismatched_reason_cannot_release_escrow(installed,mutation): + app,_root,members,_sk,seller,bk,buyer,order,opened=await configured(installed,panel_size=1,count=1) + member=opened['panel'][0] + _source,binding,base=await bound(app,members,opened) + if mutation=='wrong_digest': + base['rationale_digest']=digest(b'Not the deposited reason.') + rejected=await vote(app,members[member],member,base) + assert rejected.error.code=='decision_rationale_mismatch',wire(rejected) + else: + cast=await vote(app,members[member],member,base) + assert cast.status=='ok',wire(cast) + # Exercise actual referenced-content loss, not a pretend worker exception. + (app.contents.index / binding['rationale_digest'][7:]).unlink() + rejected=await call(app,'orders.dispute_execute',{'case_id':opened['case_id'], + 'decision_id':cast.data['decision']['id']},key=bk,subject=buyer) + assert rejected.error.code=='decision_rationale_missing',wire(rejected) + from msg.market.arbitration import resolve_cases + assert await resolve_cases(app)==[opened['case_id']] + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT state FROM arbitration_cases WHERE id=?',(opened['case_id'],))[0]=='held' + await unchanged_money(app,order,buyer,seller) + + +@pytest.mark.asyncio +async def test_only_current_panel_can_deposit_owned_nonempty_text(installed): + app,_root,members,_sk,_seller,bk,buyer,_order,opened=await configured(installed,panel_size=1,count=1) + member,cid=opened['panel'][0],opened['case_id'] + buyer_file=await source_file(app,bk,buyer,b'A party is not the deciding panel.') + args={'case_id':cid,'ref':wire(buyer_file.resources[0])} + party=await call(app,'orders.dispute_rationale',args,key=bk,subject=buyer) + assert party.error.code=='arbitrator_required',wire(party) + stolen=await call(app,'orders.dispute_rationale',args,key=members[member],subject=member) + assert stolen.error.code=='rationale_not_owned',wire(stolen) + for raw,media in ((b' \n','text/plain'),(b'\xff','text/plain'),(b'explanation','application/octet-stream')): + source=await source_file(app,members[member],member,raw,media=media) + invalid=await call(app,'orders.dispute_rationale',{'case_id':cid, + 'ref':wire(source.resources[0])},key=members[member],subject=member) + assert invalid.error.code=='rationale_text_required',wire(invalid) + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM arbitration_evidence')[0]==0 diff --git a/tests/test_market_recovery.py b/tests/test_market_recovery.py new file mode 100644 index 00000000..843e614b --- /dev/null +++ b/tests/test_market_recovery.py @@ -0,0 +1,97 @@ +"""Real restart/pg_dump/restore and readonly diagnostics for private market state.""" +import asyncio + +import pytest +from test_market_arbitration import configured, details, reasoned_base, vote +from test_service import NOW, call + +from msg.admin.backups import backup, restore +from msg.admin.market_check import inspect_market +from msg.application import Application +from msg.config import load_settings +from msg.core.codec import unb64, wire +from msg.core.errors import Failure +from msg.workers.effects import EffectWorker + + +async def snapshot(tx): + tables=('store_orders','order_contracts','order_settlements','order_transitions', + 'arbitration_cases','arbitration_votes','arbitration_decisions','arbitration_evidence', + 'money_ledger','jobs','events','audit','results','settings') + return {name:tx.rows('SELECT * FROM '+name+' ORDER BY 1') for name in tables} + + +@pytest.mark.asyncio +async def test_restart_and_backup_preserve_private_case_decision_without_reexecution(installed,tmp_path,pg_dsn): + app,_root,members,sk,seller,bk,buyer,_order,opened=await configured(installed) + cid=opened['case_id']; text='Private sealed dispute evidence. Never public.' + stored=await call(app,'orders.dispute_statement',{'case_id':cid,'visibility':'panel','text':text},key=bk,subject=buyer) + assert stored.status=='ok',wire(stored) + eid=stored.data['evidence_id'] + arb=opened['panel'][0] + reason=await reasoned_base(app,members[arb],arb,cid) + data=await details(app,bk,buyer,cid) + reason_id=next(row['id'] for row in data['evidence'] if row['kind']=='rationale') + for uid in opened['panel'][:2]: + cast=await vote(app,members[uid],uid,reason) + assert cast.status=='ok' and cast.data['decision'],wire(cast) + args={'case_id':cid,'decision_id':cast.data['decision']['id']} + # A new Application reads the same authoritative metadata; no in-memory panel. + restarted=Application(app.settings,clock=lambda:NOW) + await restarted.load() + try: + result=await call(restarted,'orders.dispute_execute',args,key=bk,subject=buyer) + assert result.status=='ok',wire(result) + finally: + await restarted.close() + archive=tmp_path/'market.zip' + await backup(app,archive) + async with app.metadata.transaction(write=False) as tx: + original=await snapshot(tx) + report=await inspect_market(tx) + assert report['orders']==1 + assert await snapshot(tx)==original + restored_config=tmp_path/'restored-etc'; restored_data=tmp_path/'restored-data' + await asyncio.to_thread(restore,archive,restored_config,restored_data,postgres_dsn=pg_dsn) + restored=Application(load_settings(restored_config),clock=lambda:NOW) + await restored.load() + try: + async with restored.metadata.transaction(write=False) as tx: + await inspect_market(tx) + # Recovery intentionally changes safety settings, not the ledger/case. + after=await snapshot(tx) + for table in set(original)-{'settings','results','events','audit'}: + assert after[table]==original[table],table + evidence=await call(restored,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=bk,subject=buyer) + assert evidence.status=='ok' and unb64(evidence.data['data']).decode()==text,wire(evidence) + other=await call(restored,'orders.dispute_evidence_get',{'case_id':cid,'evidence_id':eid},key=sk,subject=seller) + assert other.error.code=='evidence_not_found' + rationale=await call(restored,'orders.dispute_evidence_get', + {'case_id':cid,'evidence_id':reason_id},key=sk,subject=seller) + assert rationale.status=='ok',wire(rationale) + assert rationale.data['digest']==reason['rationale_digest'] + assert unb64(rationale.data['data'])==b'The signed allocation follows the case evidence.' + assert await EffectWorker(restored).run_once() is False + rejected=await call(restored,'orders.dispute_execute',args,key=bk,subject=buyer) + assert rejected.error.code=='writes_paused' + finally: + await restored.close() + + +@pytest.mark.asyncio +async def test_market_doctor_cannot_mutate_or_synthesize_a_repair(installed): + from test_market_lifecycle import buy, market + app,root=installed + _sk,_seller,bk,buyer,listing,_=await market(app,root) + result=await buy(app,bk,buyer,listing) + assert result.status=='ok' + async with app.metadata.transaction(write=False) as tx: + before=await snapshot(tx) + assert (await inspect_market(tx))['escrow_invariants'] + assert await snapshot(tx)==before + # Snapshot/settlement facts are append-only, even to a privileged DB session. + with pytest.raises(Failure,match='constraint_conflict'): + async with app.metadata.transaction(write=True) as tx: + tx.execute("UPDATE order_contracts SET digest='forged'",write=True) + async with app.metadata.transaction(write=False) as tx: + assert await snapshot(tx)==before diff --git a/tests/test_recovery_safety_units.py b/tests/test_recovery_safety_units.py new file mode 100644 index 00000000..1eda12bf --- /dev/null +++ b/tests/test_recovery_safety_units.py @@ -0,0 +1,272 @@ +"""Cryptographic and transaction-boundary units; PostgreSQL drills are separate.""" +from copy import deepcopy +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path +import subprocess + +import pytest + +from msg.constants import ROOT_SUBJECT, ROOT_SPACE, ONLINE_CA +from msg.core.codec import b64, canonical, digest, wire +from msg.core.errors import Failure +from msg.core.models import (Certificate, CertificateRequest, CapabilityGrant, Credential, + IssuancePolicy, Scope, Signature) +from msg.security.age_keys import generate_age_key, encryption_key_id, public_from_recipient +from msg.security.certificates import sign_certificate, csr_body +from msg.security.crypto import Ed25519Signer, seal_private_key +from msg.security.custody_history import history_evidence, retirement_evidence +from msg.security.quarantine import SETTING, active, require_live_authority +from msg.security.rotation_journal import authorization, validate +from msg.storage.sqlite import FakeMetadataStore + +NOW = datetime(2026, 9, 27, tzinfo=UTC) +PIN = 'isolated-rotation-unit-passphrase' + + +@pytest.fixture(scope='module') +def rotation_bundle(): + old, new, online = (Ed25519Signer.generate() for _ in range(3)) + grant = CapabilityGrant(capability='cert.issue', version=1, + scope=Scope(resource_id=ROOT_SPACE, descendants=True), + operations=frozenset({'cert.publish@1'}), constraints={}) + policy = IssuancePolicy(issue_grants=(grant,), max_cert_ttl_seconds=86400, + max_child_ca_depth=3, max_delegation_depth=8) + def certificate(signer, cid): + return sign_certificate(Certificate(resource_id=cid, serial=cid, subject_id=ROOT_SUBJECT, + key_id=signer.key_id, issuer_id=ROOT_SUBJECT, parent_certificate_id=None, + authority_sources=(), kind='ca', grants=(grant,), not_before=NOW, + expires_at=NOW+timedelta(days=2), target_service='http://testserver', + delegation_depth=8, issuance=policy, + signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b'')), signer) + old_cert, new_cert = certificate(old, 'cert_old'), certificate(new, 'cert_new') + csr = CertificateRequest(resource_id='csr_online', applicant=ONLINE_CA, subject_id=ONLINE_CA, + requested_issuer=ROOT_SUBJECT, public_key=online.public_key, kind='ca', grants=(grant,), + issuance=replace(policy, max_child_ca_depth=0), requested_ttl_seconds=86400, + target_service='http://testserver', delegation_depth=0, authority_sources=(), + request_digest='', possession_proof=Signature(key_id=online.key_id, algorithm='ed25519', value=b'')) + csr = replace(csr, request_digest=digest(csr_body(csr)), + possession_proof=online.sign(canonical(csr_body(csr)), purpose='csr')) + journal = {'version':2, 'old_certificate':wire(old_cert), + 'new_trust':{'version':1, 'public_key':b64(new.public_key), 'certificate':wire(new_cert)}, + 'new_envelope':seal_private_key(new.private_bytes(), PIN), + 'previous_envelope_digest':None, 'online_csr':wire(csr), + 'statement':{'old_certificate':old_cert.resource_id, 'old_fingerprint':digest(old.public_key), + 'new_certificate':new_cert.resource_id, 'new_fingerprint':digest(new.public_key), + 'time':wire(NOW), 'operator':'isolated-unit', 'previous_key_proved':True}} + proof = canonical(authorization(journal)) + journal.update(new_signature=wire(new.sign(proof, purpose='root-rotation')), + old_signature=wire(old.sign(proof, purpose='root-rotation'))) + return journal, old, new, online + + +def test_rotation_commitment_binds_all_installation_targets(rotation_bundle): + journal, _, new, online = rotation_bundle + signer, old_cert, new_cert, csr, proof = validate(journal, pin=PIN, + service='http://testserver', online_public=online.public_key) + assert signer.public_key == new.public_key and old_cert.resource_id != new_cert.resource_id + assert csr.public_key == online.public_key and proof == authorization(journal) + assert 'new_envelope' not in proof and new.private_bytes() not in canonical(journal) + + +@pytest.mark.parametrize('mutation', [ + lambda j: j['online_csr'].update(resource_id='csr_substituted'), + lambda j: j['new_trust']['certificate'].update(serial='substituted'), + lambda j: j['old_certificate'].update(serial='substituted'), + lambda j: j.update(previous_envelope_digest=digest(b'substituted')), + lambda j: j['statement'].update(previous_key_proved=False), + lambda j: j['statement'].update(new_fingerprint=digest(b'substituted')), + lambda j: j.update(old_signature=None), + lambda j: j.update(unexpected_field='not committed'), +]) +def test_rotation_rejects_journal_substitution(rotation_bundle, mutation): + journal, _, _, online = rotation_bundle + changed = deepcopy(journal) + mutation(changed) + with pytest.raises(Failure): + validate(changed, pin=PIN, service='http://testserver', online_public=online.public_key) + + +def test_legacy_unbound_rotation_is_not_silently_resumed(rotation_bundle): + journal, _, _, online = rotation_bundle + with pytest.raises(Failure, match='rotation_journal_upgrade_required'): + validate({**journal, 'version':1}, pin=PIN, service='http://testserver', + online_public=online.public_key) + + +def test_rotation_requires_same_online_key_and_service(rotation_bundle): + journal, _, _, online = rotation_bundle + with pytest.raises(Failure, match='rotation_online_csr_mismatch'): + validate(journal, pin=PIN, service='http://testserver', + online_public=Ed25519Signer.generate().public_key) + with pytest.raises(Failure, match='rotation_service_mismatch'): + validate(journal, pin=PIN, service='http://another-service', online_public=online.public_key) + + +@pytest.fixture +def history_bundle(): + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + challenge = {'subject_id':'u_history', 'challenge_id':'cupg_history', + 'public_key':b64(signer.public_key), 'encryption_recipient':recipient} + old = {'id':'r_old', 'revision':'v_old', 'ciphertext_digest':digest(b'old ciphertext')} + new = {'id':'r_new', 'revision':'v_new', 'ciphertext_digest':digest(b'new ciphertext')} + details = {'old_identity_key_id':'k_old', 'new_identity_key_id':signer.key_id, + 'old_encryption_key_id':'e_old', + 'new_encryption_key_id':encryption_key_id(public_from_recipient(recipient)), + 'age_inventory':[old], 'rewrap_mappings':{'v_old':{ + 'old':old, 'new':new, 'recipient':recipient}}, 'rewrap_acks':{}} + signed = {'subject_id':'u_history', 'challenge_id':'cupg_history', 'old':old, 'new':new, + 'plaintext_digest':digest(b'client declares decryption'), 'request_id':'ack-one'} + details['rewrap_acks']['v_old'] = {'old_revision':'v_old', 'new_revision':'v_new', + 'ciphertext_digest':new['ciphertext_digest'], 'plaintext_digest':signed['plaintext_digest'], + 'signature':wire(signer.sign(canonical(signed), purpose='custodial-rewrap-ack')), + 'request_id':signed['request_id']} + return challenge, details, sorted([old, new], key=lambda x:(x['id'], x['revision'])) + + +def evidence(bundle): + return history_evidence('u_history', 'cupg_history', *bundle) + + +def test_ack_is_reverified_not_just_counted(history_bundle): + result = evidence(history_bundle) + assert result['known_ciphertexts_migrated'] and result['verified_acked_count'] == 1 + assert result['verification_scope'] == 'enumerated_keystore_age_revisions' + assert result['external_coverage'] == 'unknown' + + +@pytest.mark.parametrize('mutation', [ + lambda d: d['rewrap_acks']['v_old'].update(plaintext_digest=digest(b'changed')), + lambda d: d['rewrap_acks']['v_old'].update(request_id='another-request'), + lambda d: d['rewrap_acks']['v_old'].update(old_revision='another-source'), + lambda d: d['rewrap_acks']['v_old'].update(signature={}), + lambda d: d['rewrap_mappings']['v_old'].update(old_encryption_key_id='other-key'), + lambda d: d['rewrap_mappings']['v_old'].update(new_encryption_key_id='other-key'), + lambda d: d['rewrap_mappings']['v_old'].update(recipient=generate_age_key()[1]), + lambda d: d.update(new_identity_key_id='other-signer'), + lambda d: d.update(new_encryption_key_id='other-key'), + lambda d: d.update(age_inventory=None), + lambda d: d.update(rewrap_acks=[]), + lambda d: d.update(rewrap_mappings=[]), +]) +def test_bad_history_evidence_remains_pending(history_bundle, mutation): + changed = deepcopy(history_bundle) + mutation(changed[1]) + result = evidence(changed) + assert not result['known_ciphertexts_migrated'] + assert result['historical_revisions_require_migration'] + + +def test_inventory_delta_invalidates_previously_verified_acks(history_bundle): + challenge, details, current = history_bundle + current = sorted(current+[{'id':'r_late', 'revision':'v_late', + 'ciphertext_digest':digest(b'late ciphertext')}], + key=lambda x:(x['id'], x['revision'])) + result = evidence((challenge, details, current)) + assert result['inventory_changed'] and result['historical_revisions_require_migration'] + assert not result['known_ciphertexts_migrated'] and result['verified_acked_count'] == 1 + + +def test_ack_cannot_cross_subject_or_challenge(history_bundle): + assert not history_evidence('u_other', 'cupg_history', *history_bundle)['known_ciphertexts_migrated'] + assert not history_evidence('u_history', 'cupg_other', *history_bundle)['known_ciphertexts_migrated'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('stored', [False, {}, [], 'unknown', None]) +async def test_persisted_quarantine_fails_closed_for_malformed_flags(stored): + store = FakeMetadataStore() + try: + async with store.transaction(write=True) as tx: + assert not active(tx) + tx.set_setting(SETTING, stored) + assert active(tx) + with pytest.raises(Failure, match='recovery_quarantined'): + require_live_authority(tx) + async with store.transaction(write=False) as tx: + assert active(tx) + finally: + await store.close() + + +@pytest.mark.asyncio +async def test_online_vault_destruction_is_not_backup_retirement(): + store = FakeMetadataStore() + old, new = Ed25519Signer.generate(), Ed25519Signer.generate() + details = {'old_identity_key_id':old.key_id, 'new_identity_key_id':new.key_id, + 'old_encryption_key_id':'e_old', 'new_encryption_key_id':'e_new'} + try: + async with store.transaction(write=True) as tx: + for key, primary in ((old, 0), (new, 1)): + tx.execute('INSERT INTO identity_keys VALUES(?,?,?,?,?,?)', + (key.key_id, 'u_exit', b64(key.public_key), wire(NOW), + None if primary else wire(NOW), primary), write=True) + for kid, primary in (('e_old', 0), ('e_new', 1)): + _, recipient = generate_age_key() + tx.execute('INSERT INTO encryption_subkeys VALUES(?,?,?,?,?,?,?)', + (kid, 'u_exit', recipient, b64(public_from_recipient(recipient)), wire(NOW), + None if primary else wire(NOW), primary), write=True) + cred = Credential(id=old.key_id, subject_id='u_exit', kind='signing_key', + verifier=old.public_key, ceiling=(), not_before=NOW, expires_at=None, revoked_at=NOW) + tx.execute('INSERT INTO credentials VALUES(?,?,?)', + (old.key_id, 'u_exit', canonical(cred).decode()), write=True) + tx.execute('''INSERT INTO custodial_vault(subject,signing_key_id,encryption_key_id, + status,created_at,destroyed_at) VALUES(?,?,?,?,?,?)''', + ('u_exit', old.key_id, 'e_old', 'destroyed', wire(NOW), wire(NOW)), write=True) + tx.set_setting('backup_retired', True) # An arbitrary flag is not evidence. + async with store.transaction(write=False) as tx: + result = await retirement_evidence(tx, 'u_exit', details, history_recoverable=True) + assert result['identity_switched'] and result['online_retired'] + assert not result['backup_retired'] and not result['server_key_retired'] + assert result['completion_status'] == 'pending_backup_retirement' + async with store.transaction(write=True) as tx: + tx.execute('UPDATE custodial_vault SET age_ciphertext=? WHERE subject=?', + (b'surviving encrypted key', 'u_exit'), write=True) + async with store.transaction(write=False) as tx: + assert not (await retirement_evidence(tx, 'u_exit', details, + history_recoverable=True))['online_retired'] + finally: + await store.close() + + +def test_restore_quarantine_is_inside_single_transaction(tmp_path, monkeypatch): + from msg.admin import restore_database + calls = [] + dump = tmp_path/'metadata.dump' + dump.write_bytes(b'offline dump') + gate = {'format':'msg-recovery-quarantine-v1', 'source_backup_sha256':'a'*64} + def command(argv, **kw): + calls.append(argv) + script = Path(argv[argv.index('--file')+1]) + if argv[0] == 'pg_restore': + assert '--dbname' not in argv + script.write_bytes(b'CREATE TABLE settings(key text PRIMARY KEY,value text);\n') + else: + assert '--single-transaction' in argv and '--no-psqlrc' in argv + assert 'ON_ERROR_STOP=1' in argv and kw['check'] is True + text = script.read_text() + assert text.index('CREATE TABLE') < text.index("VALUES('recovery_quarantine'") + assert canonical(gate).decode() in text + assert kw['env']['PGPASSWORD'] == 'not-in-command-line' + assert 'not-in-command-line' not in ' '.join(argv) + assert script.stat().st_mode & 0o777 == 0o600 + monkeypatch.setattr(restore_database.subprocess, 'run', command) + restore_database.restore_dump(dump, safe_dsn='service=test', + env={'PGPASSWORD':'not-in-command-line'}, quarantine=gate) + assert [call[0] for call in calls] == ['pg_restore', 'psql'] + assert not (tmp_path/'quarantined-restore.sql').exists() + + +def test_failed_dump_conversion_never_connects_to_restore_database(tmp_path, monkeypatch): + from msg.admin import restore_database + calls = [] + def fail(argv, **kw): + calls.append(argv[0]) + raise subprocess.CalledProcessError(1, argv) + monkeypatch.setattr(restore_database.subprocess, 'run', fail) + with pytest.raises(subprocess.CalledProcessError): + restore_database.restore_dump(tmp_path/'metadata.dump', safe_dsn='service=test', + env={}, quarantine={}) + assert calls == ['pg_restore'] + assert not (tmp_path/'quarantined-restore.sql').exists() diff --git a/tests/test_root_rotation.py b/tests/test_root_rotation.py index 9b9ae752..ac15e34b 100644 --- a/tests/test_root_rotation.py +++ b/tests/test_root_rotation.py @@ -3,7 +3,7 @@ from msg.security.crypto import Ed25519Signer,open_private_key from msg.core.codec import loads,wire from msg.application import Application -from msg.admin.root import _approve_csr +from msg.admin.root import _approve_csr,root_envelope from test_service import call,register,NOW @pytest.mark.asyncio @@ -31,6 +31,7 @@ async def test_root_rotation_retires_old_chain_and_requires_explicit_online_ca_a renewal=await call(fresh,'identity.certificate_renew',{},key=key,subject=uid) assert renewal.error.code=='renewal_source_required',wire(renewal) assert fresh.certificates.root_public_key==next_key.public_key - envelope=loads((app.settings.config_dir/'root'/'key.json').read_bytes()) + assert not (app.settings.config_dir/'root').exists() + envelope=loads(root_envelope(app.settings.config_dir).read_bytes()) assert open_private_key(envelope,pin)==next_key.private_bytes() finally:await fresh.close() diff --git a/tests/test_root_rotation_resume.py b/tests/test_root_rotation_resume.py new file mode 100644 index 00000000..958ad3f7 --- /dev/null +++ b/tests/test_root_rotation_resume.py @@ -0,0 +1,96 @@ +"""Use real isolated PostgreSQL; local use cases do not simulate a physical console.""" +from copy import deepcopy + +import pytest + +from msg.admin.root import root_envelope +from msg.admin.rotation import prepare, complete, journal_path +from msg.application import Application +from msg.core.codec import canonical, loads +from msg.core.errors import Failure +from msg.security.crypto import Ed25519Signer, seal_private_key, open_private_key +from msg.storage.git import durable_write +from test_service import NOW + +OLD_PIN = 'old-protected-envelope-test-passphrase' +NEW_PIN = 'new-protected-envelope-test-passphrase' + + +def old_envelope(app, root): + path = root_envelope(app.settings.config_dir) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + durable_write(path, canonical(seal_private_key(root.private_bytes(), OLD_PIN))) + return path, path.read_bytes() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('failure_at', ['key', 'trust']) +async def test_rotation_resumes_without_archiving_new_key_as_old(installed, monkeypatch, failure_at): + from msg.admin import rotation + app, root = installed + path, original_key = old_envelope(app, root) + successor = Ed25519Signer.generate() + journal = prepare(app, successor, NEW_PIN, old_signer=root, operator='isolated-test') + pending = journal_path(app) + assert journal['version'] == 2 and pending.parent.stat().st_mode & 0o777 == 0o700 + fail_path = path if failure_at == 'key' else app.settings.trust_file + real_write = rotation.durable_write + def interrupted(target, data, **kwargs): + if target == fail_path: + raise OSError('injected protected-file interruption') + return real_write(target, data, **kwargs) + with monkeypatch.context() as patch: + patch.setattr(rotation, 'durable_write', interrupted) + with pytest.raises(OSError, match='injected'): + await complete(app, journal, pin=NEW_PIN) + assert pending.exists() + async with app.metadata.transaction(write=False) as tx: + assert tx.setting('active_root_certificate') == journal['new_trust']['certificate']['resource_id'] + assert await tx.certificate_revoked(journal['old_certificate']['resource_id']) + before = tx.one('SELECT COUNT(*) FROM audit')[0] + # Resume must work before Application.load can validate partially switched trust files. + fresh = Application(app.settings, clock=lambda:NOW) + try: + result = await complete(fresh, loads(pending.read_bytes()), pin=NEW_PIN) + assert result['restart_required'] and not pending.exists() + assert root_envelope(app.settings.config_dir) == path + assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() + history = path.parent/'history'/journal['old_certificate']['resource_id']/'key.json' + assert history.read_bytes() == original_key + assert open_private_key(loads(history.read_bytes()), OLD_PIN) == root.private_bytes() + assert path.stat().st_mode & 0o777 == history.stat().st_mode & 0o777 == 0o600 + assert history.parent.stat().st_mode & 0o777 == 0o700 + assert not (app.settings.config_dir/'root').exists() + async with fresh.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM audit')[0] == before + await fresh.load() + assert fresh.certificates.root_public_key == successor.public_key + finally: + await fresh.close() + + +@pytest.mark.asyncio +async def test_replacement_csr_is_rejected_before_any_commit(installed): + app, root = installed + path, original = old_envelope(app, root) + journal = prepare(app, Ed25519Signer.generate(), NEW_PIN, old_signer=root, operator='isolated-test') + changed = deepcopy(journal) + changed['online_csr']['resource_id'] = 'csr_replaced_after_confirmation' + with pytest.raises(Failure, match='invalid_signature'): + await complete(app, changed, pin=NEW_PIN) + assert path.read_bytes() == original and journal_path(app).exists() + async with app.metadata.transaction(write=False) as tx: + assert tx.setting('active_root_certificate') == journal['old_certificate']['resource_id'] + assert not await tx.certificate_revoked(journal['old_certificate']['resource_id']) + + +@pytest.mark.asyncio +async def test_explicit_lost_root_key_rotation_uses_bound_journal(installed): + app, root = installed + path, original = old_envelope(app, root) + successor = Ed25519Signer.generate() + journal = prepare(app, successor, NEW_PIN, old_signer=None, operator='isolated-lost-key-test') + assert journal['statement']['previous_key_proved'] is False and journal['old_signature'] is None + await complete(app, journal, pin=NEW_PIN) + assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() + assert (path.parent/'history'/journal['old_certificate']['resource_id']/'key.json').read_bytes() == original