diff --git a/.changeset/lenient-store-json-decode.md b/.changeset/lenient-store-json-decode.md new file mode 100644 index 000000000..20c96327f --- /dev/null +++ b/.changeset/lenient-store-json-decode.md @@ -0,0 +1,7 @@ +--- +"@effect-app/infra": patch +--- + +Store JSON lowering is lenient on decode; `jitM` migrations run after the store boundary. + +Document stores (Memory, Disk, SQL, Cosmos) decoded every stored document with the full schema on read, so a legacy-shaped document that relies on the repository's `jitM` to add a missing key failed with `Missing key` before `jitM` ever saw it. Decode now walks the stored document's own keys and only lifts JSON to native Encoded values (Date/Map/Set and app-native declarations); missing keys stay absent, refinements and checks are not enforced, and an unparseable leaf passes through unchanged. The repository's own decode, after `jitM`, still validates the whole document. Writes keep using the strict whole-document codec. diff --git a/packages/infra/src/Store/jsonDocument.ts b/packages/infra/src/Store/jsonDocument.ts index d0302208b..bae0b7b35 100644 --- a/packages/infra/src/Store/jsonDocument.ts +++ b/packages/infra/src/Store/jsonDocument.ts @@ -1,7 +1,7 @@ import type { FieldValues } from "effect-app/Model/filter/types" import * as S from "effect-app/Schema" import type { PersistenceModelType } from "effect-app/Store" -import { toJsonQueryValue } from "./utils.ts" +import { decodeWithSchema, toJsonQueryValue } from "./utils.ts" export interface JsonDocumentCodec { readonly encode: (doc: PersistenceModelType) => PersistenceModelType @@ -19,8 +19,20 @@ const joinEtag = ( ): PersistenceModelType => (_etag === undefined ? rest : { ...rest, _etag }) /** - * Encoded document ↔ JSON document. Prefer `Schema.toCodecJson(toEncoded(schema))` - * when the store has a schema; otherwise lower Date/Map/Set structurally. + * Encoded document ↔ JSON document. + * + * Writes always carry a complete document, so `encode` uses the strict + * whole-document codec `Schema.toCodecJson(toEncoded(schema))`. + * + * `decode` is deliberately lenient (see {@link decodeWithSchema}): it only + * lifts the keys that are present back to native Encoded values (Date/Map/Set + * and app-native declarations). It enforces neither required keys, nor + * refinements, nor checks, because the store boundary runs *before* the + * repository's `jitM` migration — an older-shaped document must still reach + * `jitM`, which fills it in before the repository's own (strict) decode. + * + * Without a schema, Date/Map/Set are lowered structurally on write and + * documents are read back as stored. */ export const makeJsonDocumentCodec = (schema?: S.Top): JsonDocumentCodec => { if (schema) { @@ -32,7 +44,7 @@ export const makeJsonDocumentCodec = (schema?: S.Top): Js }, decode: (doc) => { const { rest, _etag } = splitEtag(doc) - return joinEtag(S.decodeSync(codec)(rest as S.Json), _etag) + return joinEtag(decodeWithSchema(schema, rest) as E, _etag) } } } diff --git a/packages/infra/src/Store/utils.ts b/packages/infra/src/Store/utils.ts index 8b6ae7da9..a2d46350e 100644 --- a/packages/infra/src/Store/utils.ts +++ b/packages/infra/src/Store/utils.ts @@ -5,6 +5,7 @@ import * as Option from "effect-app/Option" import * as S from "effect-app/Schema" import * as SchemaAST from "effect-app/SchemaAST" import type { PersistenceModelType, SupportedValues2 } from "effect-app/Store" +import * as Exit from "effect/Exit" import { OptimisticConcurrencyException } from "../errors.ts" const dateJson = S.toCodecJson(S.Date) @@ -151,6 +152,71 @@ const encodeJson = (ast: SchemaAST.AST | undefined, value: unknown): unknown => ) } +/** + * Decode a leaf (Date/Map/Set/app-native declaration) without ever throwing: + * the store boundary only lowers JSON into native Encoded values, it does not + * validate. A value we cannot decode is passed through unchanged, so the strict + * decode that runs after `jitM` reports it with the full path context. + */ +const decodeLeafExit = (ast: SchemaAST.AST, value: unknown): Exit.Exit => + Effect.runSyncExit( + S.decodeUnknownEffect(S.toCodecJson(S.make(ast)))(value) as Effect.Effect + ) + +const decodeLeaf = (ast: SchemaAST.AST, value: unknown): unknown => { + const exit = decodeLeafExit(ast, value) + return Exit.isSuccess(exit) ? exit.value : value +} + +/** + * Resolve the union member a JSON object belongs to: by `_tag` literal when the + * union is tagged, else the first member that decodes the value as a whole. + */ +const unionMemberAst = ( + current: SchemaAST.Union, + value: Record +): SchemaAST.AST | undefined => { + const tag = value["_tag"] + if (typeof tag === "string") { + const tagged = current.types.find((member) => { + const memberTag = astAtPath(member, ["_tag"]) + return memberTag !== undefined && SchemaAST.isLiteral(memberTag) && memberTag.literal === tag + }) + if (tagged !== undefined) return unwrapAst(tagged) + } + const hit = current.types.find((member) => Exit.isSuccess(decodeLeafExit(unwrapAst(member), value))) + return hit === undefined ? undefined : unwrapAst(hit) +} + +const decodeJson = (ast: SchemaAST.AST | undefined, value: unknown): unknown => { + if (ast === undefined) return value + const current = unwrapAst(ast) + if (isPlainObject(value)) { + if (SchemaAST.isObjects(current)) { + // iterate the document's own keys, never the schema's: a key the stored + // document does not have stays absent instead of failing to decode. + const out: Record = {} + for (const [key, child] of Object.entries(value)) { + out[key] = decodeJson(astAtPath(current, [key]), child) + } + return out + } + if (SchemaAST.isUnion(current)) { + return decodeJson(unionMemberAst(current, value), value) + } + return decodeLeaf(current, value) + } + if (Array.isArray(value)) { + if (SchemaAST.isArrays(current) || SchemaAST.isUnion(current)) { + const element = elementAst(current) + return value.map((item) => decodeJson(element, item)) + } + // e.g. a ReadonlyMap/ReadonlySet declaration, whose JSON form is an array + return decodeLeaf(current, value) + } + return decodeLeaf(current, value) +} + const encodeFilterValue = (fieldAst: SchemaAST.AST | undefined, op: Ops, value: unknown): unknown => { if (fieldAst === undefined) return toJsonQueryValue(value) if (op === "in" || op === "notIn") { @@ -252,6 +318,20 @@ export function encodeWithSchema(schema: S.Top | undefined, value: unknown): unk return encodeJson(SchemaAST.toEncoded(schema.ast), value) } +/** + * Lift JSON back to native Encoded values (Date/Map/Set and app-native + * declarations) for the keys that are present. + * + * Deliberately lenient: it enforces neither required keys, nor refinements, nor + * checks. Store reads happen before the repository's `jitM` migration, so a + * stored document may still be of an older shape here; the repository's own + * decode validates the migrated document. + */ +export function decodeWithSchema(schema: S.Top | undefined, value: unknown): unknown { + if (schema === undefined) return value + return decodeJson(SchemaAST.toEncoded(schema.ast), value) +} + export function jsonifyFilter( filter: readonly FilterResult[], schema?: S.Top diff --git a/packages/infra/test/json-document-lenient.test.ts b/packages/infra/test/json-document-lenient.test.ts new file mode 100644 index 000000000..63c2886f1 --- /dev/null +++ b/packages/infra/test/json-document-lenient.test.ts @@ -0,0 +1,196 @@ +import * as S from "effect-app/Schema" +import * as Getter from "effect/SchemaGetter" +import { describe, expect, it } from "vitest" +import { makeJsonDocumentCodec } from "../src/Store/jsonDocument.js" + +class Day { + readonly ymd: string + constructor(ymd: string) { + this.ymd = ymd + } +} + +const DayFromSelf = S.declare((u): u is Day => u instanceof Day, { + expected: "Day", + toCodecJson: () => + S.link()( + S.String, + { + decode: Getter.transform((s: string) => new Day(s)), + encode: Getter.transform((d: Day) => d.ymd) + } + ) +}) + +class Shop extends S.Class("LenientShop")({ + id: S.String, + name: S.NonEmptyString255, + createdAt: S.Date, + updatedAt: S.Date, + vatRate: S.Number, + day: DayFromSelf, + tags: S.ReadonlySet(S.String), + meta: S.ReadonlyMap({ key: S.String, value: S.String }) +}) {} + +const shopCodec = makeJsonDocumentCodec(Shop) + +const completeShop = { + id: "shop-1", + name: "Shop One", + createdAt: "2024-06-01T00:00:00.000Z", + updatedAt: "2024-06-02T00:00:00.000Z", + vatRate: 19, + day: "2024-06-01", + tags: ["a", "b"], + meta: [["k", "v"]] +} + +// the configurator shape: conditionGroups[].conditions[].rules[].groupId +class Rule extends S.Class("LenientRule")({ + id: S.String, + groupId: S.String, + at: S.Date +}) {} + +class Condition extends S.Class("LenientCondition")({ + id: S.String, + rules: S.Array(Rule) +}) {} + +class ConditionGroup extends S.Class("LenientConditionGroup")({ + id: S.String, + conditions: S.Array(Condition) +}) {} + +class Configurator extends S.Class("LenientConfigurator")({ + id: S.String, + conditionGroups: S.Array(ConditionGroup) +}) {} + +class Picking extends S.TaggedClass()("picking", { + id: S.String, + pickedAt: S.Date, + picker: S.String +}) {} + +class Assembling extends S.TaggedClass()("assembling", { + id: S.String, + assembledAt: S.Date +}) {} + +class Batch extends S.Class("LenientBatch")({ + id: S.String, + state: S.Union([Picking, Assembling]) +}) {} + +describe("makeJsonDocumentCodec lenient decode", () => { + it("lowers JSON to native Encoded values for a complete document", () => { + const decoded = shopCodec.decode({ ...completeShop, _etag: "e1" } as never) as Record + expect(decoded["createdAt"]).toBeInstanceOf(Date) + expect((decoded["createdAt"] as Date).toISOString()).toBe("2024-06-01T00:00:00.000Z") + expect(decoded["tags"]).toBeInstanceOf(Set) + expect([...(decoded["tags"] as Set)]).toEqual(["a", "b"]) + expect(decoded["meta"]).toBeInstanceOf(Map) + expect([...(decoded["meta"] as Map)]).toEqual([["k", "v"]]) + expect(decoded["day"]).toBeInstanceOf(Day) + expect((decoded["day"] as Day).ymd).toBe("2024-06-01") + expect(decoded["_etag"]).toBe("e1") + }) + + it("decodes a document missing a required top-level key, leaving it absent", () => { + const { vatRate: _vatRate, ...legacy } = completeShop + const decoded = shopCodec.decode(legacy as never) as Record + expect("vatRate" in decoded).toBe(false) + // the keys that are present are still lowered + expect(decoded["createdAt"]).toBeInstanceOf(Date) + expect(decoded["tags"]).toBeInstanceOf(Set) + }) + + it("decodes a key missing inside array[].struct[].field, lowering its siblings", () => { + const codec = makeJsonDocumentCodec(Configurator) + const decoded = codec.decode({ + id: "cfg-1", + conditionGroups: [{ + id: "group-1", + conditions: [{ + id: "condition-1", + // legacy rule: no `groupId` + rules: [{ id: "rule-1", at: "2024-06-01T00:00:00.000Z" }] + }] + }] + } as never) as any + const rule = decoded.conditionGroups[0].conditions[0].rules[0] + expect("groupId" in rule).toBe(false) + expect(rule.id).toBe("rule-1") + expect(rule.at).toBeInstanceOf(Date) + }) + + it("decodes a tagged-union field through the matching member", () => { + const codec = makeJsonDocumentCodec(Batch) + const decoded = codec.decode({ + id: "batch-1", + state: { _tag: "assembling", id: "state-1", assembledAt: "2024-06-01T00:00:00.000Z" } + } as never) as any + expect(decoded.state._tag).toBe("assembling") + expect(decoded.state.assembledAt).toBeInstanceOf(Date) + + // a key missing inside the matching member does not throw + const legacy = codec.decode({ + id: "batch-2", + state: { _tag: "picking", id: "state-2", pickedAt: "2024-06-01T00:00:00.000Z" } + } as never) as any + expect(legacy.state._tag).toBe("picking") + expect(legacy.state.pickedAt).toBeInstanceOf(Date) + expect("picker" in legacy.state).toBe(false) + }) + + it("passes an unparseable leaf through unchanged instead of throwing", () => { + const decoded = shopCodec.decode({ ...completeShop, updatedAt: null } as never) as Record + expect(decoded["updatedAt"]).toBe(null) + expect(decoded["createdAt"]).toBeInstanceOf(Date) + + const invalid = shopCodec.decode({ ...completeShop, createdAt: "not-a-date" } as never) as Record + expect(invalid["createdAt"]).toBe("not-a-date") + }) + + it("does not enforce refinements or checks at the store boundary", () => { + const decoded = shopCodec.decode({ ...completeShop, name: "", vatRate: Number.NaN } as never) as Record< + string, + unknown + > + expect(decoded["name"]).toBe("") + expect(decoded["vatRate"]).toBeNaN() + }) + + it("lowers and preserves encodeKeys-renamed fields, missing ones staying absent", () => { + const Renamed = S + .Struct({ + id: S.String, + day: DayFromSelf, + updatedAt: S.Date + }) + .pipe(S.encodeKeys({ day: "the_day", updatedAt: "updated_at" })) + const codec = makeJsonDocumentCodec(Renamed) + + const decoded = codec.decode({ + id: "r-1", + the_day: "2024-06-01", + updated_at: "2024-06-02T00:00:00.000Z" + } as never) as Record + + expect(decoded["the_day"]).toBeInstanceOf(Day) + expect(decoded["updated_at"]).toBeInstanceOf(Date) + expect(decoded["day"]).toBeUndefined() + + // a renamed key the stored document does not have stays absent + const partial = codec.decode({ id: "r-2", the_day: "2024-06-03" } as never) as Record + expect(partial["the_day"]).toBeInstanceOf(Day) + expect("updated_at" in partial).toBe(false) + }) + + it("still round-trips a complete document through encode", () => { + const decoded = shopCodec.decode(completeShop as never) + expect(shopCodec.encode(decoded)).toEqual(completeShop) + }) +}) diff --git a/packages/infra/test/repository-legacy-document.test.ts b/packages/infra/test/repository-legacy-document.test.ts new file mode 100644 index 000000000..8d0f26d2e --- /dev/null +++ b/packages/infra/test/repository-legacy-document.test.ts @@ -0,0 +1,143 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import * as Effect from "effect-app/Effect" +import * as Layer from "effect-app/Layer" +import { makeRepo } from "effect-app/Model/Repository" +import { RepositoryRegistryLive } from "effect-app/Model/Repository/Registry" +import * as Option from "effect-app/Option" +import * as S from "effect-app/Schema" +import { setupRequestContextFromCurrent } from "effect-app/setupRequest" +import { StoreMaker } from "effect-app/Store" +import { describe, expect, it } from "vitest" +import { makeJsonDocumentCodec } from "../src/Store/jsonDocument.js" +import { makeMemoryStoreInt } from "../src/Store/Memory.js" +import { makeJsonLower } from "../src/Store/utils.js" + +/** + * A Memory store seeded with *raw stored documents* instead of items encoded + * through the schema, so that a legacy-shaped document can reach the read path - + * exactly what Cosmos (`fromStored`) and SQL (`parseRow`) do in production. + * + * The inner store is schemaless (it may not validate what we seed); the JSON + * document codec of the repository's schema is applied on read, as the real + * adapters do. + */ +const LegacyDocStoreLive = (docs: readonly Record[]) => + StoreMaker.toLayer(Effect.sync(() => ({ + make: ((modelName: string, idKey: any, _seed: any, config: any) => + Effect.map( + makeMemoryStoreInt( + modelName, + idKey, + "primary", + Effect.succeed(docs as any), + config?.defaultValues, + undefined, + makeJsonLower(config) + ), + (store) => { + const decode = makeJsonDocumentCodec(config?.schema).decode + return { + ...store, + all: Effect.map(store.all, (rows) => rows.map(decode)), + find: (id: any) => Effect.map(store.find(id), Option.map(decode)), + filter: (f: any) => Effect.map(store.filter(f), (rows: any[]) => f.select ? rows : rows.map(decode)) + } + } + )) as any + }))) + +class Shop extends S.Class("LegacyDocShop")({ + id: S.String, + name: S.NonEmptyString255, + createdAt: S.Date, + vatRate: S.Number, + tags: S.ReadonlySet(S.String) +}) {} + +// `vatRate` was added later; stored documents from before that do not have it. +const jitM = (pm: typeof Shop.Encoded) => { + const raw = pm as Record + return ("vatRate" in raw ? raw : { ...raw, vatRate: 19 }) as typeof Shop.Encoded +} + +const legacyDoc = { + id: "shop-legacy", + name: "Legacy Shop", + createdAt: "2024-06-01T00:00:00.000Z", + tags: ["a"] +} + +const currentDoc = { + id: "shop-current", + name: "Current Shop", + createdAt: "2024-06-02T00:00:00.000Z", + vatRate: 7, + tags: ["b"] +} + +const TestLive = Layer.merge(LegacyDocStoreLive([legacyDoc, currentDoc]), RepositoryRegistryLive) + +describe("repository reads of legacy documents", () => { + it("all: jitM fills the key the stored document is missing", () => + Effect + .gen(function*() { + const repo = yield* makeRepo("LegacyDocShop", Shop, { jitM }) + + const items = yield* repo.all + + expect(items).toHaveLength(2) + const legacy = items.find((_) => _.id === "shop-legacy")! + expect(legacy).toBeInstanceOf(Shop) + expect(legacy.vatRate).toBe(19) + expect(legacy.createdAt).toBeInstanceOf(Date) + expect(legacy.tags).toBeInstanceOf(Set) + expect([...legacy.tags]).toEqual(["a"]) + expect(items.find((_) => _.id === "shop-current")!.vatRate).toBe(7) + }) + .pipe( + Effect.provide(TestLive), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + )) + + it("find: a legacy document decodes through jitM", () => + Effect + .gen(function*() { + const repo = yield* makeRepo("LegacyDocShop", Shop, { jitM }) + + const found = yield* repo.find("shop-legacy") + + expect(Option.isSome(found)).toBe(true) + const shop = Option.getOrThrow(found) + expect(shop.vatRate).toBe(19) + expect(shop.name).toBe("Legacy Shop") + expect(shop.createdAt).toBeInstanceOf(Date) + }) + .pipe( + Effect.provide(TestLive), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + )) + + it("validateSample: legacy documents validate because jitM runs after the store boundary", () => + Effect + .gen(function*() { + const repo = yield* makeRepo("LegacyDocShop", Shop, { jitM }) + + const result = yield* repo.validateSample({ percentage: 1.0 }) + + expect(result.total).toBe(2) + expect(result.sampled).toBe(2) + expect(result.errors).toHaveLength(0) + expect(result.valid).toBe(2) + }) + .pipe( + Effect.provide(TestLive), + setupRequestContextFromCurrent(), + Effect.scoped, + Effect.runPromise + )) +})