fix(infra): lenient store JSON decode so jitM can migrate legacy documents

.changeset/lenient-store-json-decode.md

@@ -0,0 +1,7 @@
+---
+"@effect-app/infra": patch
+---
+
+Store JSON lowering is lenient on decode; `jitM` migrations run after the store boundary.
+
+Document stores (Memory, Disk, SQL, Cosmos) decoded every stored document with the full schema on read, so a legacy-shaped document that relies on the repository's `jitM` to add a missing key failed with `Missing key` before `jitM` ever saw it. Decode now walks the stored document's own keys and only lifts JSON to native Encoded values (Date/Map/Set and app-native declarations); missing keys stay absent, refinements and checks are not enforced, and an unparseable leaf passes through unchanged. The repository's own decode, after `jitM`, still validates the whole document. Writes keep using the strict whole-document codec.

packages/infra/src/Store/jsonDocument.ts

@@ -1,7 +1,7 @@
 import type { FieldValues } from "effect-app/Model/filter/types"
 import * as S from "effect-app/Schema"
 import type { PersistenceModelType } from "effect-app/Store"
-import { toJsonQueryValue } from "./utils.ts"
+import { decodeWithSchema, toJsonQueryValue } from "./utils.ts"
 
 export interface JsonDocumentCodec<E extends FieldValues> {
   readonly encode: (doc: PersistenceModelType<E>) => PersistenceModelType<E>
@@ -19,8 +19,20 @@ const joinEtag = <E extends FieldValues>(
 ): PersistenceModelType<E> => (_etag === undefined ? rest : { ...rest, _etag })
 
 /**
- * Encoded document ↔ JSON document. Prefer `Schema.toCodecJson(toEncoded(schema))`
- * when the store has a schema; otherwise lower Date/Map/Set structurally.
+ * Encoded document ↔ JSON document.
+ *
+ * Writes always carry a complete document, so `encode` uses the strict
+ * whole-document codec `Schema.toCodecJson(toEncoded(schema))`.
+ *
+ * `decode` is deliberately lenient (see {@link decodeWithSchema}): it only
+ * lifts the keys that are present back to native Encoded values (Date/Map/Set
+ * and app-native declarations). It enforces neither required keys, nor
+ * refinements, nor checks, because the store boundary runs *before* the
+ * repository's `jitM` migration — an older-shaped document must still reach
+ * `jitM`, which fills it in before the repository's own (strict) decode.
+ *
+ * Without a schema, Date/Map/Set are lowered structurally on write and
+ * documents are read back as stored.
  */
 export const makeJsonDocumentCodec = <E extends FieldValues>(schema?: S.Top): JsonDocumentCodec<E> => {
   if (schema) {
@@ -32,7 +44,7 @@ export const makeJsonDocumentCodec = <E extends FieldValues>(schema?: S.Top): Js
       },
       decode: (doc) => {
         const { rest, _etag } = splitEtag(doc)
-        return joinEtag(S.decodeSync(codec)(rest as S.Json), _etag)
+        return joinEtag(decodeWithSchema(schema, rest) as E, _etag)
       }
     }
   }

packages/infra/src/Store/utils.ts

@@ -5,6 +5,7 @@ import * as Option from "effect-app/Option"
 import * as S from "effect-app/Schema"
 import * as SchemaAST from "effect-app/SchemaAST"
 import type { PersistenceModelType, SupportedValues2 } from "effect-app/Store"
+import * as Exit from "effect/Exit"
 import { OptimisticConcurrencyException } from "../errors.ts"
 
 const dateJson = S.toCodecJson(S.Date)
@@ -151,6 +152,71 @@ const encodeJson = (ast: SchemaAST.AST | undefined, value: unknown): unknown =>
   )
 }
 
+/**
+ * Decode a leaf (Date/Map/Set/app-native declaration) without ever throwing:
+ * the store boundary only lowers JSON into native Encoded values, it does not
+ * validate. A value we cannot decode is passed through unchanged, so the strict
+ * decode that runs after `jitM` reports it with the full path context.
+ */
+const decodeLeafExit = (ast: SchemaAST.AST, value: unknown): Exit.Exit<unknown, S.SchemaError> =>
+  Effect.runSyncExit(
+    S.decodeUnknownEffect(S.toCodecJson(S.make(ast)))(value) as Effect.Effect<unknown, S.SchemaError>
+  )
+
+const decodeLeaf = (ast: SchemaAST.AST, value: unknown): unknown => {
+  const exit = decodeLeafExit(ast, value)
+  return Exit.isSuccess(exit) ? exit.value : value
+}
+
+/**
+ * Resolve the union member a JSON object belongs to: by `_tag` literal when the
+ * union is tagged, else the first member that decodes the value as a whole.
+ */
+const unionMemberAst = (
+  current: SchemaAST.Union,
+  value: Record<string, unknown>
+): SchemaAST.AST | undefined => {
+  const tag = value["_tag"]
+  if (typeof tag === "string") {
+    const tagged = current.types.find((member) => {
+      const memberTag = astAtPath(member, ["_tag"])
+      return memberTag !== undefined && SchemaAST.isLiteral(memberTag) && memberTag.literal === tag
+    })
+    if (tagged !== undefined) return unwrapAst(tagged)
+  }
+  const hit = current.types.find((member) => Exit.isSuccess(decodeLeafExit(unwrapAst(member), value)))
+  return hit === undefined ? undefined : unwrapAst(hit)
+}
+
+const decodeJson = (ast: SchemaAST.AST | undefined, value: unknown): unknown => {
+  if (ast === undefined) return value
+  const current = unwrapAst(ast)
+  if (isPlainObject(value)) {
+    if (SchemaAST.isObjects(current)) {
+      // iterate the document's own keys, never the schema's: a key the stored
+      // document does not have stays absent instead of failing to decode.
+      const out: Record<string, unknown> = {}
+      for (const [key, child] of Object.entries(value)) {
+        out[key] = decodeJson(astAtPath(current, [key]), child)
+      }
+      return out
+    }
+    if (SchemaAST.isUnion(current)) {
+      return decodeJson(unionMemberAst(current, value), value)
+    }
+    return decodeLeaf(current, value)
+  }
+  if (Array.isArray(value)) {
+    if (SchemaAST.isArrays(current) || SchemaAST.isUnion(current)) {
+      const element = elementAst(current)
+      return value.map((item) => decodeJson(element, item))
+    }
+    // e.g. a ReadonlyMap/ReadonlySet declaration, whose JSON form is an array
+    return decodeLeaf(current, value)
+  }
+  return decodeLeaf(current, value)
+}
+
 const encodeFilterValue = (fieldAst: SchemaAST.AST | undefined, op: Ops, value: unknown): unknown => {
   if (fieldAst === undefined) return toJsonQueryValue(value)
   if (op === "in" || op === "notIn") {
@@ -252,6 +318,20 @@ export function encodeWithSchema(schema: S.Top | undefined, value: unknown): unk
   return encodeJson(SchemaAST.toEncoded(schema.ast), value)
 }
 
+/**
+ * Lift JSON back to native Encoded values (Date/Map/Set and app-native
+ * declarations) for the keys that are present.
+ *
+ * Deliberately lenient: it enforces neither required keys, nor refinements, nor
+ * checks. Store reads happen before the repository's `jitM` migration, so a
+ * stored document may still be of an older shape here; the repository's own
+ * decode validates the migrated document.
+ */
+export function decodeWithSchema(schema: S.Top | undefined, value: unknown): unknown {
+  if (schema === undefined) return value
+  return decodeJson(SchemaAST.toEncoded(schema.ast), value)
+}
+
 export function jsonifyFilter(
   filter: readonly FilterResult[],
   schema?: S.Top

packages/infra/test/json-document-lenient.test.ts

@@ -0,0 +1,196 @@
+import * as S from "effect-app/Schema"
+import * as Getter from "effect/SchemaGetter"
+import { describe, expect, it } from "vitest"
+import { makeJsonDocumentCodec } from "../src/Store/jsonDocument.js"
+
+class Day {
+  readonly ymd: string
+  constructor(ymd: string) {
+    this.ymd = ymd
+  }
+}
+
+const DayFromSelf = S.declare((u): u is Day => u instanceof Day, {
+  expected: "Day",
+  toCodecJson: () =>
+    S.link<Day>()(
+      S.String,
+      {
+        decode: Getter.transform((s: string) => new Day(s)),
+        encode: Getter.transform((d: Day) => d.ymd)
+      }
+    )
+})
+
+class Shop extends S.Class<Shop>("LenientShop")({
+  id: S.String,
+  name: S.NonEmptyString255,
+  createdAt: S.Date,
+  updatedAt: S.Date,
+  vatRate: S.Number,
+  day: DayFromSelf,
+  tags: S.ReadonlySet(S.String),
+  meta: S.ReadonlyMap({ key: S.String, value: S.String })
+}) {}
+
+const shopCodec = makeJsonDocumentCodec<typeof Shop.Encoded>(Shop)
+
+const completeShop = {
+  id: "shop-1",
+  name: "Shop One",
+  createdAt: "2024-06-01T00:00:00.000Z",
+  updatedAt: "2024-06-02T00:00:00.000Z",
+  vatRate: 19,
+  day: "2024-06-01",
+  tags: ["a", "b"],
+  meta: [["k", "v"]]
+}
+
+// the configurator shape: conditionGroups[].conditions[].rules[].groupId
+class Rule extends S.Class<Rule>("LenientRule")({
+  id: S.String,
+  groupId: S.String,
+  at: S.Date
+}) {}
+
+class Condition extends S.Class<Condition>("LenientCondition")({
+  id: S.String,
+  rules: S.Array(Rule)
+}) {}
+
+class ConditionGroup extends S.Class<ConditionGroup>("LenientConditionGroup")({
+  id: S.String,
+  conditions: S.Array(Condition)
+}) {}
+
+class Configurator extends S.Class<Configurator>("LenientConfigurator")({
+  id: S.String,
+  conditionGroups: S.Array(ConditionGroup)
+}) {}
+
+class Picking extends S.TaggedClass<Picking>()("picking", {
+  id: S.String,
+  pickedAt: S.Date,
+  picker: S.String
+}) {}
+
+class Assembling extends S.TaggedClass<Assembling>()("assembling", {
+  id: S.String,
+  assembledAt: S.Date
+}) {}
+
+class Batch extends S.Class<Batch>("LenientBatch")({
+  id: S.String,
+  state: S.Union([Picking, Assembling])
+}) {}
+
+describe("makeJsonDocumentCodec lenient decode", () => {
+  it("lowers JSON to native Encoded values for a complete document", () => {
+    const decoded = shopCodec.decode({ ...completeShop, _etag: "e1" } as never) as Record<string, unknown>
+    expect(decoded["createdAt"]).toBeInstanceOf(Date)
+    expect((decoded["createdAt"] as Date).toISOString()).toBe("2024-06-01T00:00:00.000Z")
+    expect(decoded["tags"]).toBeInstanceOf(Set)
+    expect([...(decoded["tags"] as Set<string>)]).toEqual(["a", "b"])
+    expect(decoded["meta"]).toBeInstanceOf(Map)
+    expect([...(decoded["meta"] as Map<string, string>)]).toEqual([["k", "v"]])
+    expect(decoded["day"]).toBeInstanceOf(Day)
+    expect((decoded["day"] as Day).ymd).toBe("2024-06-01")
+    expect(decoded["_etag"]).toBe("e1")
+  })
+
+  it("decodes a document missing a required top-level key, leaving it absent", () => {
+    const { vatRate: _vatRate, ...legacy } = completeShop
+    const decoded = shopCodec.decode(legacy as never) as Record<string, unknown>
+    expect("vatRate" in decoded).toBe(false)
+    // the keys that are present are still lowered
+    expect(decoded["createdAt"]).toBeInstanceOf(Date)
+    expect(decoded["tags"]).toBeInstanceOf(Set)
+  })
+
+  it("decodes a key missing inside array[].struct[].field, lowering its siblings", () => {
+    const codec = makeJsonDocumentCodec<typeof Configurator.Encoded>(Configurator)
+    const decoded = codec.decode({
+      id: "cfg-1",
+      conditionGroups: [{
+        id: "group-1",
+        conditions: [{
+          id: "condition-1",
+          // legacy rule: no `groupId`
+          rules: [{ id: "rule-1", at: "2024-06-01T00:00:00.000Z" }]
+        }]
+      }]
+    } as never) as any
+    const rule = decoded.conditionGroups[0].conditions[0].rules[0]
+    expect("groupId" in rule).toBe(false)
+    expect(rule.id).toBe("rule-1")
+    expect(rule.at).toBeInstanceOf(Date)
+  })
+
+  it("decodes a tagged-union field through the matching member", () => {
+    const codec = makeJsonDocumentCodec<typeof Batch.Encoded>(Batch)
+    const decoded = codec.decode({
+      id: "batch-1",
+      state: { _tag: "assembling", id: "state-1", assembledAt: "2024-06-01T00:00:00.000Z" }
+    } as never) as any
+    expect(decoded.state._tag).toBe("assembling")
+    expect(decoded.state.assembledAt).toBeInstanceOf(Date)
+
+    // a key missing inside the matching member does not throw
+    const legacy = codec.decode({
+      id: "batch-2",
+      state: { _tag: "picking", id: "state-2", pickedAt: "2024-06-01T00:00:00.000Z" }
+    } as never) as any
+    expect(legacy.state._tag).toBe("picking")
+    expect(legacy.state.pickedAt).toBeInstanceOf(Date)
+    expect("picker" in legacy.state).toBe(false)
+  })
+
+  it("passes an unparseable leaf through unchanged instead of throwing", () => {
+    const decoded = shopCodec.decode({ ...completeShop, updatedAt: null } as never) as Record<string, unknown>
+    expect(decoded["updatedAt"]).toBe(null)
+    expect(decoded["createdAt"]).toBeInstanceOf(Date)
+
+    const invalid = shopCodec.decode({ ...completeShop, createdAt: "not-a-date" } as never) as Record<string, unknown>
+    expect(invalid["createdAt"]).toBe("not-a-date")
+  })
+
+  it("does not enforce refinements or checks at the store boundary", () => {
+    const decoded = shopCodec.decode({ ...completeShop, name: "", vatRate: Number.NaN } as never) as Record<
+      string,
+      unknown
+    >
+    expect(decoded["name"]).toBe("")
+    expect(decoded["vatRate"]).toBeNaN()
+  })
+
+  it("lowers and preserves encodeKeys-renamed fields, missing ones staying absent", () => {
+    const Renamed = S
+      .Struct({
+        id: S.String,
+        day: DayFromSelf,
+        updatedAt: S.Date
+      })
+      .pipe(S.encodeKeys({ day: "the_day", updatedAt: "updated_at" }))
+    const codec = makeJsonDocumentCodec<typeof Renamed.Encoded>(Renamed)
+
+    const decoded = codec.decode({
+      id: "r-1",
+      the_day: "2024-06-01",
+      updated_at: "2024-06-02T00:00:00.000Z"
+    } as never) as Record<string, unknown>
+
+    expect(decoded["the_day"]).toBeInstanceOf(Day)
+    expect(decoded["updated_at"]).toBeInstanceOf(Date)
+    expect(decoded["day"]).toBeUndefined()
+
+    // a renamed key the stored document does not have stays absent
+    const partial = codec.decode({ id: "r-2", the_day: "2024-06-03" } as never) as Record<string, unknown>
+    expect(partial["the_day"]).toBeInstanceOf(Day)
+    expect("updated_at" in partial).toBe(false)
+  })
+
+  it("still round-trips a complete document through encode", () => {
+    const decoded = shopCodec.decode(completeShop as never)
+    expect(shopCodec.encode(decoded)).toEqual(completeShop)
+  })
+})