feat(sdk): named data gates v1 (#322) #353
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| import { readFileSync } from 'node:fs'; | ||
| import { createRequire } from 'node:module'; | ||
| import { deflateRawSync } from 'node:zlib'; | ||
| import { bindingDependencies } from './input-binding.js'; | ||
| import { isNamedGate, regexFlags } from './named-gates.js'; | ||
| import type { NamedDataGate, StepSpec, VerificationSpec } from './spec.js'; | ||
|
|
||
| const quote = (text: string): string => `'${text.replaceAll("'", "'\\''")}'`; | ||
| let re2Source: string | undefined; | ||
|
|
||
| /** Embed the pinned engine in the command: execution does not resolve npm or SDK paths. */ | ||
| function embeddedRE2(): string { | ||
| re2Source ??= deflateRawSync(readFileSync(createRequire(import.meta.url).resolve('re2js'))).toString('base64'); | ||
| return `const re2={};new Function('exports',require('node:zlib').inflateRawSync(Buffer.from(${JSON.stringify(re2Source)},'base64')).toString())(re2);`; | ||
|
There was a problem hiding this comment. RE2 embed exceeds exec argument limitHigh Severity
Additional Locations (1)  Reviewed by Cursor Bugbot for commit f63aab1. Configure here. |
||
| } | ||
|
|
||
| /** | ||
| * A gate is a nested deterministic step in the same journal/DAG. Keep the | ||
| * producer's identity/output, and add a barrier to every dependent (including | ||
| * implicit input edges). Internal bindings select the whole envelope so no | ||
| * invented output-path schema is needed. Missing paths fail in the gate. | ||
| */ | ||
| export function lowerNamedGates(steps: readonly StepSpec[]): StepSpec[] { | ||
| const used = new Set(steps.map(step => step.id)); | ||
| const barriers = new Map<string, string>(); | ||
| for (const step of steps) { | ||
| if (!isNamedGate(step.verification)) continue; | ||
| let id = `${step.id}.gate`; | ||
| while (used.has(id)) id += '.gate'; | ||
| used.add(id); | ||
| barriers.set(step.id, id); | ||
| } | ||
| if (barriers.size === 0) return [...steps]; | ||
| const output: StepSpec[] = []; | ||
| for (const step of steps) { | ||
| const dependencies = [...new Set([...(step.dependsOn ?? []), ...bindingDependencies(step.input)])]; | ||
| const dependsOn = [...new Set([...dependencies, ...dependencies.flatMap(id => barriers.get(id) ?? [])])]; | ||
| const producer = dependsOn.length ? { ...step, dependsOn } : step; | ||
| const gate = step.verification; | ||
| if (!isNamedGate(gate)) { | ||
| output.push(producer); | ||
| continue; | ||
| } | ||
| // The original output is still checked for a successful process/worker | ||
| // completion. This schema permits an internal whole-output binding. | ||
| output.push({ ...producer, verification: { type: 'json_schema', schema: true } }); | ||
| const input = { | ||
| output: { step: step.id }, | ||
| ...(gate.type === 'references_input' ? { reference: step.input![gate.input_key]! } : {}), | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| }; | ||
| output.push({ | ||
| id: barriers.get(step.id)!, type: 'deterministic', | ||
| dependsOn: [...new Set([step.id, ...bindingDependencies(input)])], input, | ||
| command: gateCommand(gate, step.type === 'deterministic'), | ||
| verification: gateVerification(gate), maxIterations: step.maxIterations ?? 1, | ||
| ...(step.requirements === undefined ? {} : { requirements: step.requirements }), | ||
| }); | ||
| } | ||
| return output; | ||
| } | ||
|
|
||
| function gateCommand(gate: NamedDataGate, deterministic: boolean): string { | ||
| const path = gate.type === 'subprocess_gate' ? gate.from_output | ||
| : gate.type === 'word_count_bounds' ? undefined : gate.in_output_at; | ||
| // Only compiler-owned code is serialized. Author strings are JSON literals; | ||
| // upstream output travels exclusively through FLOWS_INPUT, never shell text. | ||
| const setup = `const cp=require('node:child_process'); | ||
| const input=JSON.parse(process.env.FLOWS_INPUT); | ||
| let value=input.output; | ||
| const path=${JSON.stringify(path ?? null)}; | ||
| if(path!==null){for(const key of path){ | ||
| if(value===null||typeof value!=='object'||!Object.hasOwn(value,key)|| | ||
| (typeof key==='number'?!Array.isArray(value):Array.isArray(value)))process.exit(1); | ||
| value=value[key]; | ||
| }}else if(${deterministic})value=value.stdout_tail; | ||
|
cursor[bot] marked this conversation as resolved.
|
||
| const text=typeof value==='string'?value:JSON.stringify(value); | ||
| if(typeof text!=='string')process.exit(1); | ||
| `; | ||
| let body: string; | ||
| switch (gate.type) { | ||
| case 'references_input': | ||
| body = `const reference=input.reference; | ||
| if(typeof reference!=='string'||reference.length===0||!text.includes(reference))process.exit(1); | ||
|
There was a problem hiding this comment. Reference gate rejects non-string inputsMedium Severity
Additional Locations (1)Reviewed by Cursor Bugbot for commit f63aab1. Configure here. |
||
| process.stdout.write('references_input:pass');`; | ||
| break; | ||
| case 'subprocess_gate': | ||
| body = `if(text.includes('\\0'))process.exit(1); | ||
| const result=cp.spawnSync('/bin/sh',['-c',${JSON.stringify(gate.command)}],{ | ||
| env:{...process.env,INPUT:text},stdio:'inherit'}); | ||
| process.exit(result.status===0?0:1);`; | ||
| break; | ||
| case 'word_count_bounds': | ||
| body = `const result=cp.spawnSync('wc',['-w'],{input:text,encoding:'utf8',env:{...process.env,LC_ALL:'C'}}); | ||
| if(result.status!==0)process.exit(1); | ||
| const count=result.stdout.trim(); | ||
| if(!/^[0-9]+$/.test(count))process.exit(1); | ||
| process.stdout.write(BigInt(count).toString());`; | ||
| break; | ||
| case 'regex_match': | ||
| body = `${embeddedRE2()} | ||
| process.exit(re2.RE2JS.compile(${JSON.stringify(gate.pattern)},${regexFlags(gate.flags ?? '')}).matcher(text).find()?0:1);`; | ||
| break; | ||
| } | ||
| return `node -e ${quote(setup + body)}`; | ||
| } | ||
|
|
||
| function gateVerification(gate: NamedDataGate): VerificationSpec { | ||
| if (gate.type === 'references_input') { | ||
| // output_contains cannot carry a runtime binding. The command resolves and | ||
| // checks the literal substring, then emits this fixed, unforgeable receipt. | ||
| return { type: 'output_contains', value: 'references_input:pass' }; | ||
| } | ||
| if (gate.type === 'word_count_bounds') { | ||
| // Deterministic output is an envelope with STRING stdout, not parsed JSON. | ||
| // A bounded decimal language enforces the same inclusive numeric range | ||
| // with the existing json_schema primitive and retains the actual count. | ||
| return { type: 'json_schema', schema: { | ||
| type: 'object', required: ['stdout_tail'], properties: { | ||
| stdout_tail: { type: 'string', pattern: decimalRange(gate.min ?? 0, gate.max ?? Number.MAX_SAFE_INTEGER) }, | ||
| }, | ||
| } }; | ||
| } | ||
| return { type: 'exit_code' }; | ||
| } | ||
|
|
||
| /** Compact decimal range, bounded by 16 digits rather than by range width. */ | ||
| function decimalRange(min: number, max: number): string { | ||
| const patterns: string[] = []; | ||
| function between(low: string, high: string, prefix: string): void { | ||
| if (low === high) { patterns.push(prefix + low); return; } | ||
| if (/^0+$/.test(low) && /^9+$/.test(high)) { | ||
| patterns.push(`${prefix}[0-9]{${low.length}}`); return; | ||
| } | ||
| const a = Number(low[0]), b = Number(high[0]); | ||
| if (a === b) { between(low.slice(1), high.slice(1), prefix + a); return; } | ||
| const rest = low.length - 1; | ||
| between(low.slice(1), '9'.repeat(rest), prefix + a); | ||
| if (a + 1 <= b - 1) patterns.push(`${prefix}[${a + 1}-${b - 1}]${rest ? `[0-9]{${rest}}` : ''}`); | ||
| between('0'.repeat(rest), high.slice(1), prefix + b); | ||
| } | ||
| for (let digits = String(min).length; digits <= String(max).length; digits++) { | ||
| const low = digits === String(min).length ? String(min) : '1' + '0'.repeat(digits - 1); | ||
| const high = digits === String(max).length ? String(max) : '9'.repeat(digits); | ||
| between(low, high, ''); | ||
| } | ||
| return `^(?:${patterns.join('|')})$`; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| import { RE2JS } from 're2js'; | ||
| import type { NamedDataGate, VerificationSpec } from './spec.js'; | ||
|
|
||
| export const NAMED_GATE_FAILURE_KINDS = [ | ||
| 'unknown_gate_kind', 'gate_pattern_invalid', 'gate_command_missing', 'gate_bound_invalid', | ||
| ] as const; | ||
| export type NamedGateFailureKind = typeof NAMED_GATE_FAILURE_KINDS[number]; | ||
|
|
||
| export const NAMED_GATE_KEYS: Record<NamedDataGate['type'], readonly string[]> = { | ||
| references_input: ['type', 'input_key', 'in_output_at'], | ||
| subprocess_gate: ['type', 'command', 'from_output'], | ||
| word_count_bounds: ['type', 'min', 'max'], | ||
| regex_match: ['type', 'pattern', 'in_output_at', 'flags'], | ||
| }; | ||
|
|
||
| export function isNamedGate(gate: VerificationSpec | undefined): gate is NamedDataGate { | ||
| return gate !== undefined && Object.hasOwn(NAMED_GATE_KEYS, gate.type); | ||
| } | ||
|
|
||
| export function regexFlags(flags: string): number { | ||
| return (flags.includes('i') ? RE2JS.CASE_INSENSITIVE : 0) | ||
| | (flags.includes('m') ? RE2JS.MULTILINE : 0) | ||
| | (flags.includes('s') ? RE2JS.DOTALL : 0); | ||
| } | ||
|
|
||
| /** Called only on snapshotted data, including by the public validator. */ | ||
| export function namedGateErrors(gate: Record<string, unknown>, input: unknown, at: string): string[] { | ||
| const errors: string[] = []; | ||
| for (const key of ['in_output_at', 'from_output']) { | ||
| const path = gate[key]; | ||
| if (path !== undefined && (!Array.isArray(path) || !path.every(segment => | ||
| typeof segment === 'string' || (Number.isSafeInteger(segment) && (segment as number) >= 0)))) { | ||
| errors.push(`${at}.${key}: expected an array of object keys or non-negative integer indices`); | ||
| } | ||
| } | ||
| switch (gate.type) { | ||
| case 'references_input': | ||
| if (typeof gate.input_key !== 'string' || !gate.input_key.trim() | ||
| || input === null || typeof input !== 'object' || !Object.hasOwn(input, gate.input_key)) { | ||
| errors.push(`${at}.input_key: must name a declared input binding`); | ||
| } | ||
| break; | ||
| case 'subprocess_gate': | ||
| if (typeof gate.command !== 'string' || !gate.command.trim() || gate.command.includes('\0')) { | ||
| errors.push(`${at}.command: gate_command_missing: expected a non-empty shell command`); | ||
| } | ||
| break; | ||
| case 'word_count_bounds': { | ||
| const { min, max } = gate; | ||
| if ([min, max].some(n => n !== undefined && (!Number.isSafeInteger(n) || (n as number) < 0)) | ||
| || (typeof min === 'number' && typeof max === 'number' && min > max)) { | ||
| errors.push(`${at}: gate_bound_invalid: min and max must be non-negative safe integers with min <= max`); | ||
| } | ||
| break; | ||
| } | ||
| case 'regex_match': | ||
| try { | ||
| if (typeof gate.pattern !== 'string' || gate.pattern.length > 8192) { | ||
| throw new Error('pattern must be a string of at most 8192 characters'); | ||
| } | ||
| const flags = gate.flags ?? ''; | ||
| if (typeof flags !== 'string' || !/^(?!.*(.).*\1)[ims]*$/.test(flags)) { | ||
| throw new Error('flags must be a non-repeating subset of i, m, s'); | ||
| } | ||
| RE2JS.compile(gate.pattern, regexFlags(flags)); | ||
| } catch (error) { | ||
| errors.push(`${at}: gate_pattern_invalid: ${error instanceof Error ? error.message : 'invalid RE2 pattern'}`); | ||
| } | ||
| break; | ||
| } | ||
| return errors; | ||
| } | ||
|
|
||
| export function namedGateFailure(errors: readonly string[]): NamedGateFailureKind | undefined { | ||
| return NAMED_GATE_FAILURE_KINDS.find(kind => errors.some(error => error.includes(`${kind}:`))); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.