diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 437336999..0449e98f5 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -20,4 +20,3 @@ tempfile = "3.20" thiserror = "2.0" ulid = { version = "1.2", features = ["serde"] } wait-timeout = "0.2" - diff --git a/kernel/relayflowd/Cargo.toml b/kernel/relayflowd/Cargo.toml index 3b69b593f..b23bb3fd5 100644 --- a/kernel/relayflowd/Cargo.toml +++ b/kernel/relayflowd/Cargo.toml @@ -20,4 +20,3 @@ libc.workspace = true [dev-dependencies] tempfile.workspace = true - diff --git a/kernel/relayflowd/src/lib.rs b/kernel/relayflowd/src/lib.rs index d9315a270..a90a98c37 100644 --- a/kernel/relayflowd/src/lib.rs +++ b/kernel/relayflowd/src/lib.rs @@ -5,6 +5,6 @@ pub mod server; pub mod worker; pub use engine::{ - DriveOptions, Engine, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, StepSnapshot, - StepStatus, + DriveOptions, Engine, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, + StepSnapshot, StepStatus, }; diff --git a/ops/NEXT.md b/ops/NEXT.md index e65b9435e..45ffd6e61 100644 --- a/ops/NEXT.md +++ b/ops/NEXT.md @@ -1,108 +1,71 @@ -# NEXT — WP-GATE2-FINAL: Real workload proof for gate 2 +# NEXT — WP-GATE2-POLLER: Implement HN poller to make hn-monitor actually monitor **Target gate:** Gate 2 (per ops/TARGET.md — this run is pinned to gate 2 only) -**Work package:** WP-GATE2-FINAL — Build hn-monitor as event-triggered flow +**Work package:** WP-GATE2-POLLER — Implement deterministic HN poller ## Objective -Close gate 2 by building a **real proactive workload** that runs as a relayflow. Gate 2 primitives are DONE per ops/STATE.md (engine/wake.rs assembles wake-time context; kernel/relayflowd/tests/event_wake.rs proves one wake per unique event). RFC-0001 §3 gate 2 done-when requires the real workload, not just primitives. +Implement a deterministic poller that fetches `https://hacker-news.firebaseio.com/v0/topstories.json`, takes the first few story IDs, and submits each through `Engine::submit_event` so hn-monitor wakes on real HN data with dedupe. -Build **hn-monitor** as the smallest honest event-triggered flow that: -- Subscribes to HN story events (simulated or real webhook payload structure) -- Wakes on matching event -- Performs a real agent task (analyze story, check criteria, post summary) -- Exercises the wake path end-to-end +**Context from ops/TARGET.md:** PR #15 landed `testdata/hn-monitor.flow.yaml` and `kernel/relayflowd/tests/hn_monitor_integration.rs`, but the triggering event comes from test code, not Hacker News. Gate 2 is AMBER. **The previous run wrote a work package and no code — do not repeat that. This is a CODE task: write the missing poller.** -Constraint: ONE cycle (~10 minutes of build time) — build the smallest true version, not a complete production system. +**Current state:** The `Engine::submit_event` path exists and works (kernel/relayflowd/src/engine/wake.rs:19). The hn-monitor flow exists and the integration test proves event → wake → park works. **The only missing piece is the poller that fetches real HN data and calls submit_event.** ## Files in scope -### New flow file: -- `testdata/hn-monitor.flow.yaml` — event-triggered flow with: - - Trigger subscribing to `hn.story_posted` event type - - Pattern matching for stories (e.g., minimum score threshold) - - Dedupe key template to prevent double-processing - - Agent step that analyzes the story from wake context - - Simple verification (e.g., output must mention the story title) - -### SDK: -- `sdk/src/compile.ts` — ensure event-triggered flows compile correctly (likely already works) -- `sdk/dist/cli.js` — must resolve the hn-monitor flow via `check` command - -### Kernel test: -- `kernel/relayflowd/tests/hn_monitor_integration.rs` (NEW) — integration test that: - - Loads the hn-monitor flow spec - - Submits a simulated HN event via `submit_event` - - Asserts the run spawns and reaches Parked state - - Verifies journal entries (EventReceived, SubscriptionMatched) - - Verifies wake context contains the event payload - - Submits duplicate event, asserts dedupe works (no second run) - -OR extend existing `event_wake.rs` to use the hn-monitor flow instead of event-triggered-flow. - -## Definition of done - -All of the following must pass: - -1. **Flow file exists and resolves:** - ```bash - cd sdk && node dist/cli.js check ../testdata/hn-monitor.flow.yaml - ``` - Must succeed with preflight warnings (no executor registered, CLI missing) but NOT refuse for schema violations. - -2. **Kernel test passes:** - ```bash - cd kernel && sh ../ops/cargo.sh test - ``` - Including a test that drives hn-monitor through `submit_event` with a realistic HN story event payload. - -3. **The flow is honest, not mock:** - - Event payload structure matches real HN webhook format (story id, title, url, score, etc.) - - Agent instruction is a real task: "Analyze this HN story and determine if it's relevant to AI agents/automation. Output a summary with: story title, relevance score (1-10), and reasoning." - - Verification gate checks that output contains required fields - - NOT a no-op or echo step - -4. **Wake path is exercised end-to-end:** - - Event submission → pattern matching → subscription claim → wake context assembly → agent receives triggering event in context - - All proven by journal inspection in the test - -5. **No regressions:** - All existing tests still pass. Gate 1 remains green. +**New file to create:** +- `kernel/relayflowd/src/engine/hn_poller.rs` — poller implementation -## Explicitly OUT of scope +**Files to modify:** +- `kernel/relayflowd/src/engine.rs` — add `mod hn_poller;` and `pub use hn_poller::HnPoller;` +- `kernel/relayflowd/src/lib.rs` — re-export HnPoller if needed +- New test file or extend existing test to prove offline operation with recorded payload + +## Definition of done (all three required per ops/TARGET.md) + +1. **A new committed source file** implementing the poller exists at `kernel/relayflowd/src/engine/hn_poller.rs` with: + - Fetch `https://hacker-news.firebaseio.com/v0/topstories.json` (returns `Vec` story IDs) + - Take first N IDs (configurable, default 5) + - For each ID, construct an `Event` matching hn-monitor's trigger pattern: + - `event_type: "hn.story_posted"` + - `payload` with at least `{"id": , "type": "story"}` (matches pattern in hn-monitor.flow.yaml:9) + - Submit via `Engine::submit_event` with the hn-monitor spec + - Dedupe works: same story submitted twice yields `matched: true, deduped: true` on second call -- **Production deployment** — this is a flow file that proves the pattern, not deployed infrastructure -- **Real HN API integration** — simulated events are fine; no network calls to HN required -- **Webhook server** — event submission is via kernel's `submit_event` API, not HTTP webhook ingress (that's a future WP) -- **Multiple triggers or complex patterns** — one trigger, one pattern, one subscription -- **Trigger liveness/staleness detection** — defer to later -- **Persona import** — defer to later -- **hn-monitor's full feature set** — build the **smallest honest version** that exercises the wake path, not feature-complete hn-monitor -- **Gates 1, 3-9** — this run is pinned to gate 2 +2. **`cd kernel && sh ../ops/cargo.sh test` passes** including a test that exercises the poller offline from a recorded payload (no live network call in test). Test should verify: + - Parsing topstories JSON (`[41380628, 41378954, ...]`) + - Constructing events with correct structure + - Submitting through submit_event + - Dedupe behavior (second submit is deduped) -## Current state +3. **The run's diff contains real code outside ops/**: The poller must be substantive Rust code, not just documentation. -**What exists:** -- Event primitives: wake.rs, event.rs, dedupe, pattern matching (PR #14, merged) -- Test proving primitives: event_wake.rs passes (verified 2026-08-28 23:40 UTC per STATE.md) -- Generic event-triggered-flow.yaml demonstrates the mechanics +## Implementation approach + +- **Deterministic and testable**: Use a trait or function parameter to inject the JSON source, allowing tests to supply a recorded payload instead of hitting the network +- **Minimal scope**: ONE cycle (~10 minutes). Small working poller beats large plan. + - No daemon/background loop (just a sync function that polls once) + - No full story metadata fetching (topstories only gives IDs; construct minimal events) + - No retry/backoff logic (fail fast is fine for this proof) +- **Error handling**: If fetch fails, return an error +- **Payload structure**: Match what hn_monitor_integration.rs expects (see testdata/hn-monitor.flow.yaml pattern) + +## Explicitly OUT of scope -**What's missing (this WP delivers):** -- A real workload flow (hn-monitor) vs. a generic test fixture -- The bar shift from "primitives work" to "real workload runs as a relayflow" (RFC-0001 §3 rule 2) +- Daemon/background polling infrastructure +- Full HN story metadata (individual `/v0/item/{id}.json` fetches) +- Retry/backoff for network failures +- CLI commands to invoke the poller +- Configuration files +- Changes to hn-monitor flow spec or existing tests (beyond adding the poller test) +- Any RFC or charter edits +- Gates 1, 3-9 -**Risk assessment:** -- Time budget: ~10 minutes compile time -- Scope: Can be minimal — one trigger, one step, honest task -- Known working: event_wake.rs already proves the kernel path works -- This WP is about authoring the flow and proving it compiles/resolves, not building new kernel code +## Why this is the right work package -## Next step after this WP +Per ops/TARGET.md: "PR #15 landed testdata/hn-monitor.flow.yaml... but the triggering event comes from a test rather than Hacker News, so gate 2 is AMBER. Write the missing piece: a deterministic poller..." -If this WP completes and gate 2 is green, the Lead writes ops/GATE2-EVIDENCE.md documenting: -- Flow file path -- Test proving it works -- RFC-0001 §3 gate 2 done-when satisfied: "a real proactive workload runs as a relayflow" +Gate 2's done-when (RFC-0001 §3): "a real proactive workload runs as a relayflow." The flow exists, the wake path works, but it's not monitoring anything real yet. The poller completes the circuit. -If blocked or the real hn-monitor scope is too large for one cycle, report in ops/NEEDS_HUMAN.md and still end with ASSESS_DONE. +**ONE cycle, about ten minutes — a small working poller beats a large plan.** diff --git a/sdk/src/hn-poller.ts b/sdk/src/hn-poller.ts new file mode 100644 index 000000000..6e483c70e --- /dev/null +++ b/sdk/src/hn-poller.ts @@ -0,0 +1,81 @@ +/** + * Hacker News -> relayflow events. + * + * This lives OUTSIDE `kernel/` deliberately. An earlier version called Hacker + * News from `kernel/relayflowd` and review rejected it (PR #16, P1): a + * durable-execution kernel must not own provider-specific product logic or + * network I/O, or engine availability and dependencies become coupled to an + * external service. The kernel gained a `ureq` dependency purely to fetch a + * JSON feed — a clear sign the code was in the wrong place. + * + * So the adapter sits on the authoring surface and submits its events through + * the journal protocol (`event.submit`), which is the same path any other + * external producer would use. The kernel learns about Hacker News the way it + * learns about everything else: as an event. + */ + +const TOP_STORIES_URL = 'https://hacker-news.firebaseio.com/v0/topstories.json'; +const DEFAULT_STORY_LIMIT = 5; + +/** Anything that can submit an event through the journal protocol. */ +export interface EventSink { + eventSubmit(spec: unknown, event: { type: string; payload?: unknown; key?: string }): Promise; +} + +/** Injected so parsing and submission stay deterministic in tests. */ +export type Fetcher = (url: string) => Promise; + +const defaultFetcher: Fetcher = async (url) => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HN fetch failed: HTTP ${response.status}`); + } + return response.text(); +}; + +export interface PollOptions { + storyLimit?: number; + fetcher?: Fetcher; + createdBy?: string; +} + +/** + * Fetch the top-stories feed once and submit each story as an event. + * + * Dedupe is the kernel's job, not ours: the flow's `dedupeKeyTemplate` plus the + * (flow, subscription, key) claim means submitting the same story twice wakes + * it once. This function deliberately does not track what it has already seen. + */ +export async function pollHackerNewsOnce( + spec: unknown, + sink: EventSink, + options: PollOptions = {}, +): Promise { + const storyLimit = options.storyLimit ?? DEFAULT_STORY_LIMIT; + const fetcher = options.fetcher ?? defaultFetcher; + + const body = await fetcher(TOP_STORIES_URL); + + let storyIds: unknown; + try { + storyIds = JSON.parse(body); + } catch (cause) { + throw new Error(`HN top stories response was not JSON: ${String(cause)}`); + } + if (!Array.isArray(storyIds)) { + throw new Error('HN top stories response was not an array'); + } + + const outcomes: unknown[] = []; + for (const id of storyIds.slice(0, storyLimit)) { + outcomes.push( + await sink.eventSubmit(spec, { + type: 'hn.story_posted', + payload: { id, type: 'story' }, + }), + ); + } + return outcomes; +} + +export const HN_TOP_STORIES_URL = TOP_STORIES_URL; diff --git a/sdk/src/index.ts b/sdk/src/index.ts index 044b7ccda..a3ab5bcc9 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -112,3 +112,12 @@ export type { export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; export { JournalClient, type JournalClientOptions } from './journal-client.js'; + +// Hacker News adapter — deliberately outside kernel/ (see sdk/src/hn-poller.ts). +export { + pollHackerNewsOnce, + HN_TOP_STORIES_URL, + type EventSink, + type Fetcher, + type PollOptions, +} from './hn-poller.js'; diff --git a/sdk/tests/hn-poller.test.ts b/sdk/tests/hn-poller.test.ts new file mode 100644 index 000000000..30c894d2a --- /dev/null +++ b/sdk/tests/hn-poller.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; +import { pollHackerNewsOnce, HN_TOP_STORIES_URL } from '../src/hn-poller.js'; + +/** A recorded payload: the test never touches the network. */ +const RECORDED_TOP_STORIES = '[41000001, 41000002, 41000003, 41000004, 41000005, 41000006]'; + +function recordingSink() { + const submitted: Array<{ spec: unknown; event: { type: string; payload?: unknown } }> = []; + return { + submitted, + async eventSubmit(spec: unknown, event: { type: string; payload?: unknown }) { + submitted.push({ spec, event }); + return { matched: true, deduped: false }; + }, + }; +} + +describe('hn poller', () => { + it('submits one event per story, up to the limit, through the journal protocol', async () => { + const sink = recordingSink(); + const spec = { name: 'hn-monitor' }; + + await pollHackerNewsOnce(spec, sink, { + storyLimit: 3, + fetcher: async (url) => { + expect(url).toBe(HN_TOP_STORIES_URL); + return RECORDED_TOP_STORIES; + }, + }); + + expect(sink.submitted).toHaveLength(3); + expect(sink.submitted[0].event.type).toBe('hn.story_posted'); + expect(sink.submitted[0].event.payload).toEqual({ id: 41000001, type: 'story' }); + expect(sink.submitted[2].event.payload).toEqual({ id: 41000003, type: 'story' }); + }); + + it('refuses a response that is not a JSON array rather than submitting nothing silently', async () => { + const sink = recordingSink(); + await expect( + pollHackerNewsOnce({}, sink, { fetcher: async () => '{"error":"nope"}' }), + ).rejects.toThrow(/not an array/); + expect(sink.submitted).toHaveLength(0); + }); + + it('does not dedupe locally — that is the kernel\'s claim, not the adapter\'s', async () => { + const sink = recordingSink(); + await pollHackerNewsOnce({}, sink, { storyLimit: 2, fetcher: async () => '[7, 7]' }); + expect(sink.submitted).toHaveLength(2); + }); +});