From 010104bdd998fabfe5af65e2995bf929567efdfb Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Fri, 28 Aug 2026 22:51:33 -0400 Subject: [PATCH 1/2] drive: cloud run a1055874 Work produced by cloud run a1055874-5958-42ed-a888-8b07c5eaa9c1 in a workflow sandbox and delivered from this host, because a sandbox has no remote and no GitHub token. Verification and adversarial review ran in-run; see ops/reviews/ in the diff. --- kernel/Cargo.lock | 277 +++++++++++++++++++++- kernel/Cargo.toml | 2 +- kernel/relayflowd/Cargo.toml | 2 +- kernel/relayflowd/src/engine.rs | 2 + kernel/relayflowd/src/engine/hn_poller.rs | 75 ++++++ kernel/relayflowd/src/lib.rs | 4 +- kernel/relayflowd/tests/hn_poller.rs | 58 +++++ ops/NEXT.md | 139 ++++------- 8 files changed, 462 insertions(+), 97 deletions(-) create mode 100644 kernel/relayflowd/src/engine/hn_poller.rs create mode 100644 kernel/relayflowd/tests/hn_poller.rs diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index c7d7cda40..e11cc797a 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -61,7 +67,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -72,7 +78,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -212,6 +218,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -259,7 +274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -297,6 +312,17 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fluent-uri" version = "0.3.2" @@ -314,6 +340,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fraction" version = "0.15.4" @@ -358,6 +393,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -602,12 +648,28 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num" version = "0.4.3" @@ -908,6 +970,7 @@ dependencies = [ "sha2", "tempfile", "ulid", + "ureq", "wait-timeout", ] @@ -934,6 +997,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rusqlite" version = "0.37.0" @@ -958,7 +1035,42 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1033,6 +1145,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1057,6 +1175,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -1100,7 +1224,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1156,6 +1280,40 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + [[package]] name = "utf8_iter" version = "1.0.4" @@ -1216,6 +1374,12 @@ dependencies = [ "libc", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -1280,12 +1444,39 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1295,6 +1486,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -1371,6 +1626,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.5" @@ -1404,6 +1665,12 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 437336999..0121fcb8a 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -19,5 +19,5 @@ sha2 = "0.10" tempfile = "3.20" thiserror = "2.0" ulid = { version = "1.2", features = ["serde"] } +ureq = "2.12" wait-timeout = "0.2" - diff --git a/kernel/relayflowd/Cargo.toml b/kernel/relayflowd/Cargo.toml index 3b69b593f..fff1f3775 100644 --- a/kernel/relayflowd/Cargo.toml +++ b/kernel/relayflowd/Cargo.toml @@ -13,6 +13,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true ulid.workspace = true +ureq.workspace = true wait-timeout.workspace = true [target.'cfg(unix)'.dependencies] @@ -20,4 +21,3 @@ libc.workspace = true [dev-dependencies] tempfile.workspace = true - diff --git a/kernel/relayflowd/src/engine.rs b/kernel/relayflowd/src/engine.rs index c1f7ec0ab..d6413fc41 100644 --- a/kernel/relayflowd/src/engine.rs +++ b/kernel/relayflowd/src/engine.rs @@ -17,9 +17,11 @@ use crate::worker::{JournalObserver, StepDispatcher}; mod drive; mod effects; +mod hn_poller; mod model; mod remote; mod wake; +pub use hn_poller::HnPoller; pub use model::{RunOutcome, RunSnapshot, RunStatus, StepSnapshot, StepStatus}; use model::{outcome_from_state, snapshot_from_state}; pub use remote::OutOfBandCompletion; diff --git a/kernel/relayflowd/src/engine/hn_poller.rs b/kernel/relayflowd/src/engine/hn_poller.rs new file mode 100644 index 000000000..0cc80c7e9 --- /dev/null +++ b/kernel/relayflowd/src/engine/hn_poller.rs @@ -0,0 +1,75 @@ +use anyhow::{Context, Result}; +use relayflowd_core::{Clock, Event, RunSpec}; +use serde_json::json; + +use super::{Engine, EventSubmitOutcome}; + +const TOP_STORIES_URL: &str = "https://hacker-news.firebaseio.com/v0/topstories.json"; +const DEFAULT_STORY_LIMIT: usize = 5; + +/// One-shot adapter from Hacker News' top-stories feed to relayflow events. +pub struct HnPoller { + spec: RunSpec, + story_limit: usize, +} + +impl HnPoller { + pub fn new(spec: RunSpec) -> Self { + Self { + spec, + story_limit: DEFAULT_STORY_LIMIT, + } + } + + pub fn with_story_limit(mut self, story_limit: usize) -> Self { + self.story_limit = story_limit; + self + } + + pub fn poll_once( + &self, + engine: &Engine, + created_by: &str, + ) -> Result> { + self.poll_once_with(engine, created_by, |url| { + ureq::get(url) + .call() + .context("fetch HN top stories")? + .into_string() + .context("read HN top stories response") + }) + } + + /// Injecting the fetch operation keeps parsing and event submission fully + /// deterministic in tests without weakening the production fetch path. + pub fn poll_once_with( + &self, + engine: &Engine, + created_by: &str, + fetch: F, + ) -> Result> + where + C: Clock, + F: FnOnce(&str) -> Result, + { + let body = fetch(TOP_STORIES_URL)?; + let story_ids: Vec = + serde_json::from_str(&body).context("parse HN top stories response")?; + + story_ids + .into_iter() + .take(self.story_limit) + .map(|id| { + engine.submit_event( + self.spec.clone(), + Event { + event_type: "hn.story_posted".into(), + payload: json!({"id": id, "type": "story"}), + key: None, + }, + created_by, + ) + }) + .collect() + } +} diff --git a/kernel/relayflowd/src/lib.rs b/kernel/relayflowd/src/lib.rs index d9315a270..5b2ce4013 100644 --- a/kernel/relayflowd/src/lib.rs +++ b/kernel/relayflowd/src/lib.rs @@ -5,6 +5,6 @@ pub mod server; pub mod worker; pub use engine::{ - DriveOptions, Engine, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, StepSnapshot, - StepStatus, + DriveOptions, Engine, HnPoller, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, + StepSnapshot, StepStatus, }; diff --git a/kernel/relayflowd/tests/hn_poller.rs b/kernel/relayflowd/tests/hn_poller.rs new file mode 100644 index 000000000..440198287 --- /dev/null +++ b/kernel/relayflowd/tests/hn_poller.rs @@ -0,0 +1,58 @@ +use relayflowd::{Engine, HnPoller}; +use relayflowd_core::{EntryType, RunSpec}; + +const RECORDED_TOP_STORIES: &str = "[41380628, 41378954, 41379517]"; + +fn hn_monitor_spec() -> RunSpec { + let value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/hn-monitor.spec.canonical.json" + ))) + .unwrap(); + RunSpec::parse(&value).unwrap() +} + +#[test] +fn recorded_top_stories_are_submitted_and_deduped() { + let directory = tempfile::tempdir().unwrap(); + let engine = Engine::new(directory.path()); + let poller = HnPoller::new(hn_monitor_spec()).with_story_limit(2); + let fetch = |url: &str| { + assert_eq!(url, "https://hacker-news.firebaseio.com/v0/topstories.json"); + Ok(RECORDED_TOP_STORIES.to_owned()) + }; + + let first = poller.poll_once_with(&engine, "hn-poller", fetch).unwrap(); + assert_eq!(first.len(), 2); + assert!( + first + .iter() + .all(|outcome| outcome.matched && !outcome.deduped) + ); + + let first_run = first[0].run.as_ref().unwrap(); + let received = engine + .journal_entries(&first_run.run_id, 1, 100) + .unwrap() + .into_iter() + .find(|entry| entry.entry_type == EntryType::EventReceived) + .unwrap(); + assert_eq!(received.payload["event"]["type"], "hn.story_posted"); + assert_eq!(received.payload["event"]["payload"]["id"], 41380628); + assert_eq!(received.payload["event"]["payload"]["type"], "story"); + + let second = poller + .poll_once_with( + &engine, + "hn-poller", + |_| Ok(RECORDED_TOP_STORIES.to_owned()), + ) + .unwrap(); + assert_eq!(second.len(), 2); + assert!( + second + .iter() + .all(|outcome| outcome.matched && outcome.deduped) + ); + assert!(second.iter().all(|outcome| outcome.run.is_none())); +} diff --git a/ops/NEXT.md b/ops/NEXT.md index e65b9435e..45ffd6e61 100644 --- a/ops/NEXT.md +++ b/ops/NEXT.md @@ -1,108 +1,71 @@ -# NEXT — WP-GATE2-FINAL: Real workload proof for gate 2 +# NEXT — WP-GATE2-POLLER: Implement HN poller to make hn-monitor actually monitor **Target gate:** Gate 2 (per ops/TARGET.md — this run is pinned to gate 2 only) -**Work package:** WP-GATE2-FINAL — Build hn-monitor as event-triggered flow +**Work package:** WP-GATE2-POLLER — Implement deterministic HN poller ## Objective -Close gate 2 by building a **real proactive workload** that runs as a relayflow. Gate 2 primitives are DONE per ops/STATE.md (engine/wake.rs assembles wake-time context; kernel/relayflowd/tests/event_wake.rs proves one wake per unique event). RFC-0001 §3 gate 2 done-when requires the real workload, not just primitives. +Implement a deterministic poller that fetches `https://hacker-news.firebaseio.com/v0/topstories.json`, takes the first few story IDs, and submits each through `Engine::submit_event` so hn-monitor wakes on real HN data with dedupe. -Build **hn-monitor** as the smallest honest event-triggered flow that: -- Subscribes to HN story events (simulated or real webhook payload structure) -- Wakes on matching event -- Performs a real agent task (analyze story, check criteria, post summary) -- Exercises the wake path end-to-end +**Context from ops/TARGET.md:** PR #15 landed `testdata/hn-monitor.flow.yaml` and `kernel/relayflowd/tests/hn_monitor_integration.rs`, but the triggering event comes from test code, not Hacker News. Gate 2 is AMBER. **The previous run wrote a work package and no code — do not repeat that. This is a CODE task: write the missing poller.** -Constraint: ONE cycle (~10 minutes of build time) — build the smallest true version, not a complete production system. +**Current state:** The `Engine::submit_event` path exists and works (kernel/relayflowd/src/engine/wake.rs:19). The hn-monitor flow exists and the integration test proves event → wake → park works. **The only missing piece is the poller that fetches real HN data and calls submit_event.** ## Files in scope -### New flow file: -- `testdata/hn-monitor.flow.yaml` — event-triggered flow with: - - Trigger subscribing to `hn.story_posted` event type - - Pattern matching for stories (e.g., minimum score threshold) - - Dedupe key template to prevent double-processing - - Agent step that analyzes the story from wake context - - Simple verification (e.g., output must mention the story title) - -### SDK: -- `sdk/src/compile.ts` — ensure event-triggered flows compile correctly (likely already works) -- `sdk/dist/cli.js` — must resolve the hn-monitor flow via `check` command - -### Kernel test: -- `kernel/relayflowd/tests/hn_monitor_integration.rs` (NEW) — integration test that: - - Loads the hn-monitor flow spec - - Submits a simulated HN event via `submit_event` - - Asserts the run spawns and reaches Parked state - - Verifies journal entries (EventReceived, SubscriptionMatched) - - Verifies wake context contains the event payload - - Submits duplicate event, asserts dedupe works (no second run) - -OR extend existing `event_wake.rs` to use the hn-monitor flow instead of event-triggered-flow. - -## Definition of done - -All of the following must pass: - -1. **Flow file exists and resolves:** - ```bash - cd sdk && node dist/cli.js check ../testdata/hn-monitor.flow.yaml - ``` - Must succeed with preflight warnings (no executor registered, CLI missing) but NOT refuse for schema violations. - -2. **Kernel test passes:** - ```bash - cd kernel && sh ../ops/cargo.sh test - ``` - Including a test that drives hn-monitor through `submit_event` with a realistic HN story event payload. - -3. **The flow is honest, not mock:** - - Event payload structure matches real HN webhook format (story id, title, url, score, etc.) - - Agent instruction is a real task: "Analyze this HN story and determine if it's relevant to AI agents/automation. Output a summary with: story title, relevance score (1-10), and reasoning." - - Verification gate checks that output contains required fields - - NOT a no-op or echo step - -4. **Wake path is exercised end-to-end:** - - Event submission → pattern matching → subscription claim → wake context assembly → agent receives triggering event in context - - All proven by journal inspection in the test - -5. **No regressions:** - All existing tests still pass. Gate 1 remains green. +**New file to create:** +- `kernel/relayflowd/src/engine/hn_poller.rs` — poller implementation -## Explicitly OUT of scope +**Files to modify:** +- `kernel/relayflowd/src/engine.rs` — add `mod hn_poller;` and `pub use hn_poller::HnPoller;` +- `kernel/relayflowd/src/lib.rs` — re-export HnPoller if needed +- New test file or extend existing test to prove offline operation with recorded payload + +## Definition of done (all three required per ops/TARGET.md) + +1. **A new committed source file** implementing the poller exists at `kernel/relayflowd/src/engine/hn_poller.rs` with: + - Fetch `https://hacker-news.firebaseio.com/v0/topstories.json` (returns `Vec` story IDs) + - Take first N IDs (configurable, default 5) + - For each ID, construct an `Event` matching hn-monitor's trigger pattern: + - `event_type: "hn.story_posted"` + - `payload` with at least `{"id": , "type": "story"}` (matches pattern in hn-monitor.flow.yaml:9) + - Submit via `Engine::submit_event` with the hn-monitor spec + - Dedupe works: same story submitted twice yields `matched: true, deduped: true` on second call -- **Production deployment** — this is a flow file that proves the pattern, not deployed infrastructure -- **Real HN API integration** — simulated events are fine; no network calls to HN required -- **Webhook server** — event submission is via kernel's `submit_event` API, not HTTP webhook ingress (that's a future WP) -- **Multiple triggers or complex patterns** — one trigger, one pattern, one subscription -- **Trigger liveness/staleness detection** — defer to later -- **Persona import** — defer to later -- **hn-monitor's full feature set** — build the **smallest honest version** that exercises the wake path, not feature-complete hn-monitor -- **Gates 1, 3-9** — this run is pinned to gate 2 +2. **`cd kernel && sh ../ops/cargo.sh test` passes** including a test that exercises the poller offline from a recorded payload (no live network call in test). Test should verify: + - Parsing topstories JSON (`[41380628, 41378954, ...]`) + - Constructing events with correct structure + - Submitting through submit_event + - Dedupe behavior (second submit is deduped) -## Current state +3. **The run's diff contains real code outside ops/**: The poller must be substantive Rust code, not just documentation. -**What exists:** -- Event primitives: wake.rs, event.rs, dedupe, pattern matching (PR #14, merged) -- Test proving primitives: event_wake.rs passes (verified 2026-08-28 23:40 UTC per STATE.md) -- Generic event-triggered-flow.yaml demonstrates the mechanics +## Implementation approach + +- **Deterministic and testable**: Use a trait or function parameter to inject the JSON source, allowing tests to supply a recorded payload instead of hitting the network +- **Minimal scope**: ONE cycle (~10 minutes). Small working poller beats large plan. + - No daemon/background loop (just a sync function that polls once) + - No full story metadata fetching (topstories only gives IDs; construct minimal events) + - No retry/backoff logic (fail fast is fine for this proof) +- **Error handling**: If fetch fails, return an error +- **Payload structure**: Match what hn_monitor_integration.rs expects (see testdata/hn-monitor.flow.yaml pattern) + +## Explicitly OUT of scope -**What's missing (this WP delivers):** -- A real workload flow (hn-monitor) vs. a generic test fixture -- The bar shift from "primitives work" to "real workload runs as a relayflow" (RFC-0001 §3 rule 2) +- Daemon/background polling infrastructure +- Full HN story metadata (individual `/v0/item/{id}.json` fetches) +- Retry/backoff for network failures +- CLI commands to invoke the poller +- Configuration files +- Changes to hn-monitor flow spec or existing tests (beyond adding the poller test) +- Any RFC or charter edits +- Gates 1, 3-9 -**Risk assessment:** -- Time budget: ~10 minutes compile time -- Scope: Can be minimal — one trigger, one step, honest task -- Known working: event_wake.rs already proves the kernel path works -- This WP is about authoring the flow and proving it compiles/resolves, not building new kernel code +## Why this is the right work package -## Next step after this WP +Per ops/TARGET.md: "PR #15 landed testdata/hn-monitor.flow.yaml... but the triggering event comes from a test rather than Hacker News, so gate 2 is AMBER. Write the missing piece: a deterministic poller..." -If this WP completes and gate 2 is green, the Lead writes ops/GATE2-EVIDENCE.md documenting: -- Flow file path -- Test proving it works -- RFC-0001 §3 gate 2 done-when satisfied: "a real proactive workload runs as a relayflow" +Gate 2's done-when (RFC-0001 §3): "a real proactive workload runs as a relayflow." The flow exists, the wake path works, but it's not monitoring anything real yet. The poller completes the circuit. -If blocked or the real hn-monitor scope is too large for one cycle, report in ops/NEEDS_HUMAN.md and still end with ASSESS_DONE. +**ONE cycle, about ten minutes — a small working poller beats a large plan.** From e59154436ad120e345bc17139792d63ca06f7f9e Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Sat, 29 Aug 2026 01:04:22 -0400 Subject: [PATCH 2/2] fix: move the Hacker News adapter out of the kernel (P1 review finding) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review was right: calling Hacker News from kernel/relayflowd made the durable-execution kernel own provider-specific product logic and network I/O, coupling engine availability and dependencies to an external service. The kernel had gained a ureq dependency purely to fetch a JSON feed, which is itself the tell that the code was in the wrong place. Removed from the kernel: engine/hn_poller.rs, tests/hn_poller.rs, the HnPoller export, and ureq from both manifests. The kernel no longer has an HTTP client. Added on the authoring surface: sdk/src/hn-poller.ts, which submits each story through event.submit — the journal protocol path any external producer would use — so the kernel learns about Hacker News the way it learns about everything else, as an event. It deliberately does not dedupe locally: that is the kernel's (flow, subscription, key) claim, and a test asserts the adapter submits duplicates rather than quietly swallowing them. Two more tests cover the happy path from a recorded payload and a refusal on a non-array response. Verified locally: kernel 19+19+1+1+26+5+6 passed / 0 failed with ureq gone; sdk 153 passed across 10 files; npx tsc --noEmit clean. Co-Authored-By: Claude Fable 5 --- kernel/Cargo.lock | 277 +--------------------- kernel/Cargo.toml | 1 - kernel/relayflowd/Cargo.toml | 1 - kernel/relayflowd/src/engine.rs | 2 - kernel/relayflowd/src/engine/hn_poller.rs | 75 ------ kernel/relayflowd/src/lib.rs | 2 +- kernel/relayflowd/tests/hn_poller.rs | 58 ----- sdk/src/hn-poller.ts | 81 +++++++ sdk/src/index.ts | 9 + sdk/tests/hn-poller.test.ts | 50 ++++ 10 files changed, 146 insertions(+), 410 deletions(-) delete mode 100644 kernel/relayflowd/src/engine/hn_poller.rs delete mode 100644 kernel/relayflowd/tests/hn_poller.rs create mode 100644 sdk/src/hn-poller.ts create mode 100644 sdk/tests/hn-poller.test.ts diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index e11cc797a..c7d7cda40 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "ahash" version = "0.8.12" @@ -67,7 +61,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -78,7 +72,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -218,15 +212,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" -dependencies = [ - "cfg-if", -] - [[package]] name = "crypto-common" version = "0.1.7" @@ -274,7 +259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -312,17 +297,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" -[[package]] -name = "flate2" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - [[package]] name = "fluent-uri" version = "0.3.2" @@ -340,15 +314,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - [[package]] name = "fraction" version = "0.15.4" @@ -393,17 +358,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "getrandom" version = "0.3.4" @@ -648,28 +602,12 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "num" version = "0.4.3" @@ -970,7 +908,6 @@ dependencies = [ "sha2", "tempfile", "ulid", - "ureq", "wait-timeout", ] @@ -997,20 +934,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rusqlite" version = "0.37.0" @@ -1035,42 +958,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls" -version = "0.23.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" -dependencies = [ - "log", - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", + "windows-sys", ] [[package]] @@ -1145,12 +1033,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - [[package]] name = "slab" version = "0.4.12" @@ -1175,12 +1057,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.119" @@ -1224,7 +1100,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1280,40 +1156,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "ureq" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" -dependencies = [ - "base64", - "flate2", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "url", - "webpki-roots 0.26.11", -] - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - [[package]] name = "utf8_iter" version = "1.0.4" @@ -1374,12 +1216,6 @@ dependencies = [ "libc", ] -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -1444,39 +1280,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.9", -] - -[[package]] -name = "webpki-roots" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1486,70 +1295,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -1626,12 +1371,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - [[package]] name = "zerotrie" version = "0.2.5" @@ -1665,12 +1404,6 @@ dependencies = [ "syn 3.0.4", ] -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" version = "1.0.23" diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 0121fcb8a..0449e98f5 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -19,5 +19,4 @@ sha2 = "0.10" tempfile = "3.20" thiserror = "2.0" ulid = { version = "1.2", features = ["serde"] } -ureq = "2.12" wait-timeout = "0.2" diff --git a/kernel/relayflowd/Cargo.toml b/kernel/relayflowd/Cargo.toml index fff1f3775..b23bb3fd5 100644 --- a/kernel/relayflowd/Cargo.toml +++ b/kernel/relayflowd/Cargo.toml @@ -13,7 +13,6 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true ulid.workspace = true -ureq.workspace = true wait-timeout.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/kernel/relayflowd/src/engine.rs b/kernel/relayflowd/src/engine.rs index d6413fc41..c1f7ec0ab 100644 --- a/kernel/relayflowd/src/engine.rs +++ b/kernel/relayflowd/src/engine.rs @@ -17,11 +17,9 @@ use crate::worker::{JournalObserver, StepDispatcher}; mod drive; mod effects; -mod hn_poller; mod model; mod remote; mod wake; -pub use hn_poller::HnPoller; pub use model::{RunOutcome, RunSnapshot, RunStatus, StepSnapshot, StepStatus}; use model::{outcome_from_state, snapshot_from_state}; pub use remote::OutOfBandCompletion; diff --git a/kernel/relayflowd/src/engine/hn_poller.rs b/kernel/relayflowd/src/engine/hn_poller.rs deleted file mode 100644 index 0cc80c7e9..000000000 --- a/kernel/relayflowd/src/engine/hn_poller.rs +++ /dev/null @@ -1,75 +0,0 @@ -use anyhow::{Context, Result}; -use relayflowd_core::{Clock, Event, RunSpec}; -use serde_json::json; - -use super::{Engine, EventSubmitOutcome}; - -const TOP_STORIES_URL: &str = "https://hacker-news.firebaseio.com/v0/topstories.json"; -const DEFAULT_STORY_LIMIT: usize = 5; - -/// One-shot adapter from Hacker News' top-stories feed to relayflow events. -pub struct HnPoller { - spec: RunSpec, - story_limit: usize, -} - -impl HnPoller { - pub fn new(spec: RunSpec) -> Self { - Self { - spec, - story_limit: DEFAULT_STORY_LIMIT, - } - } - - pub fn with_story_limit(mut self, story_limit: usize) -> Self { - self.story_limit = story_limit; - self - } - - pub fn poll_once( - &self, - engine: &Engine, - created_by: &str, - ) -> Result> { - self.poll_once_with(engine, created_by, |url| { - ureq::get(url) - .call() - .context("fetch HN top stories")? - .into_string() - .context("read HN top stories response") - }) - } - - /// Injecting the fetch operation keeps parsing and event submission fully - /// deterministic in tests without weakening the production fetch path. - pub fn poll_once_with( - &self, - engine: &Engine, - created_by: &str, - fetch: F, - ) -> Result> - where - C: Clock, - F: FnOnce(&str) -> Result, - { - let body = fetch(TOP_STORIES_URL)?; - let story_ids: Vec = - serde_json::from_str(&body).context("parse HN top stories response")?; - - story_ids - .into_iter() - .take(self.story_limit) - .map(|id| { - engine.submit_event( - self.spec.clone(), - Event { - event_type: "hn.story_posted".into(), - payload: json!({"id": id, "type": "story"}), - key: None, - }, - created_by, - ) - }) - .collect() - } -} diff --git a/kernel/relayflowd/src/lib.rs b/kernel/relayflowd/src/lib.rs index 5b2ce4013..a90a98c37 100644 --- a/kernel/relayflowd/src/lib.rs +++ b/kernel/relayflowd/src/lib.rs @@ -5,6 +5,6 @@ pub mod server; pub mod worker; pub use engine::{ - DriveOptions, Engine, HnPoller, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, + DriveOptions, Engine, OutOfBandCompletion, RunOutcome, RunSnapshot, RunStatus, StepSnapshot, StepStatus, }; diff --git a/kernel/relayflowd/tests/hn_poller.rs b/kernel/relayflowd/tests/hn_poller.rs deleted file mode 100644 index 440198287..000000000 --- a/kernel/relayflowd/tests/hn_poller.rs +++ /dev/null @@ -1,58 +0,0 @@ -use relayflowd::{Engine, HnPoller}; -use relayflowd_core::{EntryType, RunSpec}; - -const RECORDED_TOP_STORIES: &str = "[41380628, 41378954, 41379517]"; - -fn hn_monitor_spec() -> RunSpec { - let value = serde_json::from_str(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../testdata/hn-monitor.spec.canonical.json" - ))) - .unwrap(); - RunSpec::parse(&value).unwrap() -} - -#[test] -fn recorded_top_stories_are_submitted_and_deduped() { - let directory = tempfile::tempdir().unwrap(); - let engine = Engine::new(directory.path()); - let poller = HnPoller::new(hn_monitor_spec()).with_story_limit(2); - let fetch = |url: &str| { - assert_eq!(url, "https://hacker-news.firebaseio.com/v0/topstories.json"); - Ok(RECORDED_TOP_STORIES.to_owned()) - }; - - let first = poller.poll_once_with(&engine, "hn-poller", fetch).unwrap(); - assert_eq!(first.len(), 2); - assert!( - first - .iter() - .all(|outcome| outcome.matched && !outcome.deduped) - ); - - let first_run = first[0].run.as_ref().unwrap(); - let received = engine - .journal_entries(&first_run.run_id, 1, 100) - .unwrap() - .into_iter() - .find(|entry| entry.entry_type == EntryType::EventReceived) - .unwrap(); - assert_eq!(received.payload["event"]["type"], "hn.story_posted"); - assert_eq!(received.payload["event"]["payload"]["id"], 41380628); - assert_eq!(received.payload["event"]["payload"]["type"], "story"); - - let second = poller - .poll_once_with( - &engine, - "hn-poller", - |_| Ok(RECORDED_TOP_STORIES.to_owned()), - ) - .unwrap(); - assert_eq!(second.len(), 2); - assert!( - second - .iter() - .all(|outcome| outcome.matched && outcome.deduped) - ); - assert!(second.iter().all(|outcome| outcome.run.is_none())); -} diff --git a/sdk/src/hn-poller.ts b/sdk/src/hn-poller.ts new file mode 100644 index 000000000..6e483c70e --- /dev/null +++ b/sdk/src/hn-poller.ts @@ -0,0 +1,81 @@ +/** + * Hacker News -> relayflow events. + * + * This lives OUTSIDE `kernel/` deliberately. An earlier version called Hacker + * News from `kernel/relayflowd` and review rejected it (PR #16, P1): a + * durable-execution kernel must not own provider-specific product logic or + * network I/O, or engine availability and dependencies become coupled to an + * external service. The kernel gained a `ureq` dependency purely to fetch a + * JSON feed — a clear sign the code was in the wrong place. + * + * So the adapter sits on the authoring surface and submits its events through + * the journal protocol (`event.submit`), which is the same path any other + * external producer would use. The kernel learns about Hacker News the way it + * learns about everything else: as an event. + */ + +const TOP_STORIES_URL = 'https://hacker-news.firebaseio.com/v0/topstories.json'; +const DEFAULT_STORY_LIMIT = 5; + +/** Anything that can submit an event through the journal protocol. */ +export interface EventSink { + eventSubmit(spec: unknown, event: { type: string; payload?: unknown; key?: string }): Promise; +} + +/** Injected so parsing and submission stay deterministic in tests. */ +export type Fetcher = (url: string) => Promise; + +const defaultFetcher: Fetcher = async (url) => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HN fetch failed: HTTP ${response.status}`); + } + return response.text(); +}; + +export interface PollOptions { + storyLimit?: number; + fetcher?: Fetcher; + createdBy?: string; +} + +/** + * Fetch the top-stories feed once and submit each story as an event. + * + * Dedupe is the kernel's job, not ours: the flow's `dedupeKeyTemplate` plus the + * (flow, subscription, key) claim means submitting the same story twice wakes + * it once. This function deliberately does not track what it has already seen. + */ +export async function pollHackerNewsOnce( + spec: unknown, + sink: EventSink, + options: PollOptions = {}, +): Promise { + const storyLimit = options.storyLimit ?? DEFAULT_STORY_LIMIT; + const fetcher = options.fetcher ?? defaultFetcher; + + const body = await fetcher(TOP_STORIES_URL); + + let storyIds: unknown; + try { + storyIds = JSON.parse(body); + } catch (cause) { + throw new Error(`HN top stories response was not JSON: ${String(cause)}`); + } + if (!Array.isArray(storyIds)) { + throw new Error('HN top stories response was not an array'); + } + + const outcomes: unknown[] = []; + for (const id of storyIds.slice(0, storyLimit)) { + outcomes.push( + await sink.eventSubmit(spec, { + type: 'hn.story_posted', + payload: { id, type: 'story' }, + }), + ); + } + return outcomes; +} + +export const HN_TOP_STORIES_URL = TOP_STORIES_URL; diff --git a/sdk/src/index.ts b/sdk/src/index.ts index 044b7ccda..a3ab5bcc9 100644 --- a/sdk/src/index.ts +++ b/sdk/src/index.ts @@ -112,3 +112,12 @@ export type { export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; export { JournalClient, type JournalClientOptions } from './journal-client.js'; + +// Hacker News adapter — deliberately outside kernel/ (see sdk/src/hn-poller.ts). +export { + pollHackerNewsOnce, + HN_TOP_STORIES_URL, + type EventSink, + type Fetcher, + type PollOptions, +} from './hn-poller.js'; diff --git a/sdk/tests/hn-poller.test.ts b/sdk/tests/hn-poller.test.ts new file mode 100644 index 000000000..30c894d2a --- /dev/null +++ b/sdk/tests/hn-poller.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; +import { pollHackerNewsOnce, HN_TOP_STORIES_URL } from '../src/hn-poller.js'; + +/** A recorded payload: the test never touches the network. */ +const RECORDED_TOP_STORIES = '[41000001, 41000002, 41000003, 41000004, 41000005, 41000006]'; + +function recordingSink() { + const submitted: Array<{ spec: unknown; event: { type: string; payload?: unknown } }> = []; + return { + submitted, + async eventSubmit(spec: unknown, event: { type: string; payload?: unknown }) { + submitted.push({ spec, event }); + return { matched: true, deduped: false }; + }, + }; +} + +describe('hn poller', () => { + it('submits one event per story, up to the limit, through the journal protocol', async () => { + const sink = recordingSink(); + const spec = { name: 'hn-monitor' }; + + await pollHackerNewsOnce(spec, sink, { + storyLimit: 3, + fetcher: async (url) => { + expect(url).toBe(HN_TOP_STORIES_URL); + return RECORDED_TOP_STORIES; + }, + }); + + expect(sink.submitted).toHaveLength(3); + expect(sink.submitted[0].event.type).toBe('hn.story_posted'); + expect(sink.submitted[0].event.payload).toEqual({ id: 41000001, type: 'story' }); + expect(sink.submitted[2].event.payload).toEqual({ id: 41000003, type: 'story' }); + }); + + it('refuses a response that is not a JSON array rather than submitting nothing silently', async () => { + const sink = recordingSink(); + await expect( + pollHackerNewsOnce({}, sink, { fetcher: async () => '{"error":"nope"}' }), + ).rejects.toThrow(/not an array/); + expect(sink.submitted).toHaveLength(0); + }); + + it('does not dedupe locally — that is the kernel\'s claim, not the adapter\'s', async () => { + const sink = recordingSink(); + await pollHackerNewsOnce({}, sink, { storyLimit: 2, fetcher: async () => '[7, 7]' }); + expect(sink.submitted).toHaveLength(2); + }); +});