diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..8b1e8423 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "asap-control-core" +version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..c3365c5b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +members = [ + "crates/core", +] +resolver = "2" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml new file mode 100644 index 00000000..99972465 --- /dev/null +++ b/crates/core/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "asap-control-core" +version = "0.1.0" +edition = "2021" diff --git a/crates/core/src/intent_algebra/expr.rs b/crates/core/src/intent_algebra/expr.rs new file mode 100644 index 00000000..6860d6fc --- /dev/null +++ b/crates/core/src/intent_algebra/expr.rs @@ -0,0 +1,305 @@ +use std::rc::Rc; +use std::time::Duration; + +use crate::types::AccuracyTarget; +use super::schema::{HasSchema, L3Schema, SchemaCatalog}; + +// ── Stub leaf / supporting types ────────────────────────────────────────────── +// Full definitions will be added as the respective layers are implemented. + +/// A row-level filter predicate (WHERE clause / PromQL label matcher). +#[derive(Debug, Clone)] pub struct Predicate; +/// One item in a SELECT projection list. +#[derive(Debug, Clone)] pub struct ProjectItem; +/// A GROUP BY key reference. +#[derive(Debug, Clone)] pub struct GroupKey; +/// A reference to a column by name. +#[derive(Debug, Clone)] pub struct ColumnRef; +/// A set of partitioning keys (sharding hint for L5 stage allocator). +#[derive(Debug, Clone)] pub struct PartitionKeys; +/// One key in an ORDER BY clause. +#[derive(Debug, Clone)] pub struct SortKey; +/// An analytic window frame (ROWS / RANGE BETWEEN …). +#[derive(Debug, Clone)] pub struct WindowFrame; +/// PromQL vector-match modifiers (`on`/`ignoring` + `group_left`/`group_right`). +#[derive(Debug, Clone)] pub struct VectorMatch; +/// Reference to a metric by name (PromQL / OTLP). +#[derive(Debug, Clone)] pub struct MetricRef; +/// Closed time interval for a time-series scan. +#[derive(Debug, Clone)] pub struct TimeRange; +/// Label matchers applied to a time-series scan. +#[derive(Debug, Clone)] pub struct LabelFilter; +/// Reference to a relational table by name. +#[derive(Debug, Clone)] pub struct TableRef; +/// Join key specification (USING / ON column reference). +#[derive(Debug, Clone)] pub struct JoinKey; + +// ── Enum supporting types ───────────────────────────────────────────────────── + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JoinKind { + Inner, + Left, + Right, + Full, + Cross, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SetOpKind { + Union, + Intersect, + Except, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BinaryOpKind { + // Arithmetic + Add, Sub, Mul, Div, Mod, + // Comparison + Eq, NotEq, Lt, LtEq, Gt, GtEq, + // Boolean / PromQL set operators + And, Or, Unless, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WindowFuncKind { + RowNumber, Rank, DenseRank, + Lag, Lead, + FirstValue, LastValue, + NthValue(u64), + Sum, Avg, Count, Min, Max, +} + +/// Which data model a `Source` or `AggIntent` operates over. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DataModel { + TimeSeries, + Tabular, + /// Agnostic — works over either data model. + Any, +} + +// ── Time window kind ────────────────────────────────────────────────────────── + +/// The lifecycle / flush semantics of a streaming time window. +/// Used by `QueryExpr::TimeWindow`; distinct from SQL analytic frames +/// (`QueryExpr::WindowFunc`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TimeWindowKind { + /// Non-overlapping fixed-size windows. + Tumbling, + /// Overlapping windows advancing by `slide` interval. + Sliding, + /// Windows that open on activity and close after a gap of inactivity. + Session, +} + +// ── Leaf data source ────────────────────────────────────────────────────────── + +/// The leaf data source of a query. Carried by `QueryExpr::Scan` to keep +/// L3 data-model-agnostic: everything above `Scan` (`Filter`, `Aggregate`, +/// etc.) works identically regardless of `Source` variant. +#[derive(Debug, Clone)] +pub enum Source { + /// Time-series input — deployment-model-asapquery / asaplifecycle shape. + TimeSeries { + metric: MetricRef, + time: TimeRange, + labels: LabelFilter, + }, + /// Tabular input — deployment-model-asapfusion / future-OLAP shape. + Table { + table_ref: TableRef, + columns: Vec, + }, + /// Recursive join over sources (multi-table tabular queries). + Join { + left: Box, + right: Box, + on: JoinKey, + }, +} + +impl Source { + pub fn data_model(&self) -> DataModel { + match self { + Source::TimeSeries { .. } => DataModel::TimeSeries, + Source::Table { .. } | Source::Join { .. } => DataModel::Tabular, + } + } +} + +// ── Aggregation intent ──────────────────────────────────────────────────────── + +/// What to compute, not how. Sketch type and parameters are chosen by L4 +/// rules; `AggIntent` is the L3 statement of intent only. +/// +/// Heavy-hitter top-k (`TopK`) is a first-class intent because dedicated +/// sketch primitives (SpaceSaving, CMS-with-heap) compute it in one pass. +/// Generic ordering+limit stays as `QueryExpr::Sort + QueryExpr::Limit`. +#[derive(Debug, Clone)] +pub enum AggIntent { + // ── Data-model-agnostic ─────────────────────────────────────────────────── + Count { accuracy: AccuracyTarget }, + Sum, + Min, + Max, + Quantile { q: f64, accuracy: AccuracyTarget }, + /// Heavy-hitter top-k. Distinct from generic `Sort + Limit` — a + /// dedicated sketch (SpaceSaving, CMS-with-heap) computes it as a single + /// primitive. Recognised by L1→L2→L3 lowering on `ORDER BY count DESC + /// LIMIT k` / PromQL `topk(k, …)`. + TopK { k: usize, by: Vec, accuracy: AccuracyTarget }, + Cardinality { accuracy: AccuracyTarget }, + + // ── Time-series streaming derivatives ──────────────────────────────────── + // Include PromQL counter-reset adjustment; not equivalent to Sum/Count + // over a Window. Kept distinct so delta-set aggregators bind directly. + Rate { window: Duration }, + Increase { window: Duration }, +} + +impl AggIntent { + /// Which data model this intent semantically requires. L4 rules consult + /// this to skip non-applicable intents (e.g. `Rate` over a `Source::Table`). + pub fn requires(&self) -> DataModel { + todo!() + } + + /// Output column type — used by L3 schema derivation for `Aggregate`. + pub fn output_type(&self, _input: &super::schema::L3Field) -> super::schema::L3DataType { + todo!() + } +} + +// ── L3 DAG node ─────────────────────────────────────────────────────────────── + +/// A node in the L3 DAG. Wraps the expression and its derived output schema +/// so that every edge implicitly carries a typed schema: holding an +/// `Rc` gives you both the child expression and the schema of the +/// data flowing on that edge. +#[derive(Debug, Clone)] +pub struct L3Node { + pub expr: QueryExpr, + /// Output schema of `expr` — the schema of the data flowing on the edge + /// leading *from* this node to its parent(s). + pub schema: L3Schema, +} + +// ── L3 intent algebra IR ────────────────────────────────────────────────────── + +/// Language- and deployment-independent intent-only IR. No sketch types, +/// no sketch parameters, no language-specific operators. Traversing from +/// the root node yields a DAG; shared sub-expressions appear as multiple +/// `Rc` references to the same `L3Node`. +#[derive(Debug, Clone)] +pub enum QueryExpr { + // ── Base relations ──────────────────────────────────────────────────────── + /// Outermost leaf. `source` carries the data-model-specific leaf shape. + Scan { source: Source, predicates: Vec }, + /// Reference to a named `LetBinding` sub-expression; resolved at plan time. + Ref(String), + + // ── Filtering & projection ──────────────────────────────────────────────── + /// σ — row-level filter. Output schema = child schema (unchanged). + Filter { child: Rc, pred: Predicate }, + /// π — column projection. Output schema = child schema projected to `cols`. + Project { child: Rc, cols: Vec }, + + // ── Aggregation ─────────────────────────────────────────────────────────── + /// γ + α — GROUP BY + aggregate intents. Concrete operator (HashAgg / + /// SortAgg / SketchAgg) chosen by L4; `aggs` carry intent only. + Aggregate { + child: Rc, + by: Vec, + aggs: Vec, + having: Option, + }, + + // ── Time / streaming windows ────────────────────────────────────────────── + /// ψ — tumbling / sliding / session window over the time axis. Defines + /// the flush / reset lifecycle for aggregates in its sub-DAG. SQL analytic + /// frames are a different node (`WindowFunc`). + TimeWindow { + child: Rc, + kind: TimeWindowKind, + size: Duration, + slide: Option, + }, + + // ── Distributed-execution structure ─────────────────────────────────────── + /// Logical-only partitioning marker. Output schema = child schema. + /// Carries a sharding hint for the L5 stage allocator. + Partition { child: Rc, keys: PartitionKeys }, + /// δ — SQL `DISTINCT` / row deduplication. + Distinct { child: Rc, cols: Vec }, + /// ⊕ — exact union of sub-results from independent stages or shards. + /// Sketch unions are a separate node in `SummaryExpr` because they carry + /// sketch-family / params type constraints. + Merge { children: Vec> }, + + // ── Joins ───────────────────────────────────────────────────────────────── + /// Logical join. L4 picks the physical alternative (HashJoin / + /// SortMergeJoin / SketchJoin) based on selectivity, memory budget, and + /// accuracy target. + Join { + kind: JoinKind, + left: Rc, + right: Rc, + pred: Option, + }, + + // ── Set operators ───────────────────────────────────────────────────────── + SetOp { + kind: SetOpKind, + all: bool, + left: Rc, + right: Rc, + }, + + // ── Ordering & limiting ─────────────────────────────────────────────────── + /// Generic order-by for non-heavy-hitter cases (`ORDER BY name LIMIT 10`). + /// Heavy-hitter shapes lower to `AggIntent::TopK` instead. + Sort { child: Rc, keys: Vec }, + Limit { child: Rc, n: u64, offset: u64 }, + + // ── Subquery / CTE ──────────────────────────────────────────────────────── + Subquery { child: Rc, alias: String }, + /// SQL `WITH name AS (expr) … body`; lowering target for PromQL + /// recording-rule bindings. The `expr` sub-DAG may be referenced N times + /// via `Ref(name)` in `body`, giving the DAG its fan-in. + LetBinding { + name: String, + expr: Rc, + body: Rc, + }, + + // ── Analytic window functions ───────────────────────────────────────────── + /// SQL `OVER (PARTITION BY … ORDER BY … ROWS BETWEEN …)`. + /// Distinct from `TimeWindow` — that is a streaming window over the time + /// axis; this is an analytic frame over already-grouped rows. + WindowFunc { + child: Rc, + func: WindowFuncKind, + partition_by: Vec, + order_by: Vec, + frame: Option, + }, + + // ── Binary composition ──────────────────────────────────────────────────── + /// Arithmetic / comparison / boolean composition (PromQL binary ops + /// including `and` / `or` / `unless`, SQL boolean composition). + BinaryOp { + op: BinaryOpKind, + lhs: Rc, + rhs: Rc, + vector_match: Option, + }, +} + +impl HasSchema for QueryExpr { + fn output_schema(&self, _input_schemas: &[&L3Schema], _catalog: &SchemaCatalog) -> L3Schema { + todo!() + } +} diff --git a/crates/core/src/intent_algebra/mod.rs b/crates/core/src/intent_algebra/mod.rs new file mode 100644 index 00000000..a5117cb3 --- /dev/null +++ b/crates/core/src/intent_algebra/mod.rs @@ -0,0 +1,10 @@ +pub mod expr; +pub mod schema; + +pub use expr::{ + AggIntent, BinaryOpKind, ColumnRef, DataModel, GroupKey, JoinKey, JoinKind, L3Node, + LabelFilter, MetricRef, PartitionKeys, Predicate, ProjectItem, QueryExpr, SetOpKind, + SortKey, Source, TableRef, TimeRange, TimeWindowKind, VectorMatch, WindowFrame, + WindowFuncKind, +}; +pub use schema::{HasSchema, L3DataType, L3Field, L3Schema, SchemaCatalog}; diff --git a/crates/core/src/intent_algebra/schema.rs b/crates/core/src/intent_algebra/schema.rs new file mode 100644 index 00000000..4f6d1fc0 --- /dev/null +++ b/crates/core/src/intent_algebra/schema.rs @@ -0,0 +1,53 @@ +/// Opaque handle to the external data-source catalog (Prometheus metric +/// metadata, SQL `information_schema`, DataFusion catalog). Used only by +/// `Scan` schema derivation to resolve leaf column types; all other nodes +/// derive their output schemas purely from their input schemas. +pub struct SchemaCatalog; + +// ── Data types ──────────────────────────────────────────────────────────────── + +/// Column types that may appear on an L3 DAG edge. +/// L4 extends this set with `L4DataType::Sketch`; L3 edges never carry +/// sketch-state columns. +#[derive(Debug, Clone, PartialEq)] +pub enum L3DataType { + Int64, + Float64, + Utf8, + Boolean, + Timestamp, + Duration, + /// Key→Value map (e.g. PromQL label set encoded as a column). + Map(Box, Box), + List(Box), +} + +// ── Schema ──────────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub struct L3Field { + pub name: String, + pub dtype: L3DataType, + pub nullable: bool, +} + +/// Schema carried on every edge of the L3 DAG. Describes the columns +/// flowing between two operators. Type-checked at plan construction time: +/// a node whose predicate references a column absent from its child's +/// `L3Schema` is a plan-time error. +#[derive(Debug, Clone)] +pub struct L3Schema { + pub fields: Vec, + /// Index into `fields` for the time axis, if any. + /// PromQL `Scan` leaves always carry one; SQL leaves may or may not. + pub time_index: Option, +} + +// ── Schema derivation trait ─────────────────────────────────────────────────── + +/// Implemented by `QueryExpr` to compute the output schema of a node given +/// its children's output schemas. The `L3Node` wrapper stores the derived +/// schema so derivation runs once at construction, not on every traversal. +pub trait HasSchema { + fn output_schema(&self, input_schemas: &[&L3Schema], catalog: &SchemaCatalog) -> L3Schema; +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs new file mode 100644 index 00000000..5d62afa9 --- /dev/null +++ b/crates/core/src/lib.rs @@ -0,0 +1,4 @@ +pub mod intent_algebra; +pub mod sketch_algebra; +pub mod types; +pub mod workload; diff --git a/crates/core/src/sketch_algebra/expr.rs b/crates/core/src/sketch_algebra/expr.rs new file mode 100644 index 00000000..09772a8e --- /dev/null +++ b/crates/core/src/sketch_algebra/expr.rs @@ -0,0 +1,96 @@ +use std::rc::Rc; + +use crate::intent_algebra::{ColumnRef, GroupKey, L3Node}; +use super::schema::L4Schema; +use super::sketch::{SummaryKind, SummaryParams, SketchQuery}; + +// ── L4 DAG node ─────────────────────────────────────────────────────────────── + +/// A node in the L4 DAG. Mirrors `L3Node`: wraps the expression and its +/// derived output schema so every edge carries a typed schema. +/// `L4Schema` may contain `L4DataType::Sketch` columns; `L3Schema` cannot. +#[derive(Debug, Clone)] +pub struct L4Node { + pub expr: SummaryExpr, + /// Output schema of `expr` — the schema of the data flowing on the edge + /// leading *from* this node to its parent(s). + pub schema: L4Schema, +} + +// ── L4 sketch-bound IR ──────────────────────────────────────────────────────── + +/// Sketch-bound IR produced by L4 optimizer rules. L4 rules selectively +/// replace logical aggregates and joins in the L3 `QueryExpr` with their +/// sketch-bound counterparts; everything not rewritten passes through as +/// `Logical(Rc)`. +/// +/// Traversing from the root node yields a DAG; shared sub-expressions appear +/// as multiple `Rc` references to the same `L4Node` or `L3Node`. +#[derive(Debug, Clone)] +pub enum SummaryExpr { + /// Any L3 node that no L4 rule rewrote (e.g. `Filter`, `Project`, `Sort`). + /// Output schema is the inner L3 node's schema, lifted to `L4Schema` + /// with all fields as `L4DataType::Primitive`. + Logical(Rc), + + /// Sketch aggregation. L4 chose `sketch` + `params` from the catalog + /// for `AggIntent` under `DeploymentConstraints`. + /// Output schema: `by` columns (verbatim) + one `Sketch(sketch, params)` + /// field carrying partial sketch state per group. + SummaryAgg { + child: Rc, + sketch: SummaryKind, + params: SummaryParams, + /// The column being summarised (fed into the sketch). + col: ColumnRef, + /// GROUP BY keys carried through to the output schema. + by: Vec, + }, + + /// Sketch-aware join (KMV / theta for join-cardinality; join-sample for + /// sampling). Emitted only when a `Bind*OnJoin` rule fires. + /// Output schema: one `Sketch(sketch, params)` field read by a downstream + /// `SummaryEstimate`. + SummaryJoin { + outer: Rc, + inner: Rc, + key: ColumnRef, + sketch: SummaryKind, + params: SummaryParams, + }, + + /// Subtract one sketch from another. Valid only for sketches with a + /// linear-inverse property (CMS, theta, count-based). Catalog flag + /// `subtractable` must be true for the sketch family. + /// Output schema: one `Sketch(s, p)` field (same family + params as inputs). + SummarySubtract { + left: Rc, + right: Rc, + }, + + /// Delete a key from a sketch (CMS update with −1, deletable Bloom + /// filter). Catalog flag `deletable` must be true. Output schema = + /// input schema unchanged in type (same `Sketch(s, p)` field). + SummaryDelete { + sketch_input: Rc, + key: ColumnRef, + }, + + /// Read out a query result from a built sketch. The `Sketch(…)` field + /// type does *not* propagate downstream of an estimate — the output + /// schema is a regular row-shaped schema (Float64 for quantile, Int64 + /// for count/cardinality, `[(key, count)]` for top-k). + SummaryEstimate { + sketch_input: Rc, + query: SketchQuery, + }, + + /// ⊕ — union of sketches across stages / shards. Distinct from L3 + /// `Merge` because sketch union has type constraints: all inputs must + /// agree on `(sketch, params)` and the catalog flag `mergeable` must + /// be true. Inserted by the L5 stage allocator on cut edges. + /// Output schema: one `Sketch(s, p)` field (same family + params as inputs). + SummaryMerge { + children: Vec>, + }, +} diff --git a/crates/core/src/sketch_algebra/mod.rs b/crates/core/src/sketch_algebra/mod.rs new file mode 100644 index 00000000..a2f589e4 --- /dev/null +++ b/crates/core/src/sketch_algebra/mod.rs @@ -0,0 +1,7 @@ +pub mod expr; +pub mod schema; +pub mod sketch; + +pub use expr::{L4Node, SummaryExpr}; +pub use schema::{L4DataType, L4Field, L4Schema}; +pub use sketch::{SummaryKind, SummaryParams, SketchQuery}; diff --git a/crates/core/src/sketch_algebra/schema.rs b/crates/core/src/sketch_algebra/schema.rs new file mode 100644 index 00000000..b6d5d14e --- /dev/null +++ b/crates/core/src/sketch_algebra/schema.rs @@ -0,0 +1,42 @@ +use crate::intent_algebra::L3DataType; +use super::sketch::{SummaryKind, SummaryParams}; + +// ── L4 data types ───────────────────────────────────────────────────────────── + +/// Column types that may appear on an L4 DAG edge. A strict superset of +/// `L3DataType`: L4 adds `Sketch` for edges that carry partial sketch state +/// between a `SummaryAgg` and a downstream `SummaryEstimate` or `SummaryMerge`. +/// +/// The `Sketch` variant carries `(kind, params)` so the type system can reject +/// merges of incompatible sketches at plan construction time — a `SummaryMerge` +/// over `Sketch(Kll, …)` and `Sketch(Cms, …)` inputs is a plan-time error. +#[derive(Debug, Clone, PartialEq)] +pub enum L4DataType { + /// Any base L3 column type — passed through unchanged from L3 edges. + Primitive(L3DataType), + /// Opaque summary state (exact accumulator or approximate sketch). + /// The `(kind, params)` pair is the type identity: two summary columns + /// are compatible only if both match exactly. + Sketch(SummaryKind, SummaryParams), +} + +// ── L4 schema ───────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub struct L4Field { + pub name: String, + pub dtype: L4DataType, + pub nullable: bool, +} + +/// Schema carried on every edge of the L4 DAG. Extends `L3Schema` with the +/// ability to express sketch-state columns. L3 and L4 schemas are separate +/// types so L3 nodes structurally cannot carry `Sketch`-typed columns — any +/// attempt to do so is a compile-time type error. +#[derive(Debug, Clone)] +pub struct L4Schema { + pub fields: Vec, + /// Index into `fields` for the time axis, if any (same semantics as + /// `L3Schema::time_index`). + pub time_index: Option, +} diff --git a/crates/core/src/sketch_algebra/sketch.rs b/crates/core/src/sketch_algebra/sketch.rs new file mode 100644 index 00000000..36aea623 --- /dev/null +++ b/crates/core/src/sketch_algebra/sketch.rs @@ -0,0 +1,75 @@ +use crate::intent_algebra::ColumnRef; + +// ── Summary kind identifiers ─────────────────────────────────────────────────── + +/// Identifies a precomputed aggregation family — either an exact accumulator +/// or an approximate sketch. Used as a type tag in `L4DataType` and as the +/// binding choice recorded in `SummaryExpr` nodes. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SummaryKind { + // ── Exact accumulators (no approximation error) ────────────────────────── + /// Exact sum accumulator (mergeable by addition). + Sum, + /// Exact count accumulator (mergeable by addition). + Count, + /// Exact min/max accumulator (mergeable by comparison). + MinMax, + /// Exact increase accumulator (counter-reset-aware delta). + Increase, + /// Rate accumulator (increase / time window duration). + Rate, + // ── Approximate sketches ───────────────────────────────────────────────── + /// KLL quantile sketch (mergeable, ε-accurate rank queries). + Kll, + /// Count-Min Sketch (mergeable, (ε,δ)-accurate frequency queries). + Cms, + /// HyperLogLog (mergeable, (ε,δ)-accurate cardinality). + Hll, + /// DDSketch (mergeable, relative-error quantile queries). + DDSketch, + /// CMS augmented with a min-heap for top-k / heavy-hitter queries. + CmsWithHeap, + /// K-Minimum Values sketch (mergeable, join-cardinality estimation). + Kmv, + /// Theta sketch (mergeable, set operations + cardinality). + Theta, +} + +// ── Sketch parameters ───────────────────────────────────────────────────────── + +/// Concrete, catalog-validated parameters for a specific summary instance. +/// The variant must correspond to the associated `SummaryKind`; mismatches +/// are caught at L4 bind time before L5 ever sees the plan. +/// Exact-accumulator variants carry no parameters (their semantics are fixed). +#[derive(Debug, Clone, PartialEq)] +pub enum SummaryParams { + // Exact accumulators — no tuning parameters + Sum, + Count, + MinMax, + Increase, + Rate, + // Approximate sketches — algorithm-specific tuning parameters + Kll { k: u32 }, + Cms { width: u32, depth: u32 }, + Hll { precision: u8 }, + DDSketch { alpha: f64 }, + CmsWithHeap { width: u32, depth: u32, heap_size: u32 }, + Kmv { k: u32 }, + Theta { k: u32 }, +} + +// ── Sketch read-out queries ─────────────────────────────────────────────────── + +/// What to extract from a built sketch. Carried by `SummaryEstimate`. +#[derive(Debug, Clone)] +pub enum SketchQuery { + /// Extract the value at quantile rank `q` ∈ (0, 1]. + Quantile { q: f64 }, + /// Estimated count / frequency for a specific key. + PointCount { key: ColumnRef }, + /// Estimated number of distinct elements. + Cardinality, + /// Top-k most frequent (key, count) pairs. + TopK { k: usize }, +} diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs new file mode 100644 index 00000000..4dfad52c --- /dev/null +++ b/crates/core/src/types.rs @@ -0,0 +1,10 @@ +/// Accuracy requirement that a query result must satisfy. +#[derive(Debug, Clone, PartialEq)] +pub enum AccuracyTarget { + /// Additive error bound ε: |estimate − true| ≤ ε · (domain size). + Epsilon(f64), + /// Probabilistic (ε, δ) guarantee: error ≤ ε with probability ≥ 1 − δ. + EpsilonDelta { epsilon: f64, delta: f64 }, + /// No approximation permitted; result must be exact. + Exact, +} diff --git a/crates/core/src/workload.rs b/crates/core/src/workload.rs new file mode 100644 index 00000000..ee63b07a --- /dev/null +++ b/crates/core/src/workload.rs @@ -0,0 +1,119 @@ +use crate::types::AccuracyTarget; + +// ── Query surface ───────────────────────────────────────────────────────────── + +/// A raw query string in its source language, before any parsing. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Query(pub String); + +/// How often a repeating query fires, in milliseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RepetitionInterval(pub u32); + +/// SQL dialect variant — different dialects have different syntax and +/// function sets that affect how the query string is parsed at L1. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum SqlDialect { + DataFusionSQL, + ClickhouseSQL, + ElasticSQL, +} + +/// Source language of every query in the workload. +/// All queries in a single `QueryWorkload` share the same language. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum QueryLanguage { + PromQL, + SQL(SqlDialect), + DataFusion, + ElasticDSL, +} + +// ── Per-query requirements ──────────────────────────────────────────────────── + +/// SLA constraints attached to a single query. +/// Both fields are optional: an absent bound means "no constraint on this axis." +#[derive(Debug, Clone)] +pub struct QueryRequirements { + /// Maximum acceptable approximation error. + pub accuracy: Option, + /// Maximum acceptable end-to-end query latency in milliseconds. + pub latency_ms: Option, +} + +// ── Workload entries ────────────────────────────────────────────────────────── + +/// One entry in a one-shot batch: a query plus its optional SLA constraints. +#[derive(Debug, Clone)] +pub struct BatchEntry { + pub query: Query, + pub requirements: Option, +} + +/// One entry in a repeating (streaming) workload: a query that fires +/// every `interval` milliseconds. +#[derive(Debug, Clone)] +pub struct RepeatingEntry { + pub query: Query, + /// How often the query fires, in milliseconds. + pub interval: RepetitionInterval, + pub requirements: Option, +} + +// ── Data characteristics ────────────────────────────────────────────────────── + +/// Statistical distribution of keys in the incoming data stream. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum DataDistribution { + /// Zipf-distributed keys (s ≈ 1.1). A small number of keys dominate, + /// so only a fraction of sketch cells are touched per window. Typical + /// production case. + #[default] + Zipf, + /// All keys are equally probable. Every window fills the sketch more + /// uniformly; delta-compression benefit is lower. + Uniform, + /// Traffic arrives in bursts with a concentrated key set. Effective + /// fill rate is lower on average but spikes can reach Uniform levels. + Bursty, +} + +/// Characteristics of the data arriving at the ingestion layer. +/// Used by the cost model and sketch-parameter binder to size sketches +/// and estimate transmission cost without running the query. +#[derive(Debug, Clone)] +pub struct DataCharacteristics { + /// Number of distinct active time series for this metric. + pub series_count: u64, + /// Sample rate per series at the SDK / agent (Hz). + pub samples_per_sec_per_series: f64, + /// Wire size of one raw OTLP metric data point after protobuf encoding + /// (bytes). Typical range: 50–200 bytes. + pub bytes_per_raw_sample: u32, + /// Known distinct key values per flush period for frequency / cardinality + /// sketches. `None` → inferred analytically from inserts and distribution. + pub distinct_keys_per_window: Option, + /// Statistical distribution of keys in the stream. + pub data_distribution: DataDistribution, +} + +// ── Top-level workload ──────────────────────────────────────────────────────── + +/// The single normalised input type accepted by every entry point into the +/// controller (HTTP POST /plan, YAML file, query-log replay, OpAMP callback). +/// +/// `query_batch` and `repeating_queries` are mutually exclusive today; both +/// may be present in the future when mixed batch+streaming workloads are +/// supported. +#[derive(Debug, Clone)] +pub struct QueryWorkload { + /// Source language shared by all queries in this workload. + pub language: QueryLanguage, + /// One-shot queries executed together as a batch. + pub query_batch: Option>, + /// Queries that repeat on a fixed interval (streaming / continuous). + pub repeating_queries: Option>, + /// Workload-level data characteristics used for sketch sizing and cost + /// estimation. Applies to all queries in this workload. + pub data_characteristics: Option, +}