diff --git a/Cargo.lock b/Cargo.lock index 91c4a2bd..37d4f72d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,6 +376,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "asap_otel_proto" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "protoc-bin-vendored", + "tonic", + "tonic-build", +] + [[package]] name = "asap_planner" version = "0.1.0" @@ -3124,54 +3135,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "opentelemetry" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror 2.0.18", - "tracing", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" -dependencies = [ - "base64 0.22.1", - "hex", - "opentelemetry", - "opentelemetry_sdk", - "prost", - "serde", - "tonic", - "tracing", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry", - "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 2.0.18", -] - [[package]] name = "ordered-float" version = "2.10.1" @@ -3607,6 +3570,70 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "psm" version = "0.1.30" @@ -3623,6 +3650,7 @@ version = "0.1.0" dependencies = [ "anyhow", "arrow", + "asap_otel_proto", "asap_planner", "asap_sketchlib", "asap_types", @@ -3644,7 +3672,6 @@ dependencies = [ "futures", "hex", "lazy_static", - "opentelemetry-proto", "prometheus", "promql-parser", "promql_utilities", @@ -4768,6 +4795,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.117", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/Cargo.toml b/Cargo.toml index e656e44a..a6cf64bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "asap-common/dependencies/rs/promql_utilities", "asap-common/dependencies/rs/sql_utilities", "asap-common/dependencies/rs/elastic_dsl_utilities", + "asap-common/dependencies/rs/asap_otel_proto", "asap-common/dependencies/rs/asap_types", "asap-common/dependencies/rs/datafusion_summary_library", "asap-common/tests/compare_matched_tokens/rust_tests", diff --git a/asap-common/dependencies/rs/asap_otel_proto/Cargo.toml b/asap-common/dependencies/rs/asap_otel_proto/Cargo.toml new file mode 100644 index 00000000..563d742f --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "asap_otel_proto" +version.workspace = true +edition.workspace = true + +# Vendored Rust bindings for DataCollector's modified opentelemetry-proto. +# See ../../../../docs/pipeline-query-catalog.md §5.4 for the reasoning. +# +# The .proto files in proto/opentelemetry/proto/ are copied verbatim from +# DataCollector's opentelemetry-proto submodule and add first-class sketch +# variants (DDSketch, KLLSketch, CountSketch, CountMinSketch, HLLSketch) to +# the standard OTLP Metric.data oneof. To re-vendor when DataCollector +# updates its proto schema, copy the four files referenced in build.rs +# from the corresponding paths in the DataCollector worktree. + +[lib] +path = "src/lib.rs" + +[dependencies] +prost = "0.13" +tonic = { version = "0.12", features = ["transport", "codegen", "prost"] } + +[build-dependencies] +prost-build = "0.13" +tonic-build = { version = "0.12", features = ["prost"] } +# Ship a modern protoc so the build does not depend on the host's system +# protoc version. The system protoc on many distros is < 3.15 which +# rejects proto3 `optional` keyword used in opentelemetry-proto. +protoc-bin-vendored = "3" diff --git a/asap-common/dependencies/rs/asap_otel_proto/build.rs b/asap-common/dependencies/rs/asap_otel_proto/build.rs new file mode 100644 index 00000000..674149d8 --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/build.rs @@ -0,0 +1,24 @@ +fn main() -> Result<(), Box> { + let proto_root = "proto"; + let proto_files = [ + "proto/opentelemetry/proto/common/v1/common.proto", + "proto/opentelemetry/proto/resource/v1/resource.proto", + "proto/opentelemetry/proto/metrics/v1/metrics.proto", + "proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto", + ]; + for f in &proto_files { + println!("cargo:rerun-if-changed={f}"); + } + println!("cargo:rerun-if-changed=build.rs"); + + // Use the vendored protoc binary so the build is independent of the + // host's system protoc version. Many distros still ship protoc < 3.15 + // which rejects proto3 `optional` keyword used in opentelemetry-proto. + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path()?); + + tonic_build::configure() + .build_server(true) + .build_client(false) + .compile_protos(&proto_files, &[proto_root])?; + Ok(()) +} diff --git a/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto new file mode 100644 index 00000000..a2e98e02 --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/collector/metrics/v1/metrics_service.proto @@ -0,0 +1,115 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.metrics.v1; + +import "opentelemetry/proto/metrics/v1/metrics.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Metrics.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.metrics.v1"; +option java_outer_classname = "MetricsServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/metrics/v1"; + +// Service that can be used to push metrics between one Application +// instrumented with OpenTelemetry and a collector, or between a collector and a +// central collector. +service MetricsService { + rpc Export(ExportMetricsServiceRequest) returns (ExportMetricsServiceResponse) {} +} + +message ExportMetricsServiceRequest { + // An array of ResourceMetrics. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.metrics.v1.ResourceMetrics resource_metrics = 1; +} + +message ExportMetricsServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportMetricsPartialSuccess partial_success = 1; + + // List of compact descriptors for series IDs that were minted by the + // collector while processing this request. Exporters SHOULD cache these + // assignments and omit the associated attributes on subsequent exports once + // a series_id is known. + repeated SeriesAssignment series_assignments = 2; +} + +message ExportMetricsPartialSuccess { + // The number of rejected data points. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_data_points = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; +} + +// SeriesAssignment communicates collector-assigned series identifiers back to +// exporters. Identifiers are expressed using the same deterministic key format +// as the collector's attribute de-duplication logic: resource_key and scope_key +// are built by concatenating the schema URL with sorted attribute pairs, +// metric_name is the OTLP Metric name, metric_type is the string form of +// opentelemetry.proto.metrics.v1.Metric.MetricType, and attributes_fingerprint +// is an opaque fingerprint of the sorted point-level attributes. +message SeriesAssignment { + // Deterministic encoding of the Resource associated with the series. The + // collector MUST build this using schema_url + sorted attributes separated + // by '|' to match the lookup key it uses internally. + string resource_key = 1; + + // Deterministic encoding of the instrumentation scope. Built from + // scope.name + scope.version + scope.schema_url + sorted attributes separated + // by '|'. + string scope_key = 2; + + // Fully-qualified metric identity. + string metric_name = 3; + + // String representation of the metric type (e.g. "Gauge", "Histogram"). + string metric_type = 4; + + // Opaque fingerprint of the data point's attribute set. The exporter MUST + // use the same fingerprinting algorithm when looking up cached series_ids. + bytes attributes_fingerprint = 5; + + // The collector-assigned series identifier. + uint64 series_id = 6; +} diff --git a/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto new file mode 100644 index 00000000..7f9ffab3 --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/common/v1/common.proto @@ -0,0 +1,129 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.common.v1; + +option csharp_namespace = "OpenTelemetry.Proto.Common.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.common.v1"; +option java_outer_classname = "CommonProto"; +option go_package = "go.opentelemetry.io/proto/otlp/common/v1"; + +// Represents any type of attribute value. AnyValue may contain a +// primitive value such as a string or integer or it may contain an arbitrary nested +// object containing arrays, key-value lists and primitives. +message AnyValue { + // The value is one of the listed fields. It is valid for all values to be unspecified + // in which case this AnyValue is considered to be "empty". + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValue array_value = 5; + KeyValueList kvlist_value = 6; + bytes bytes_value = 7; + } +} + +// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message +// since oneof in AnyValue does not allow repeated fields. +message ArrayValue { + // Array of values. The array may be empty (contain 0 elements). + repeated AnyValue values = 1; +} + +// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message +// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need +// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to +// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +// are semantically equivalent. +message KeyValueList { + // A collection of key/value pairs of key-value pairs. The list may be empty (may + // contain 0 elements). + // + // The keys MUST be unique (it is not allowed to have more than one + // value with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated KeyValue values = 1; +} + +// Represents a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValue { + // The key name of the pair. + string key = 1; + + // The value of the pair. + AnyValue value = 2; +} + +// InstrumentationScope is a message representing the instrumentation scope information +// such as the fully qualified name and version. +message InstrumentationScope { + // A name denoting the Instrumentation scope. + // An empty instrumentation scope name means the name is unknown. + string name = 1; + + // Defines the version of the instrumentation scope. + // An empty instrumentation scope version means the version is unknown. + string version = 2; + + // Additional attributes that describe the scope. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated KeyValue attributes = 3; + + // The number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 4; +} + +// A reference to an Entity. +// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs. +// +// Status: [Development] +message EntityRef { + // The Schema URL, if known. This is the identifier of the Schema that the entity data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // + // This schema_url applies to the data in this message and to the Resource attributes + // referenced by id_keys and description_keys. + // TODO: discuss if we are happy with this somewhat complicated definition of what + // the schema_url applies to. + // + // This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs. + string schema_url = 1; + + // Defines the type of the entity. MUST not change during the lifetime of the entity. + // For example: "service" or "host". This field is required and MUST not be empty + // for valid entities. + string type = 2; + + // Attribute Keys that identify the entity. + // MUST not change during the lifetime of the entity. The Id must contain at least one attribute. + // These keys MUST exist in the containing {message}.attributes. + repeated string id_keys = 3; + + // Descriptive (non-identifying) attribute keys of the entity. + // MAY change over the lifetime of the entity. MAY be empty. + // These attribute keys are not part of entity's identity. + // These keys MUST exist in the containing {message}.attributes. + repeated string description_keys = 4; +} \ No newline at end of file diff --git a/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto new file mode 100644 index 00000000..3a909e47 --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/metrics/v1/metrics.proto @@ -0,0 +1,984 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.metrics.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Metrics.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.metrics.v1"; +option java_outer_classname = "MetricsProto"; +option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; + +// MetricsData represents the metrics data that can be stored in a persistent +// storage, OR can be embedded by other protocols that transfer OTLP metrics +// data but do not implement the OTLP protocol. +// +// MetricsData +// └─── ResourceMetrics +// ├── Resource +// ├── SchemaURL +// └── ScopeMetrics +// ├── Scope +// ├── SchemaURL +// └── Metric +// ├── Name +// ├── Description +// ├── Unit +// └── data +// ├── Gauge +// ├── Sum +// ├── Histogram +// ├── ExponentialHistogram +// ├── DDSketch +// └── Summary +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message MetricsData { + // An array of ResourceMetrics. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceMetrics resource_metrics = 1; +} + +// A collection of ScopeMetrics from a Resource. +message ResourceMetrics { + reserved 1000; + + // The resource for the metrics in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of metrics that originate from a resource. + repeated ScopeMetrics scope_metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_metrics" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Metrics produced by an Scope. +message ScopeMetrics { + // The instrumentation scope information for the metrics in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of metrics that originate from an instrumentation library. + repeated Metric metrics = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the metric data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "scope" field and all metrics in the + // "metrics" field. + string schema_url = 3; +} + +// Defines a Metric which has one or more timeseries. The following is a +// brief summary of the Metric data model. For more details, see: +// +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md +// +// The data model and relation between entities is shown in the +// diagram below. Here, "DataPoint" is the term used to refer to any +// one of the specific data point value types, and "points" is the term used +// to refer to any one of the lists of points contained in the Metric. +// +// - Metric is composed of a metadata and data. +// - Metadata part contains a name, description, unit. +// - Data is one of the possible types (Sum, Gauge, Histogram, Summary). +// - DataPoint contains timestamps, attributes, and one of the possible value type +// fields. +// +// Metric +// +------------+ +// |name | +// |description | +// |unit | +------------------------------------+ +// |data |---> |Gauge, Sum, Histogram, Summary, ... | +// +------------+ +------------------------------------+ +// +// Data [One of Gauge, Sum, Histogram, Summary, ...] +// +-----------+ +// |... | // Metadata about the Data. +// |points |--+ +// +-----------+ | +// | +---------------------------+ +// | |DataPoint 1 | +// v |+------+------+ +------+ | +// +-----+ ||label |label |...|label | | +// | 1 |-->||value1|value2|...|valueN| | +// +-----+ |+------+------+ +------+ | +// | . | |+-----+ | +// | . | ||value| | +// | . | |+-----+ | +// | . | +---------------------------+ +// | . | . +// | . | . +// | . | . +// | . | +---------------------------+ +// | . | |DataPoint M | +// +-----+ |+------+------+ +------+ | +// | M |-->||label |label |...|label | | +// +-----+ ||value1|value2|...|valueN| | +// |+------+------+ +------+ | +// |+-----+ | +// ||value| | +// |+-----+ | +// +---------------------------+ +// +// Each distinct type of DataPoint represents the output of a specific +// aggregation function, the result of applying the DataPoint's +// associated function of to one or more measurements. +// +// All DataPoint types have three common fields: +// - Attributes includes key-value pairs associated with the data point +// - TimeUnixNano is required, set to the end time of the aggregation +// - StartTimeUnixNano is optional, but strongly encouraged for DataPoints +// having an AggregationTemporality field, as discussed below. +// +// Both TimeUnixNano and StartTimeUnixNano values are expressed as +// UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. +// +// # TimeUnixNano +// +// This field is required, having consistent interpretation across +// DataPoint types. TimeUnixNano is the moment corresponding to when +// the data point's aggregate value was captured. +// +// Data points with the 0 value for TimeUnixNano SHOULD be rejected +// by consumers. +// +// # StartTimeUnixNano +// +// StartTimeUnixNano in general allows detecting when a sequence of +// observations is unbroken. This field indicates to consumers the +// start time for points with cumulative and delta +// AggregationTemporality, and it should be included whenever possible +// to support correct rate calculation. Although it may be omitted +// when the start time is truly unknown, setting StartTimeUnixNano is +// strongly encouraged. +message Metric { + reserved 4, 6, 8; + + // The name of the metric. + string name = 1; + + // A description of the metric, which can be used in documentation. + string description = 2; + + // The unit in which the metric value is reported. Follows the format + // described by https://unitsofmeasure.org/ucum.html. + string unit = 3; + + // Data determines the aggregation type (if any) of the metric, what is the + // reported value type for the data points, as well as the relatationship to + // the time interval over which they are reported. + oneof data { + Gauge gauge = 5; + Sum sum = 7; + Histogram histogram = 9; + ExponentialHistogram exponential_histogram = 10; + DDSketch ddsketch = 13; + Summary summary = 11; + KLLSketch kllsketch = 14; + CountSketch countsketch = 15; + CountMinSketch countminsketch = 16; + HLLSketch hllsketch = 17; + } + + // Additional metadata attributes that describe the metric. [Optional]. + // Attributes are non-identifying. + // Consumers SHOULD NOT need to be aware of these attributes. + // These attributes MAY be used to encode information allowing + // for lossless roundtrip translation to / from another data model. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue metadata = 12; +} + +// Gauge represents the type of a scalar metric that always exports the +// "current value" for every data point. It should be used for an "unknown" +// aggregation. +// +// A Gauge does not support different aggregation temporalities. Given the +// aggregation is unknown, points cannot be combined using the same +// aggregation, regardless of aggregation temporalities. Therefore, +// AggregationTemporality is not included. Consequently, this also means +// "StartTimeUnixNano" is ignored for all data points. +message Gauge { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated NumberDataPoint data_points = 1; +} + +// Sum represents the type of a scalar metric that is calculated as a sum of all +// reported measurements over a time interval. +message Sum { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated NumberDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; + + // Represents whether the sum is monotonic. + bool is_monotonic = 3; +} + +// Histogram represents the type of a metric that is calculated by aggregating +// as a Histogram of all reported measurements over a time interval. +message Histogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated HistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// ExponentialHistogram represents the type of a metric that is calculated by aggregating +// as a ExponentialHistogram of all reported double measurements over a time interval. +message ExponentialHistogram { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated ExponentialHistogramDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// DDSketch represents the type of a metric that encodes measurements using DDSketch buckets. +message DDSketch { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated DDSketchDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// Summary metric data are used to convey quantile summaries, +// a Prometheus (see: https://prometheus.io/docs/concepts/metric_types/#summary) +// and OpenMetrics (see: https://github.com/prometheus/OpenMetrics/blob/4dbf6075567ab43296eed941037c12951faafb92/protos/prometheus.proto#L45) +// data type. These data points cannot always be merged in a meaningful way. +// While they can be useful in some applications, histogram data points are +// recommended for new applications. +// Summary metrics do not have an aggregation temporality field. This is +// because the count and sum fields of a SummaryDataPoint are assumed to be +// cumulative values. +message Summary { + // The time series data points. + // Note: Multiple time series may be included (same timestamp, different attributes). + repeated SummaryDataPoint data_points = 1; +} + +// DDSketchDataPoint is a single data point that encodes a distribution using the DDSketch format. +message DDSketchDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list MUST be omitted (or present but empty) + // whenever `series_id` is non-zero, and MUST be populated when `series_id` + // is zero so the receiver can mint a new entry. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // Count is the number of measurements that were aggregated into this sketch. + fixed64 count = 4; + + // Sum is the total of all measurements aggregated for this sketch. + // At most one of the fields in this oneof may be set at a time. + oneof sum { + double sum_as_double = 5; + sfixed64 sum_as_int = 12; + } + + // Min is the minimum value over (start_time, end_time]. Optional. + // At most one of the fields in this oneof may be set at a time. + oneof min { + double min_as_double = 6; + sfixed64 min_as_int = 13; + } + + // Max is the maximum value over (start_time, end_time]. Optional. + // At most one of the fields in this oneof may be set at a time. + oneof max { + double max_as_double = 7; + sfixed64 max_as_int = 14; + } + + // Serialized DDSketch payload. The encoding is described by the encoding field. + bytes sketch = 8; + + // Encoding that describes how the sketch bytes should be decoded. + DDSketchEncoding encoding = 10; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 11; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 15; + + // series_id references a collector-assigned series descriptor. Receivers MUST + // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) + // is true. + uint64 series_id = 16; +} + +// DDSketchEncoding identifies how the DDSketch payload bytes are encoded. +enum DDSketchEncoding { + DDSKETCH_ENCODING_UNSPECIFIED = 0; + + // DDSKETCH_ENCODING_PROTO indicates the sketch bytes contain the serialized + // github.com/DataDog/sketches-go/ddsketch/pb/sketchpb.DDSketch message. + DDSKETCH_ENCODING_PROTO = 1; + DDSKETCH_ENCODING_PROTO_DELTA = 2; +} + +// KLLSketch represents the type of a metric that encodes measurements using the KLL quantile sketch. +message KLLSketch { + // The time series data points. + repeated KLLSketchDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// KLLSketchDataPoint is a single data point that encodes a distribution using the KLL sketch format. +message KLLSketchDataPoint { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + fixed64 count = 4; + double sum = 5; + double min = 6; + double max = 7; + bytes sketch = 8; + KLLSketchEncoding encoding = 9; + uint32 flags = 10; + uint64 series_id = 11; +} + +// KLLSketchEncoding identifies how the KLL sketch payload bytes are encoded. +enum KLLSketchEncoding { + KLL_SKETCH_ENCODING_UNSPECIFIED = 0; + KLL_SKETCH_ENCODING_PROTO = 1; +} + +// CountSketch represents the type of a metric that encodes frequency estimations using CountSketch. +message CountSketch { + // The time series data points. + repeated CountSketchDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// CountSketchDataPoint is a single data point encoding frequency estimations. +message CountSketchDataPoint { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + bytes sketch = 4; + CountSketchEncoding encoding = 5; + string dimension = 6; + double epsilon = 7; + double delta = 8; + uint32 flags = 9; + uint64 series_id = 10; +} + +// CountSketchEncoding identifies how the CountSketch payload bytes are encoded. +enum CountSketchEncoding { + COUNT_SKETCH_ENCODING_UNSPECIFIED = 0; + COUNT_SKETCH_ENCODING_PROTO = 1; + COUNT_SKETCH_ENCODING_DELTA = 2; +} + +// CountMinSketch represents the type of a metric encoding frequency estimations using CountMinSketch. +message CountMinSketch { + // The time series data points. + repeated CountMinSketchDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// CountMinSketchDataPoint is a single data point encoding frequency estimations. +message CountMinSketchDataPoint { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + uint64 sample_count = 4; + bytes sketch = 5; + CountMinSketchEncoding encoding = 6; + int32 rows = 7; + int32 cols = 8; + uint32 flags = 9; + uint64 series_id = 10; +} + +// CountMinSketchEncoding identifies how the CountMinSketch payload bytes are encoded. +enum CountMinSketchEncoding { + COUNT_MIN_SKETCH_ENCODING_UNSPECIFIED = 0; + COUNT_MIN_SKETCH_ENCODING_PROTO = 1; + COUNT_MIN_SKETCH_ENCODING_DELTA = 2; +} + +// HLLSketch represents the type of a metric that encodes cardinality estimations using HyperLogLog. +message HLLSketch { + // The time series data points. + repeated HLLSketchDataPoint data_points = 1; + + // aggregation_temporality describes if the aggregator reports delta changes + // since last report time, or cumulative changes since a fixed start time. + AggregationTemporality aggregation_temporality = 2; +} + +// HLLSketchDataPoint is a single data point encoding cardinality estimations. +message HLLSketchDataPoint { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + uint64 count = 4; + uint64 cardinality = 5; + bytes sketch = 6; + HLLSketchEncoding encoding = 7; + uint32 precision = 8; + uint32 flags = 9; + uint64 series_id = 10; +} + +// HLLSketchEncoding identifies how the HLL sketch payload bytes are encoded. +enum HLLSketchEncoding { + HLL_SKETCH_ENCODING_UNSPECIFIED = 0; + HLL_SKETCH_ENCODING_PROTO = 1; + HLL_SKETCH_ENCODING_DELTA = 2; +} + +// AggregationTemporality defines how a metric aggregator reports aggregated +// values. It describes how those values relate to the time interval over +// which they are aggregated. +enum AggregationTemporality { + // UNSPECIFIED is the default AggregationTemporality, it MUST not be used. + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + // DELTA is an AggregationTemporality for a metric aggregator which reports + // changes since last report time. Successive metrics contain aggregation of + // values from continuous and non-overlapping intervals. + // + // The values for a DELTA metric are based only on the time interval + // associated with one measurement cycle. There is no dependency on + // previous measurements like is the case for CUMULATIVE metrics. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // DELTA metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0+1 to + // t_0+2 with a value of 2. + AGGREGATION_TEMPORALITY_DELTA = 1; + + // CUMULATIVE is an AggregationTemporality for a metric aggregator which + // reports changes since a fixed start time. This means that current values + // of a CUMULATIVE metric depend on all previous measurements since the + // start time. Because of this, the sender is required to retain this state + // in some form. If this state is lost or invalidated, the CUMULATIVE metric + // values MUST be reset and a new fixed start time following the last + // reported measurement time sent MUST be used. + // + // For example, consider a system measuring the number of requests that + // it receives and reports the sum of these requests every second as a + // CUMULATIVE metric: + // + // 1. The system starts receiving at time=t_0. + // 2. A request is received, the system measures 1 request. + // 3. A request is received, the system measures 1 request. + // 4. A request is received, the system measures 1 request. + // 5. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+1 with a value of 3. + // 6. A request is received, the system measures 1 request. + // 7. A request is received, the system measures 1 request. + // 8. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_0 to + // t_0+2 with a value of 5. + // 9. The system experiences a fault and loses state. + // 10. The system recovers and resumes receiving at time=t_1. + // 11. A request is received, the system measures 1 request. + // 12. The 1 second collection cycle ends. A metric is exported for the + // number of requests received over the interval of time t_1 to + // t_0+1 with a value of 1. + // + // Note: Even though, when reporting changes since last report time, using + // CUMULATIVE is valid, it is not recommended. This may cause problems for + // systems that do not use start_time to determine when the aggregation + // value was reset (e.g. Prometheus). + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +// DataPointFlags is defined as a protobuf 'uint32' type and is to be used as a +// bit-field representing 32 distinct boolean flags. Each flag defined in this +// enum is a bit-mask. To test the presence of a single flag in the flags of +// a data point, for example, use an expression like: +// +// (point.flags & DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK) == DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK +// +enum DataPointFlags { + // The zero value for the enum. Should not be used for comparisons. + // Instead use bitwise "and" with the appropriate mask as shown above. + DATA_POINT_FLAGS_DO_NOT_USE = 0; + + // This DataPoint is valid but has no recorded value. This value + // SHOULD be used to reflect explicitly missing data in a series, as + // for an equivalent to the Prometheus "staleness marker". + DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK = 1; + + // Bits 2-31 are reserved for future use. +} + +// NumberDataPoint is a single data point in a timeseries that describes the +// time-varying scalar value of a metric. +message NumberDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list MUST be omitted (or present but empty) + // whenever `series_id` is non-zero, and MUST be populated when `series_id` + // is zero so the receiver can mint a new entry. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // The value itself. A point is considered invalid when one of the recognized + // value fields is not present inside this oneof. + oneof value { + double as_double = 4; + sfixed64 as_int = 6; + } + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 5; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; + + // series_id references a collector-assigned series descriptor. Receivers MUST + // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) + // is true. + uint64 series_id = 9; +} + +// HistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Histogram. A Histogram contains summary statistics +// for a population of values, it may optionally contain the distribution of +// those values across a set of buckets. +// +// If the histogram contains the distribution of values, then both +// "explicit_bounds" and "bucket counts" fields must be defined. +// If the histogram does not contain the distribution of values, then both +// "explicit_bounds" and "bucket_counts" must be omitted and only "count" and +// "sum" are known. +message HistogramDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list MUST be omitted (or present but empty) + // whenever `series_id` is non-zero, and MUST be populated when `series_id` + // is zero so the receiver can mint a new entry. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 9; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. This + // value must be equal to the sum of the "count" fields in buckets if a + // histogram is provided. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // bucket_counts is an optional field contains the count values of histogram + // for each bucket. + // + // The sum of the bucket_counts must equal the value in the count field. + // + // The number of elements in bucket_counts array must be by one greater than + // the number of elements in explicit_bounds array. The exception to this rule + // is when the length of bucket_counts is 0, then the length of explicit_bounds + // must also be 0. + repeated fixed64 bucket_counts = 6; + + // explicit_bounds specifies buckets with explicitly defined bounds for values. + // + // The boundaries for bucket at index i are: + // + // (-infinity, explicit_bounds[i]] for i == 0 + // (explicit_bounds[i-1], explicit_bounds[i]] for 0 < i < size(explicit_bounds) + // (explicit_bounds[i-1], +infinity) for i == size(explicit_bounds) + // + // The values in the explicit_bounds array must be strictly increasing. + // + // Histogram buckets are inclusive of their upper boundary, except the last + // bucket where the boundary is at infinity. This format is intentionally + // compatible with the OpenMetrics histogram definition. + // + // If bucket_counts length is 0 then explicit_bounds length must also be 0, + // otherwise the data point is invalid. + repeated double explicit_bounds = 7; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 8; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // min is the minimum value over (start_time, end_time]. + optional double min = 11; + + // max is the maximum value over (start_time, end_time]. + optional double max = 12; + + // series_id references a collector-assigned series descriptor. Receivers MUST + // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) + // is true. + uint64 series_id = 13; +} + +// ExponentialHistogramDataPoint is a single data point in a timeseries that describes the +// time-varying values of a ExponentialHistogram of double values. A ExponentialHistogram contains +// summary statistics for a population of values, it may optionally contain the +// distribution of those values across a set of buckets. +// +message ExponentialHistogramDataPoint { + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list MUST be omitted (or present but empty) + // whenever `series_id` is non-zero, and MUST be populated when `series_id` + // is zero so the receiver can mint a new entry. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // The number of values in the population. Must be + // non-negative. This value must be equal to the sum of the "bucket_counts" + // values in the positive and negative Buckets plus the "zero_count" field. + fixed64 count = 4; + + // The sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram + optional double sum = 5; + + // scale describes the resolution of the histogram. Boundaries are + // located at powers of the base, where: + // + // base = (2^(2^-scale)) + // + // The histogram bucket identified by `index`, a signed integer, + // contains values that are greater than (base^index) and + // less than or equal to (base^(index+1)). + // + // The positive and negative ranges of the histogram are expressed + // separately. Negative values are mapped by their absolute value + // into the negative range using the same scale as the positive range. + // + // scale is not restricted by the protocol, as the permissible + // values depend on the range of the data. + sint32 scale = 6; + + // The count of values that are either exactly zero or + // within the region considered zero by the instrumentation at the + // tolerated degree of precision. This bucket stores values that + // cannot be expressed using the standard exponential formula as + // well as values that have been rounded to zero. + // + // Implementations MAY consider the zero bucket to have probability + // mass equal to (zero_count / count). + fixed64 zero_count = 7; + + // positive carries the positive range of exponential bucket counts. + Buckets positive = 8; + + // negative carries the negative range of exponential bucket counts. + Buckets negative = 9; + + // Buckets are a set of bucket counts, encoded in a contiguous array + // of counts. + message Buckets { + // The bucket index of the first entry in the bucket_counts array. + // + // Note: This uses a varint encoding as a simple form of compression. + sint32 offset = 1; + + // An array of count values, where bucket_counts[i] carries + // the count of the bucket at index (offset+i). bucket_counts[i] is the count + // of values greater than base^(offset+i) and less than or equal to + // base^(offset+i+1). + // + // Note: By contrast, the explicit HistogramDataPoint uses + // fixed64. This field is expected to have many buckets, + // especially zeros, so uint64 has been selected to ensure + // varint encoding. + repeated uint64 bucket_counts = 2; + } + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 10; + + // (Optional) List of exemplars collected from + // measurements that were used to form the data point + repeated Exemplar exemplars = 11; + + // The minimum value over (start_time, end_time]. + optional double min = 12; + + // The maximum value over (start_time, end_time]. + optional double max = 13; + + // ZeroThreshold may be optionally set to convey the width of the zero + // region. Where the zero region is defined as the closed interval + // [-ZeroThreshold, ZeroThreshold]. + // When ZeroThreshold is 0, zero count bucket stores values that cannot be + // expressed using the standard exponential formula as well as values that + // have been rounded to zero. + double zero_threshold = 14; + + // series_id references a collector-assigned series descriptor. Receivers MUST + // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) + // is true. + uint64 series_id = 15; +} + +// SummaryDataPoint is a single data point in a timeseries that describes the +// time-varying values of a Summary metric. The count and sum fields represent +// cumulative values. +message SummaryDataPoint { + reserved 1; + + // The set of key/value pairs that uniquely identify the timeseries from + // where this point belongs. The list MUST be omitted (or present but empty) + // whenever `series_id` is non-zero, and MUST be populated when `series_id` + // is zero so the receiver can mint a new entry. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 7; + + // StartTimeUnixNano is optional but strongly encouraged, see the + // the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 start_time_unix_nano = 2; + + // TimeUnixNano is required, see the detailed comments above Metric. + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 3; + + // count is the number of values in the population. Must be non-negative. + fixed64 count = 4; + + // sum of the values in the population. If count is zero then this field + // must be zero. + // + // Note: Sum should only be filled out when measuring non-negative discrete + // events, and is assumed to be monotonic over the values of these events. + // Negative events *can* be recorded, but sum should not be filled out when + // doing so. This is specifically to enforce compatibility w/ OpenMetrics, + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary + double sum = 5; + + // Represents the value at a given quantile of a distribution. + // + // To record Min and Max values following conventions are used: + // - The 1.0 quantile is equivalent to the maximum value observed. + // - The 0.0 quantile is equivalent to the minimum value observed. + // + // See the following issue for more context: + // https://github.com/open-telemetry/opentelemetry-proto/issues/125 + message ValueAtQuantile { + // The quantile of a distribution. Must be in the interval + // [0.0, 1.0]. + double quantile = 1; + + // The value at the given quantile of a distribution. + // + // Quantile values must NOT be negative. + double value = 2; + } + + // (Optional) list of values at different quantiles of the distribution calculated + // from the current snapshot. The quantiles must be strictly increasing. + repeated ValueAtQuantile quantile_values = 6; + + // Flags that apply to this specific data point. See DataPointFlags + // for the available flags and their meaning. + uint32 flags = 8; + + // series_id references a collector-assigned series descriptor. Receivers MUST + // reject the point unless exactly one of (`series_id` != 0, `attributes` populated) + // is true. + uint64 series_id = 9; +} + +// A representation of an exemplar, which is a sample input measurement. +// Exemplars also hold information about the environment when the measurement +// was recorded, for example the span and trace ID of the active span when the +// exemplar was recorded. +message Exemplar { + reserved 1; + + // The set of key/value pairs that were filtered out by the aggregator, but + // recorded alongside the original measurement. Only key/value pairs that were + // filtered out by the aggregator should be included + repeated opentelemetry.proto.common.v1.KeyValue filtered_attributes = 7; + + // time_unix_nano is the exact time when this exemplar was recorded + // + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January + // 1970. + fixed64 time_unix_nano = 2; + + // The value of the measurement that was recorded. An exemplar is + // considered invalid when one of the recognized value fields is not present + // inside this oneof. + oneof value { + double as_double = 3; + sfixed64 as_int = 6; + } + + // (Optional) Span ID of the exemplar trace. + // span_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes span_id = 4; + + // (Optional) Trace ID of the exemplar trace. + // trace_id may be missing if the measurement is not recorded inside a trace + // or if the trace is not sampled. + bytes trace_id = 5; +} diff --git a/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto new file mode 100644 index 00000000..42c5913c --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/proto/opentelemetry/proto/resource/v1/resource.proto @@ -0,0 +1,45 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.resource.v1; + +import "opentelemetry/proto/common/v1/common.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Resource.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.resource.v1"; +option java_outer_classname = "ResourceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/resource/v1"; + +// Resource information. +message Resource { + // Set of attributes that describe the resource. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + // The behavior of software that receives duplicated keys can be unpredictable. + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // The number of dropped attributes. If the value is 0, then + // no attributes were dropped. + uint32 dropped_attributes_count = 2; + + // Set of entities that participate in this Resource. + // + // Note: keys in the references MUST exist in attributes of this message. + // + // Status: [Development] + repeated opentelemetry.proto.common.v1.EntityRef entity_refs = 3; +} diff --git a/asap-common/dependencies/rs/asap_otel_proto/src/lib.rs b/asap-common/dependencies/rs/asap_otel_proto/src/lib.rs new file mode 100644 index 00000000..b0b2120d --- /dev/null +++ b/asap-common/dependencies/rs/asap_otel_proto/src/lib.rs @@ -0,0 +1,41 @@ +//! Vendored Rust bindings for DataCollector's modified OTLP proto. +//! +//! Adds first-class `DDSketch`, `KLLSketch`, `CountSketch`, `CountMinSketch`, +//! and `HLLSketch` variants to the `Metric.data` oneof on tags 13–17, plus +//! per-sketch encoding enums (with full and `*_DELTA` variants) and a +//! `series_id` field on every data point. See +//! `docs/pipeline-query-catalog.md` §5.4 in the DataCollector repo for the +//! full motivation and the corresponding upstream proto path. +//! +//! The generated Rust modules mirror `opentelemetry_proto::tonic::*` so call +//! sites only need to swap `opentelemetry_proto` for `asap_otel_proto`. + +#![allow(clippy::all)] + +pub mod tonic { + pub mod common { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.common.v1"); + } + } + + pub mod resource { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.resource.v1"); + } + } + + pub mod metrics { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.metrics.v1"); + } + } + + pub mod collector { + pub mod metrics { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.collector.metrics.v1"); + } + } + } +} diff --git a/asap-query-engine/Cargo.toml b/asap-query-engine/Cargo.toml index b432627a..e81c8539 100644 --- a/asap-query-engine/Cargo.toml +++ b/asap-query-engine/Cargo.toml @@ -47,7 +47,11 @@ datafusion = "43" arrow = "53.4.1" futures = "0.3" prost = "0.13" -opentelemetry-proto = { version = "0.28", features = ["gen-tonic", "gen-tonic-messages", "metrics"] } +# Vendored modified opentelemetry-proto with first-class sketch metric +# variants from DataCollector (DDSketch / KLLSketch / CountSketch / +# CountMinSketch / HLLSketch). See docs/pipeline-query-catalog.md §5.4 in the +# DataCollector repo for the rationale. +asap_otel_proto = { path = "../asap-common/dependencies/rs/asap_otel_proto" } tonic = "0.12" tokio-stream = "0.1" snap = "1" diff --git a/asap-query-engine/src/drivers/ingest/otel.rs b/asap-query-engine/src/drivers/ingest/otel.rs index 8f0aed3b..3677e54f 100644 --- a/asap-query-engine/src/drivers/ingest/otel.rs +++ b/asap-query-engine/src/drivers/ingest/otel.rs @@ -28,15 +28,15 @@ use crate::data_model::AggregateCore; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::IngestState; use crate::precompute_operators::sketch_envelope_accumulator::SketchEnvelopeAccumulator; -use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; -use axum::{body::Bytes, extract::State, routing::post, Json, Router}; -use flate2::read::GzDecoder; -use opentelemetry_proto::tonic::collector::metrics::v1::{ +use asap_otel_proto::tonic::collector::metrics::v1::{ metrics_service_server::MetricsService, ExportMetricsServiceRequest, ExportMetricsServiceResponse, }; -use opentelemetry_proto::tonic::common::v1::any_value::Value as AnyValueVariant; -use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as NumberValue; +use asap_otel_proto::tonic::common::v1::any_value::Value as AnyValueVariant; +use asap_otel_proto::tonic::metrics::v1::number_data_point::Value as NumberValue; +use asap_sketchlib::proto::sketchlib::{sketch_envelope, SketchEnvelope}; +use axum::{body::Bytes, extract::State, routing::post, Json, Router}; +use flate2::read::GzDecoder; use prost::Message; use std::sync::Arc; use std::time::Instant; @@ -98,7 +98,7 @@ impl OtlpReceiver { shared: shared.clone(), }; let grpc_svc = - opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsServiceServer::new( + asap_otel_proto::tonic::collector::metrics::v1::metrics_service_server::MetricsServiceServer::new( grpc_svc, ); @@ -150,6 +150,10 @@ impl MetricsService for MetricsServiceImpl { debug!("OTLP sending response via gRPC"); Ok(Response::new(ExportMetricsServiceResponse { partial_success: None, + // Modified-OTLP collector hands out stable series descriptors via + // this field; not yet wired (PR B will populate it when the + // backend learns to mint series_ids). + series_assignments: Vec::new(), })) } } @@ -229,7 +233,7 @@ fn format_series_key(name: &str, labels: &HashMap) -> String { } fn get_sketch_payload_from_attrs( - attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue], + attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], ) -> Option<(String, Vec)> { for kv in attrs { match kv.key.as_str() { @@ -501,7 +505,7 @@ fn otlp_to_record_count(request: &ExportMetricsServiceRequest) -> usize { continue; } - use opentelemetry_proto::tonic::metrics::v1::metric::Data; + use asap_otel_proto::tonic::metrics::v1::metric::Data; match &metric.data { Some(Data::Gauge(g)) => count += g.data_points.len(), Some(Data::Sum(s)) => count += s.data_points.len(), @@ -530,6 +534,11 @@ fn otlp_to_record_count(request: &ExportMetricsServiceRequest) -> usize { count += dp.quantile_values.len(); } } + Some(Data::Ddsketch(d)) => count += d.data_points.len(), + Some(Data::Kllsketch(k)) => count += k.data_points.len(), + Some(Data::Countsketch(c)) => count += c.data_points.len(), + Some(Data::Countminsketch(c)) => count += c.data_points.len(), + Some(Data::Hllsketch(h)) => count += h.data_points.len(), None => {} } } @@ -569,7 +578,7 @@ fn otlp_to_metric_points_and_sketches(request: &ExportMetricsServiceRequest) -> .map(|(k, v)| (k.clone(), v.clone())) .collect(); - use opentelemetry_proto::tonic::metrics::v1::metric::Data; + use asap_otel_proto::tonic::metrics::v1::metric::Data; match &metric.data { Some(Data::Gauge(g)) => { for dp in &g.data_points { @@ -710,6 +719,46 @@ fn otlp_to_metric_points_and_sketches(request: &ExportMetricsServiceRequest) -> }); } } + // Modified-OTLP first-class sketch metric variants. PR A + // vendors the proto and surfaces the new arms; PR B will + // populate them with per-variant decoders that route via + // WorkerMessage::AccumulatorInput. For now, drop with a + // debug log so the metric is visible in the ingest path. + Some(Data::Ddsketch(d)) => { + debug!( + "OTLP modified-proto Ddsketch received (metric={}, dps={}); decoder is PR B", + metric.name, + d.data_points.len() + ); + } + Some(Data::Kllsketch(k)) => { + debug!( + "OTLP modified-proto Kllsketch received (metric={}, dps={}); decoder is PR B", + metric.name, + k.data_points.len() + ); + } + Some(Data::Countsketch(c)) => { + debug!( + "OTLP modified-proto Countsketch received (metric={}, dps={}); decoder is PR B", + metric.name, + c.data_points.len() + ); + } + Some(Data::Countminsketch(c)) => { + debug!( + "OTLP modified-proto Countminsketch received (metric={}, dps={}); decoder is PR B", + metric.name, + c.data_points.len() + ); + } + Some(Data::Hllsketch(h)) => { + debug!( + "OTLP modified-proto Hllsketch received (metric={}, dps={}); decoder is PR B", + metric.name, + h.data_points.len() + ); + } None => {} } } @@ -720,7 +769,7 @@ fn otlp_to_metric_points_and_sketches(request: &ExportMetricsServiceRequest) -> fn merge_point_attributes( base: &HashMap, - attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue], + attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], ) -> HashMap { let mut m = base.clone(); for (k, v) in attributes_to_map(attrs) { @@ -737,8 +786,8 @@ fn number_value_to_f64(v: &Option) -> f64 { } } -fn any_value_to_string(v: &opentelemetry_proto::tonic::common::v1::AnyValue) -> String { - use opentelemetry_proto::tonic::common::v1::any_value::Value as AnyValueVariant; +fn any_value_to_string(v: &asap_otel_proto::tonic::common::v1::AnyValue) -> String { + use asap_otel_proto::tonic::common::v1::any_value::Value as AnyValueVariant; match &v.value { Some(AnyValueVariant::StringValue(s)) => s.clone(), Some(AnyValueVariant::IntValue(i)) => i.to_string(), @@ -750,7 +799,7 @@ fn any_value_to_string(v: &opentelemetry_proto::tonic::common::v1::AnyValue) -> } fn attributes_to_map( - attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue], + attrs: &[asap_otel_proto::tonic::common::v1::KeyValue], ) -> HashMap { let mut m = HashMap::new(); for kv in attrs {