From 07dae3a17ddcdd3cc9d4319aee291706f1e9139b Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Mon, 2 Nov 2026 13:57:25 +0100 Subject: [PATCH 1/6] chores(deps): removed useless build dependencies --- core/src/components/metrics_tracer/Cargo.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/components/metrics_tracer/Cargo.toml b/core/src/components/metrics_tracer/Cargo.toml index 4816770c..2c48094f 100644 --- a/core/src/components/metrics_tracer/Cargo.toml +++ b/core/src/components/metrics_tracer/Cargo.toml @@ -8,9 +8,6 @@ aya-ebpf = { git = "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/aya-rs/aya" } aya-log-ebpf = { git = "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/aya-rs/aya" } bytemuck = {version ="1.23.0",features = ["derive"]} -[build-dependencies] -which = { version = "7.0.0", default-features = false } - [[bin]] name = "metrics_tracer" path = "src/main.rs" From d2e1d6023e839f711ed3a8c9f293ed8a927791fc Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Mon, 2 Nov 2026 13:58:10 +0100 Subject: [PATCH 2/6] refactor(optimization): removed unnecessary mut in metrics/helpers.rs --- core/src/components/metrics/src/helpers.rs | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/core/src/components/metrics/src/helpers.rs b/core/src/components/metrics/src/helpers.rs index 66473def..6d9fe844 100644 --- a/core/src/components/metrics/src/helpers.rs +++ b/core/src/components/metrics/src/helpers.rs @@ -186,8 +186,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let net_metrics_handle = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = net_perf_buffer; - let mut buffers = net_metrics_buffers; + let array_buffers = net_perf_buffer; + let buffers = net_metrics_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -203,8 +203,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let time_stamp_handle = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = time_stamp_events_perf_buffer; - let mut buffers = time_stamp_events_buffers; + let array_buffers = time_stamp_events_perf_buffer; + let buffers = time_stamp_events_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -220,8 +220,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let cpu_frequency_metrics = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = cpu_frequency_perf_buffer; - let mut buffers = cpu_frequency_events_buffers; + let array_buffers = cpu_frequency_perf_buffer; + let buffers = cpu_frequency_events_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -237,8 +237,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let cpu_idle_metrics = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = cpu_idle_perf_buffer; - let mut buffers = cpu_idle_buffers; + let array_buffers = cpu_idle_perf_buffer; + let buffers = cpu_idle_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -254,8 +254,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let mem_alloc_metrics = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = mem_alloc_perf_buffer; - let mut buffers = mem_alloc_buffers; + let array_buffers = mem_alloc_perf_buffer; + let buffers = mem_alloc_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -272,8 +272,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = sched_stat_wait_perf_buffer; - let mut buffers = sched_stat_wait_buffers; + let array_buffers = sched_stat_wait_perf_buffer; + let buffers = sched_stat_wait_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -289,8 +289,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let sched_stat_runtime_metrics = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = sched_stat_runtime_perf_buffer; - let mut buffers = sched_stat_runtime_buffers; + let array_buffers = sched_stat_runtime_perf_buffer; + let buffers = sched_stat_runtime_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, @@ -306,8 +306,8 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a let ssl_events_metrics = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); - let mut array_buffers = ssl_events_perf_buffer; - let mut buffers = ssl_events_buffers; + let array_buffers = ssl_events_perf_buffer; + let buffers = ssl_events_buffers; tokio::spawn(async move { read_perf_buffer( array_buffers, From 10b03577d576d4f7e6f2dc24509fb082b9dcd95a Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Mon, 2 Nov 2026 13:58:24 +0100 Subject: [PATCH 3/6] chores(deps): updated cargo.lock --- core/Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index df8fb621..5a137228 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1500,7 +1500,6 @@ dependencies = [ "aya-ebpf", "aya-log-ebpf", "bytemuck", - "which", ] [[package]] From 5934de5607db5a81582f70ab02988d676ee1b848 Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Fri, 6 Nov 2026 14:19:44 +0100 Subject: [PATCH 4/6] feat(metrics) updated libssl resolver to locate the openssl library and dynamically attach the uprobe --- core/common/src/map_handlers.rs | 27 ++-- core/src/components/metrics/src/helpers.rs | 177 +++++++++++++++++++-- 2 files changed, 180 insertions(+), 24 deletions(-) diff --git a/core/common/src/map_handlers.rs b/core/common/src/map_handlers.rs index b246b701..d5b853b3 100644 --- a/core/common/src/map_handlers.rs +++ b/core/common/src/map_handlers.rs @@ -8,8 +8,7 @@ use kube::{Api, Client}; use std::net::Ipv4Addr; use std::path::PathBuf; use std::str::FromStr; -use std::sync::Arc; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use tracing::warn; use tracing::{error, info}; @@ -23,6 +22,7 @@ use tracing::{error, info}; pub struct BpfMapsData { pub bpf_obj_names: Vec, pub bpf_obj_map: Vec, + pub bpf: Arc>, } #[cfg(feature = "map-handlers")] @@ -30,21 +30,25 @@ pub fn init_bpf_maps( bpf: Arc>, map_names: Vec, ) -> Result { - let mut bpf_new = bpf - .lock() - .map_err(|e| anyhow::anyhow!("Cannot get value from lock. Reason: {}", e))?; - let mut maps = Vec::new(); // stores bpf_maps_objects - for name in &map_names { - let bpf_map_init = bpf_new - .take_map(&name) - .ok_or_else(|| anyhow::anyhow!("{} map not found", &name))?; - maps.push(bpf_map_init); + { + let mut bpf_new = bpf + .lock() + .map_err(|e| anyhow::anyhow!("Cannot get value from lock. Reason: {}", e))?; + + for name in &map_names { + let bpf_map_init = bpf_new + .take_map(&name) + .ok_or_else(|| anyhow::anyhow!("{} map not found", &name))?; + maps.push(bpf_map_init); + } } + Ok(BpfMapsData { bpf_obj_names: map_names.clone(), bpf_obj_map: maps, + bpf, }) } @@ -68,6 +72,7 @@ pub fn map_pinner(maps: BpfMapsData, path: &PathBuf) -> Result Vec { + let mut paths = Vec::new(); + + if let Ok(path) = env::var(constants::LIBSSL_PATH) { + if Path::new(&path).is_file() { + paths.push(path); + } else if !path.is_empty() { + INVALID_LIBSSL_PATH.call_once(|| warn!("Invalid LIBSSL_PATH: {}", path)); + } + } + + let Some(proc_dir) = ["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/host/proc", "/proc"] + .into_iter() + .find(|dir| Path::new(dir).is_dir()) + else { + return paths; + }; + let Ok(entries) = fs::read_dir(proc_dir) else { + return paths; + }; + + let mut roots = HashSet::new(); + for entry in entries.flatten() { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + let pid = pid.to_string(); + let root = Path::new(proc_dir).join(&pid).join("root"); + let Ok(metadata) = fs::metadata(&root) else { + continue; + }; + if !roots.insert((metadata.dev(), metadata.ino())) { + continue; + } + + for dir in LIBSSL_DIRS { + for name in LIBSSL_NAMES { + let path = root.join(dir.trim_start_matches('/')).join(name); + if path.is_file() { + paths.push(path.to_string_lossy().into_owned()); + } + } + } + } + + paths +} + +fn attach_uprobe( + bpf: &Arc>, + program_name: &str, + symbol: &str, + target: &str, +) -> Result<(), anyhow::Error> { + let mut bpf = bpf + .lock() + .map_err(|e| anyhow::anyhow!("Cannot get value from lock. Reason: {}", e))?; + let program: &mut UProbe = bpf + .program_mut(program_name) + .ok_or_else(|| anyhow::anyhow!("Program {} not found", program_name))? + .try_into() + .map_err(|e| anyhow::anyhow!("Failed to convert program: {:?}", e))?; + match program.load() { + Ok(()) | Err(ProgramError::AlreadyLoaded) => {} + Err(e) => { + return Err(anyhow::anyhow!( + "Cannot load program: {}. Error: {}", + program_name, + e + )); + } + } + program + .attach(Some(symbol), 0, target, None) + .map_err(|e| anyhow::anyhow!("Failed to attach {} to {}: {:?}", program_name, target, e))?; + info!( + "{} program attached successfully to user space function {}", + program_name, target + ); + Ok(()) +} + +fn attach_ssl_uprobes(bpf: &Arc>, attached: &mut HashSet<(u64, u64)>) { + for path in resolve_libssl_paths() { + let Ok(metadata) = fs::metadata(&path) else { + continue; + }; + if !attached.insert((metadata.dev(), metadata.ino())) { + continue; + } + for (program, symbol) in SSL_UPROBES { + if let Err(e) = attach_uprobe(bpf, program, symbol, &path) { + debug!("SSL uprobe {} -> {}: {}", program, path, e); + } + } + } +} + /// Listen for eBPF perf-buffer events and record OpenTelemetry metrics. /// /// This function bridges the eBPF perf-buffer layer with the OpenTelemetry @@ -97,6 +231,7 @@ pub async fn init_metrics_and_cache( pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), anyhow::Error> { info!("Getting CPU count..."); + let bpf = Arc::clone(&bpf_maps.bpf); let mut maps = map_manager(bpf_maps)?; let cpu_count = online_cpus().map_err(|e| anyhow::anyhow!("Error {:?}", e))?; @@ -183,6 +318,22 @@ pub async fn event_listener(bpf_maps: BpfMapsData, meter: Meter) -> Result<(), a }) }; + let mut attached = HashSet::new(); + if let Ok(Some(path)) = resolve_libssl_path() { + if let Ok(metadata) = fs::metadata(&path) { + attached.insert((metadata.dev(), metadata.ino())); + } + } + let _ssl_handle = { + tokio::spawn(async move { + let mut interval = time::interval(Duration::from_secs(60)); + loop { + interval.tick().await; + attach_ssl_uprobes(&bpf, &mut attached); + } + }) + }; + let net_metrics_handle = { let metrics = Arc::clone(&metrics); let service_cache = Arc::clone(&cache); From 080396f74da880bc3475d10326d1e6ce41a5baaf Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Fri, 6 Nov 2026 17:19:38 +0100 Subject: [PATCH 5/6] feat(metrics): added DEFAULT_MAP_READING_TIME parameter to set the sleep time while reading the perf_buffers --- Examples/run-with-docker/docker-compose.yaml | 1 + core/common/src/constants.rs | 2 ++ core/common/src/consumer.rs | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Examples/run-with-docker/docker-compose.yaml b/Examples/run-with-docker/docker-compose.yaml index 63f3a443..167509c1 100644 --- a/Examples/run-with-docker/docker-compose.yaml +++ b/Examples/run-with-docker/docker-compose.yaml @@ -12,6 +12,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 - OTEL_EXPORTER_OTLP_PROTOCOL=grpc - LIBSSL_PATH=/host/usr/lib/x86_64-linux-gnu/libssl.so.3 + - DEFAULT_MAP_READING_TIME=100 volumes: - type: bind source: /sys/fs/bpf diff --git a/core/common/src/constants.rs b/core/common/src/constants.rs index b364dac4..c656933c 100644 --- a/core/common/src/constants.rs +++ b/core/common/src/constants.rs @@ -10,3 +10,5 @@ pub const PIN_MAP_PATH: &str = "PIN_MAP_PATH"; /// When set, this path is used directly. When unset, the library is located /// by searching the default system library directories. pub const LIBSSL_PATH: &str = "LIBSSL_PATH"; + +pub const DEFAULT_MAP_READING_TIME : &str = "DEFAULT_MAP_READING_TIME"; diff --git a/core/common/src/consumer.rs b/core/common/src/consumer.rs index 00d68436..de83bc8c 100644 --- a/core/common/src/consumer.rs +++ b/core/common/src/consumer.rs @@ -16,6 +16,7 @@ use crate::buffer_type::{ CpuFrequency, CpuIdle, MemAlloc, PacketLossMetrics, SchedStatRuntime, SchedStatWait, TimeStampMetrics, }; +use crate::constants::DEFAULT_MAP_READING_TIME; #[cfg(feature = "network-structs")] use crate::buffer_type::{PacketLog, TcpPacketRegistry, VethLog}; #[cfg(feature = "monitoring-structs")] @@ -899,6 +900,6 @@ pub async fn read_perf_buffer>( } } } - tokio::time::sleep(std::time::Duration::from_millis(5000)).await; + tokio::time::sleep(std::time::Duration::from_millis(std::env::var(DEFAULT_MAP_READING_TIME).ok().and_then(|v| v.parse::().ok()).unwrap_or(5000))).await; } } From 3cc3272d946de0902b58e54bcd2ca8556d3ec112 Mon Sep 17 00:00:00 2001 From: LorenzoTettamanti Date: Fri, 6 Nov 2026 17:20:41 +0100 Subject: [PATCH 6/6] fix(metrics_tracer): added fix candidate for the sched_stat_wait metrics --- core/src/components/metrics_tracer/src/cpu.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/components/metrics_tracer/src/cpu.rs b/core/src/components/metrics_tracer/src/cpu.rs index 580b9d09..a4cacc40 100644 --- a/core/src/components/metrics_tracer/src/cpu.rs +++ b/core/src/components/metrics_tracer/src/cpu.rs @@ -1,7 +1,3 @@ -//tracepoint:power:cpu_frequency -//tracepoint:power:cpu_frequency_limits -//tracepoint:power:cpu_idle -//tracepoint:power:cpu_idle_miss use aya_ebpf::{ EbpfContext, helpers::{bpf_get_current_pid_tgid, generated::bpf_get_current_cgroup_id}, @@ -39,7 +35,6 @@ pub fn cpu_idle(ctx: TracePointContext) -> Result<(), i64> { } pub fn per_cpu_bytes_alloc(ctx: &TracePointContext) -> Result<((u32, u32, [u8; 16])), i64> { - // TODO: this tracepoint needs debug since it's not triggering let bytes_alloc_offset = 64; let pid_offset = 4; let bytes_alloc = unsafe { ctx.read_at(bytes_alloc_offset) }?; @@ -57,9 +52,11 @@ pub fn per_cpu_bytes_alloc(ctx: &TracePointContext) -> Result<((u32, u32, [u8; 1 Ok((bytes_alloc, tgid, command)) } - +// sometimes this tracepoint does not work. A possible fix is to enable the tracepoint using +// sudo sysctl kernel.sched_schedstats=1 pub fn sched_stat_wait(ctx: &TracePointContext) -> Result<((u32, u64, [u8; 16], u64)), i64> { let delay_offset = 16; + let command_offset = 8; //let tgid: u32 = unsafe { ctx.read_at(tgid_offset) }?; let pid_tgid: u64 = bpf_get_current_pid_tgid(); @@ -67,7 +64,7 @@ pub fn sched_stat_wait(ctx: &TracePointContext) -> Result<((u32, u64, [u8; 16], let cgroup_id: u64 = unsafe { bpf_get_current_cgroup_id() }; let delay = unsafe { ctx.read_at(delay_offset) }?; - let command = ctx.command()?; + let command = unsafe{ctx.read_at(command_offset)}?; Ok((tgid, delay, command, cgroup_id)) }