diff --git a/CMakeLists.txt b/CMakeLists.txt index a4a008e3d..36008c160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ set(DD_PROFILING_SOURCES src/ringbuffer_utils.cc src/signal_helper.cc src/sys_utils.cc + src/tracepoint_config.cc src/user_override.cc) if(BUILD_UNIVERSAL_DDPROF) diff --git a/include/ddprof_perf_event.hpp b/include/ddprof_perf_event.hpp new file mode 100644 index 000000000..ceda11982 --- /dev/null +++ b/include/ddprof_perf_event.hpp @@ -0,0 +1,37 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#pragma once + +#include +#include + +// Extend the perf event types +// There are <30 different perf events (starting at 1000 seems safe) +enum : uint32_t { + PERF_CUSTOM_EVENT_DEALLOCATION = 1000, + PERF_CUSTOM_EVENT_CLEAR_LIVE_ALLOCATION +}; + +static_assert(static_cast(PERF_CUSTOM_EVENT_DEALLOCATION) > + PERF_RECORD_MAX, + "Error from PERF_CUSTOM_EVENT_DEALLOCATION definition"); + +namespace ddprof { + +// Custom sample type +struct DeallocationEvent { + perf_event_header hdr; + struct sample_id sample_id; + uintptr_t ptr; +}; + +// Event to notify we have tracked too many allocations +struct ClearLiveAllocationEvent { + perf_event_header hdr; + struct sample_id sample_id; +}; + +} // namespace ddprof diff --git a/include/ddprof_worker_context.hpp b/include/ddprof_worker_context.hpp index de05c3dca..f7c969ace 100644 --- a/include/ddprof_worker_context.hpp +++ b/include/ddprof_worker_context.hpp @@ -5,6 +5,7 @@ #pragma once +#include "live_allocation.hpp" #include "pevent.hpp" #include "proc_status.hpp" @@ -36,4 +37,5 @@ struct DDProfWorkerContext { int64_t send_nanos; // Last time an export was sent uint32_t count_worker; // exports since last cache clear std::array lost_events_per_watcher; + ddprof::LiveAllocation live_allocation; }; diff --git a/include/ddres_list.hpp b/include/ddres_list.hpp index 399808907..8488fc814 100644 --- a/include/ddres_list.hpp +++ b/include/ddres_list.hpp @@ -50,6 +50,7 @@ X(DSO, "") \ X(JIT, "Error parsing JIT files") \ X(NO_JIT_FILE, "File not readable for JIT") \ + X(UNHANDLED_CONFIG, "unhandled configuration") \ X(UNHANDLED_DSO, "ignore dso type") \ X(WORKERLOOP_INIT, "error initializing the worker loop") \ X(SYS_CONFIG, "error checking system configuration") \ diff --git a/include/dwfl_hdr.hpp b/include/dwfl_hdr.hpp index 2669156e2..c37dde031 100644 --- a/include/dwfl_hdr.hpp +++ b/include/dwfl_hdr.hpp @@ -64,7 +64,8 @@ struct DwflWrapper { class DwflHdr { public: DwflWrapper &get_or_insert(pid_t pid); - void clear_unvisited(); + std::vector get_unvisited() const; + void reset_unvisited(); void clear_pid(pid_t pid); // get number of accessed modules diff --git a/include/event_config.hpp b/include/event_config.hpp index 9ddcc51e1..f7f1c4a46 100644 --- a/include/event_config.hpp +++ b/include/event_config.hpp @@ -5,39 +5,45 @@ #pragma once -#include - #include +#include // Defines how a sample is aggregated when it is received -enum class EventConfMode { +enum class EventConfMode : uint32_t { kDisabled = 0, - kCallgraph = 1 << 0, - kMetric = 1 << 1, - kAll = kCallgraph | kMetric, + kCallgraph = 1 << 0, // flamegraph of resource usage + kMetric = 1 << 1, // gauge of resource usage + kLiveCallgraph = 1 << 2, // report callgraph of resources still in use + kAll = kCallgraph | kMetric | kLiveCallgraph, }; -// EventConfMode &operator|=(EventConfMode &A, const EventConfMode &B); -// EventConfMode operator&(const EventConfMode &A, const EventConfMode &B); -// bool operator<=(const EventConfMode A, const EventConfMode B); // inclusion -// -constexpr EventConfMode &operator|=(EventConfMode &A, const EventConfMode &B) { - A = static_cast(static_cast(A) | - static_cast(B)); - return A; +bool operator<=(EventConfMode A, EventConfMode B) = delete; +bool operator<(EventConfMode A, EventConfMode B) = delete; +bool operator>(EventConfMode A, EventConfMode B) = delete; +bool operator>=(EventConfMode A, EventConfMode B) = delete; + +constexpr EventConfMode operator|(EventConfMode A, const EventConfMode B) { + return static_cast(static_cast(A) | + static_cast(B)); +} + +constexpr EventConfMode operator|=(EventConfMode &A, const EventConfMode B) { + return A = A | B; +} + +constexpr EventConfMode operator&(const EventConfMode A, + const EventConfMode B) { + return static_cast(static_cast(A) & + static_cast(B)); } -constexpr EventConfMode operator&(const EventConfMode &A, - const EventConfMode &B) { - // & on bitmask enums is valid only in the space spanned by the values - return static_cast(static_cast(A) & - static_cast(B) & - static_cast(EventConfMode::kAll)); +constexpr bool Any(EventConfMode arg) { + return arg != EventConfMode::kDisabled; } -// Bitmask inclusion -constexpr bool operator<=(const EventConfMode A, const EventConfMode B) { - return EventConfMode::kDisabled != ((EventConfMode::kAll & A) & B); +constexpr bool AnyCallgraph(EventConfMode arg) { + return Any((arg & EventConfMode::kLiveCallgraph) | + (arg & EventConfMode::kCallgraph)); } // Defines how samples are weighted @@ -139,7 +145,7 @@ enum class EventConfField { struct EventConf { EventConfMode mode; - uint64_t id; + int64_t id; std::string eventname; std::string groupname; diff --git a/include/ipc.hpp b/include/ipc.hpp index 47ed8c507..6c78343cc 100644 --- a/include/ipc.hpp +++ b/include/ipc.hpp @@ -94,6 +94,7 @@ struct RingBufferInfo { }; struct ReplyMessage { + enum { kLiveCallgraph = 0 }; // reply with the request flags from the request uint32_t request = 0; // profiler pid @@ -102,6 +103,7 @@ struct ReplyMessage { // RingBufferInfo is returned if request & kRingBuffer // cppcheck-suppress unusedStructMember RingBufferInfo ring_buffer; + int32_t allocation_flags = 0; }; class Client { diff --git a/include/lib/allocation_tracker.hpp b/include/lib/allocation_tracker.hpp index 7da487222..43efb31c4 100644 --- a/include/lib/allocation_tracker.hpp +++ b/include/lib/allocation_tracker.hpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace ddprof { @@ -38,6 +39,8 @@ class AllocationTracker { AllocationTracker(const AllocationTracker &) = delete; AllocationTracker &operator=(const AllocationTracker &) = delete; + ~AllocationTracker() { free(); } + enum AllocationTrackingFlags { kTrackDeallocations = 0x1, kDeterministicSampling = 0x2 @@ -60,13 +63,22 @@ class AllocationTracker { static inline bool is_active(); private: + using AdressSet = std::unordered_set; + struct TrackerState { + void init(bool track_alloc, bool track_dealloc) { + track_allocations = track_alloc; + track_deallocations = track_dealloc; + lost_count = 0; + failure_count = 0; + pid = 0; + } std::mutex mutex; std::atomic track_allocations = false; std::atomic track_deallocations = false; std::atomic lost_count; // count number of lost events std::atomic failure_count; - std::atomic pid; // cache of pid + std::atomic pid; // lazy cache of pid (0 is un-init value) }; AllocationTracker(); @@ -82,16 +94,24 @@ class AllocationTracker { TrackerThreadLocalState &tl_state); void track_deallocation(uintptr_t addr, TrackerThreadLocalState &tl_state); - DDRes push_sample(uint64_t allocated_size, TrackerThreadLocalState &tl_state); + DDRes push_alloc_sample(uintptr_t addr, uint64_t allocated_size, + TrackerThreadLocalState &tl_state); - // Return true if consumer should be notified + // If notify_needed is true, consumer should be notified DDRes push_lost_sample(MPSCRingBufferWriter &writer, bool ¬ify_needed); + DDRes push_dealloc_sample(uintptr_t addr, TrackerThreadLocalState &tl_state); + + DDRes push_clear_live_allocation(TrackerThreadLocalState &tl_state); + + void free_on_consecutive_failures(bool success); + TrackerState _state; uint64_t _sampling_interval; std::mt19937 _gen; PEvent _pevent; bool _deterministic_sampling; + AdressSet _address_set; static thread_local TrackerThreadLocalState _tl_state; static AllocationTracker *_instance; @@ -128,7 +148,21 @@ void AllocationTracker::track_allocation(uintptr_t addr, size_t size) { } } -void AllocationTracker::track_deallocation(uintptr_t) {} +void AllocationTracker::track_deallocation(uintptr_t addr) { + // same pattern as track_allocation + AllocationTracker *instance = _instance; + + if (!instance) { + return; + } + TrackerThreadLocalState &tl_state = _tl_state; + + if (instance->_state.track_deallocations.load(std::memory_order_relaxed)) { + // not cool as we are always calling this (high overhead). Can we do better + // ? + instance->track_deallocation(addr, tl_state); + } +} bool AllocationTracker::is_active() { auto instance = _instance; diff --git a/include/live_allocation-c.hpp b/include/live_allocation-c.hpp new file mode 100644 index 000000000..657e84b1b --- /dev/null +++ b/include/live_allocation-c.hpp @@ -0,0 +1,17 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#pragma once + +namespace ddprof { +namespace liveallocation { +#ifdef KMAX_TRACKED_ALLOCATIONS +// build time override to reduce execution time of test +static constexpr auto kMaxTracked = KMAX_TRACKED_ALLOCATIONS; +#else +static constexpr auto kMaxTracked = 500000; +#endif +} // namespace liveallocation +} // namespace ddprof diff --git a/include/live_allocation.hpp b/include/live_allocation.hpp new file mode 100644 index 000000000..f19ff339b --- /dev/null +++ b/include/live_allocation.hpp @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#pragma once + +#include "ddprof_defs.hpp" +#include "unlikely.hpp" +#include "unwind_output_hash.hpp" + +#include +#include +#include + +namespace ddprof { + +template +T &access_resize(std::vector &v, size_t index, + const T &default_value = T()) { + if (unlikely(index >= v.size())) { + v.resize(index + 1, default_value); + } + return v[index]; +} + +class LiveAllocation { +public: + // For allocations Value is the size + // This is the cumulative value and count for a given stack + struct ValueAndCount { + int64_t _value = 0; + int64_t _count = 0; + }; + + using PprofStacks = + std::unordered_map; + + struct ValuePerAddress { + int64_t _value = 0; + PprofStacks::value_type *_unique_stack = nullptr; + }; + + using AddressMap = std::unordered_map; + struct PidStacks { + AddressMap _address_map; + PprofStacks _unique_stacks; + }; + + using PidMap = std::unordered_map; + using WatcherVector = std::vector; + WatcherVector _watcher_vector; + + // Allocation should be aggregated per stack trace + // instead of a stack, we would have a total size for this unique stack trace + // and a count. + void register_allocation(const UnwindOutput &uo, uintptr_t addr, size_t size, + int watcher_pos, pid_t pid) { + PidMap &pid_map = access_resize(_watcher_vector, watcher_pos); + PidStacks &pid_stacks = pid_map[pid]; + register_allocation(uo, addr, size, pid_stacks._unique_stacks, + pid_stacks._address_map); + } + + void register_deallocation(uintptr_t addr, int watcher_pos, pid_t pid) { + PidMap &pid_map = access_resize(_watcher_vector, watcher_pos); + PidStacks &pid_stacks = pid_map[pid]; + register_deallocation(addr, pid_stacks._unique_stacks, + pid_stacks._address_map); + } + + void clear_pid_for_watcher(int watcher_pos, pid_t pid) { + PidMap &pid_map = access_resize(_watcher_vector, watcher_pos); + pid_map.erase(pid); + } + + void clear_pid(pid_t pid) { + for (auto &pid_map : _watcher_vector) { + pid_map.erase(pid); + } + } + +private: + static void register_deallocation(uintptr_t address, PprofStacks &stacks, + AddressMap &address_map); + + static void register_allocation(const UnwindOutput &uo, uintptr_t address, + int64_t value, PprofStacks &stacks, + AddressMap &address_map); +}; + +} // namespace ddprof \ No newline at end of file diff --git a/include/perf.hpp b/include/perf.hpp index f1910beb4..d18fb98b4 100644 --- a/include/perf.hpp +++ b/include/perf.hpp @@ -150,3 +150,6 @@ all_perf_configs_from_watcher(const PerfWatcher *watcher, bool extras); uint64_t perf_value_from_sample(const PerfWatcher *watcher, const perf_event_sample *sample); } // namespace ddprof + +perf_event_attr perf_config_from_watcher(const PerfWatcher *watcher, + bool extras); diff --git a/include/perf_watcher.hpp b/include/perf_watcher.hpp index 7fd0a7d38..0f9ae8f98 100644 --- a/include/perf_watcher.hpp +++ b/include/perf_watcher.hpp @@ -56,6 +56,7 @@ struct PerfWatcher { bool suppress_tid; int pprof_sample_idx; // index into the SampleType in the pprof int pprof_count_sample_idx; // index into the pprof for the count + bool instrument_self; // do my own perfopen, etc EventConfMode output_mode; // defines how sample data is aggregated }; @@ -80,7 +81,9 @@ typedef enum DDPROF_SAMPLE_TYPES { // Define our own event type on top of perf event types enum DDProfTypeId { kDDPROF_TYPE_CUSTOM = PERF_TYPE_MAX + 100 }; -enum DDProfCustomCountId { kDDPROF_COUNT_ALLOCATIONS = 0 }; +enum DDProfCustomCountId { + kDDPROF_COUNT_ALLOCATIONS = 0, +}; // Kernel events are necessary to get a full accounting of CPU // This depend on the state of configuration (capabilities / @@ -109,29 +112,30 @@ enum DDProfCustomCountId { kDDPROF_COUNT_ALLOCATIONS = 0 }; // events are marked as tracepoint unless they represent a well-known profiling // type! // clang-format off -// short desc perf event type perf event count type period/freq profile sample type addtl. configs +// short desc perf event type perf event count type period/freq profile sample type addtl. configs // cppcheck-suppress preprocessorErrorDirective #define EVENT_CONFIG_TABLE(X) \ - X(hCPU, "CPU Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(hREF, "Ref. CPU Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_REF_CPU_CYCLES, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(hINST, "Instr. Count", PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(hCREF, "Cache Ref.", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, 999, DDPROF_PWT_TRACEPOINT, {}) \ - X(hCMISS, "Cache Miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, 999, DDPROF_PWT_TRACEPOINT, {}) \ - X(hBRANCH, "Branche Instr.", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_INSTRUCTIONS, 999, DDPROF_PWT_TRACEPOINT, {}) \ - X(hBMISS, "Branch Miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES, 999, DDPROF_PWT_TRACEPOINT, {}) \ - X(hBUS, "Bus Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BUS_CYCLES, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(hBSTF, "Bus Stalls(F)", PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(hBSTB, "Bus Stalls(B)", PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_BACKEND, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(sCPU, "CPU Time", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK, 99, DDPROF_PWT_CPU_NANOS, IS_FREQ_TRY_KERNEL) \ - X(sPF, "Page Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS, 1, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ - X(sCS, "Con. Switch", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES, 1, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ - X(sMig, "CPU Migrations", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_MIGRATIONS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(sPFMAJ, "Minor Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MIN, 99, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ - X(sPFMIN, "Major Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MAJ, 99, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ - X(sALGN, "Align. Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_ALIGNMENT_FAULTS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(sEMU, "Emu. Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EMULATION_FAULTS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ - X(sDUM, "Dummy", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_DUMMY, 1, DDPROF_PWT_NOCOUNT, {}) \ - X(sALLOC, "Allocations", kDDPROF_TYPE_CUSTOM, kDDPROF_COUNT_ALLOCATIONS, 524288, DDPROF_PWT_ALLOC_SPACE, SKIP_FRAMES) + X(hCPU, "CPU Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(hREF, "Ref. CPU Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_REF_CPU_CYCLES, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(hINST, "Instr. Count", PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(hCREF, "Cache Ref.", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, 999, DDPROF_PWT_TRACEPOINT, {}) \ + X(hCMISS, "Cache Miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, 999, DDPROF_PWT_TRACEPOINT, {}) \ + X(hBRANCH, "Branche Instr.", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_INSTRUCTIONS, 999, DDPROF_PWT_TRACEPOINT, {}) \ + X(hBMISS, "Branch Miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES, 999, DDPROF_PWT_TRACEPOINT, {}) \ + X(hBUS, "Bus Cycles", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BUS_CYCLES, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(hBSTF, "Bus Stalls(F)", PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(hBSTB, "Bus Stalls(B)", PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_BACKEND, 1000, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(sCPU, "CPU Time", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK, 99, DDPROF_PWT_CPU_NANOS, IS_FREQ_TRY_KERNEL) \ + X(sPF, "Page Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS, 1, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ + X(sCS, "Con. Switch", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES, 1, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ + X(sMig, "CPU Migrations", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_MIGRATIONS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(sPFMAJ, "Major Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MAJ, 99, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ + X(sPFMIN, "Minor Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MIN, 99, DDPROF_PWT_TRACEPOINT, USE_KERNEL) \ + X(sALGN, "Align. Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_ALIGNMENT_FAULTS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(sEMU, "Emu. Faults", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EMULATION_FAULTS, 99, DDPROF_PWT_TRACEPOINT, IS_FREQ) \ + X(sDUM, "Dummy", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_DUMMY, 1, DDPROF_PWT_NOCOUNT, {}) \ + X(sALLOC, "Allocations", kDDPROF_TYPE_CUSTOM, kDDPROF_COUNT_ALLOCATIONS, 524288, DDPROF_PWT_ALLOC_SPACE, SKIP_FRAMES) + // clang-format on #define X_ENUM(a, b, c, d, e, f, g) DDPROF_PWE_##a, @@ -157,3 +161,4 @@ int sample_type_id_to_count_sample_type_id(int idx); // Helper functions, mostly for tests uint64_t perf_event_default_sample_type(); +void log_watcher(const PerfWatcher *w, int idx); \ No newline at end of file diff --git a/include/pevent.hpp b/include/pevent.hpp index d2016c8b4..f4c17efcd 100644 --- a/include/pevent.hpp +++ b/include/pevent.hpp @@ -24,6 +24,8 @@ typedef struct PEvent { bool custom_event; // true if custom event (not handled by perf, eg. memory // allocations) RingBuffer rb; // metadata and buffers for processing perf ringbuffer + int child_fds[MAX_NB_PERF_EVENT_OPEN]; + int current_child_fd; } PEvent; typedef struct PEventHdr { diff --git a/include/pprof/ddprof_pprof.hpp b/include/pprof/ddprof_pprof.hpp index 226f5e972..2e4d555f7 100644 --- a/include/pprof/ddprof_pprof.hpp +++ b/include/pprof/ddprof_pprof.hpp @@ -33,7 +33,7 @@ DDRes pprof_create_profile(DDProfPProf *pprof, DDProfContext *ctx); * @param pprof */ DDRes pprof_aggregate(const UnwindOutput *uw_output, - const SymbolHdr *symbol_hdr, uint64_t value, + const SymbolHdr &symbol_hdr, uint64_t value, uint64_t count, const PerfWatcher *watcher, DDProfPProf *pprof); diff --git a/include/tracepoint_config.hpp b/include/tracepoint_config.hpp new file mode 100644 index 000000000..87fbaa53a --- /dev/null +++ b/include/tracepoint_config.hpp @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#pragma once + +#include +#include + +namespace ddprof { +// Returns the ID of the given Linux tracepoint, or -1 if an error occurs. +int64_t tracepoint_get_id(std::string_view global_name, + std::string_view tracepoint_name); +} // namespace ddprof diff --git a/include/unwind_output.hpp b/include/unwind_output.hpp index 72e8e7bb6..1076e48d6 100644 --- a/include/unwind_output.hpp +++ b/include/unwind_output.hpp @@ -8,22 +8,27 @@ #pragma once #include +#include #include "ddprof_defs.hpp" -#include "string_view.hpp" typedef struct FunLoc { uint64_t ip; // Relative to file, not VMA SymbolIdx_t _symbol_idx; MapInfoIdx_t _map_info_idx; + + auto operator<=>(const FunLoc &) const = default; } FunLoc; -typedef struct UnwindOutput { - FunLoc locs[DD_MAX_STACK_DEPTH]; - uint64_t nb_locs; +struct UnwindOutput { + void clear() { + locs.clear(); + is_incomplete = true; + } + std::vector locs; int pid; int tid; bool is_incomplete; -} UnwindOutput; -void uw_output_clear(UnwindOutput *); + auto operator<=>(const UnwindOutput &) const = default; +}; diff --git a/include/unwind_output_hash.hpp b/include/unwind_output_hash.hpp new file mode 100644 index 000000000..604dbabb5 --- /dev/null +++ b/include/unwind_output_hash.hpp @@ -0,0 +1,29 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#include "unwind_output.hpp" + +namespace ddprof { + +template inline void hash_combine(std::size_t &seed, const T &v) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +struct UnwindOutputHash { + std::size_t operator()(const UnwindOutput &uo) const noexcept { + std::size_t seed = 0; + hash_combine(seed, uo.pid); + hash_combine(seed, uo.tid); + for (const auto &fl : uo.locs) { + hash_combine(seed, fl.ip); + hash_combine(seed, fl._symbol_idx); + hash_combine(seed, fl._map_info_idx); + } + return seed; + } +}; + +} // namespace ddprof \ No newline at end of file diff --git a/include/unwind_state.hpp b/include/unwind_state.hpp index f1bc7626a..57b415991 100644 --- a/include/unwind_state.hpp +++ b/include/unwind_state.hpp @@ -39,7 +39,8 @@ struct UnwindState { explicit UnwindState(int dd_profiling_fd = -1) : _dwfl_wrapper(nullptr), dso_hdr("", dd_profiling_fd), pid(-1), stack(nullptr), stack_sz(0), current_ip(0) { - uw_output_clear(&output); + output.clear(); + output.locs.reserve(DD_MAX_STACK_DEPTH); } ddprof::DwflHdr dwfl_hdr; diff --git a/src/ddprof_cmdline.cc b/src/ddprof_cmdline.cc index 11b0645c6..08662cc73 100644 --- a/src/ddprof_cmdline.cc +++ b/src/ddprof_cmdline.cc @@ -7,19 +7,11 @@ #include #include -#include -#include -#include -#include -#include -#include -#include - -#include "ddres_helpers.hpp" + #include "event_config.hpp" -#include "event_parser.h" -#include "perf_archmap.hpp" +#include "logger.hpp" #include "perf_watcher.hpp" +#include "tracepoint_config.hpp" int arg_which(const char *str, char const *const *set, int sz_set) { if (!str || !set) @@ -48,40 +40,8 @@ bool arg_yesno(const char *str, int mode) { return false; } -unsigned int tracepoint_id_from_event(const char *eventname, - const char *groupname) { - if (!eventname || !*eventname || !groupname || !*groupname) - return 0; - - static char path[4096]; // Arbitrary, but path sizes limits are difficult - static char buf[sizeof("4294967296")]; // For reading 32-bit decimal int - char *buf_copy = buf; - size_t pathsz = - snprintf(path, sizeof(path), "/sys/kernel/tracing/events/%s/%s/id", - groupname, eventname); - if (pathsz >= sizeof(path)) - return 0; - int fd = open(path, O_RDONLY); - if (-1 == fd) - return 0; - - // Read the data in an eintr-safe way - int read_ret = -1; - long trace_id = 0; - do { - read_ret = read(fd, buf, sizeof(buf)); - } while (read_ret == -1 && errno == EINTR); - close(fd); - if (read_ret > 0) - trace_id = strtol(buf, &buf_copy, 10); - if (*buf_copy && *buf_copy != '\n') - return 0; - - return trace_id; -} - // If this returns false, then the passed watcher should be regarded as invalid -constexpr uint64_t kIgnoredWatcherID = -1ul; +constexpr int64_t kIgnoredWatcherID = -1l; bool watcher_from_str(const char *str, PerfWatcher *watcher) { EventConf *conf = EventConf_parse(str); if (!conf) { @@ -112,23 +72,23 @@ bool watcher_from_str(const char *str, PerfWatcher *watcher) { return false; } - // The most likely thing to be invalid is the selection of the tracepoint - // from the trace events system. If the conf has a nonzero number for the id - // we assume the user has privileged information and knows what they want. - // Else, we use the group/event combination to extract that id from the - // tracefs filesystem in the canonical way. - uint64_t tracepoint_id = 0; - if (conf->id > 0) { - tracepoint_id = conf->id; - } else { - tracepoint_id = tracepoint_id_from_event(conf->eventname.c_str(), - conf->groupname.c_str()); - } - - // 0 is an error, "-1" is ignored - if (!tracepoint_id) { - return false; - } else if (tracepoint_id != kIgnoredWatcherID) { + if (conf->id != kIgnoredWatcherID) { + // The most likely thing to be invalid is the selection of the tracepoint + // from the trace events system. If the conf has a nonzero number for the + // id we assume the user has privileged information and knows what they + // want. Else, we use the group/event combination to extract that id from + // the tracefs filesystem in the canonical way. + int64_t tracepoint_id = 0; + if (conf->id > 0) { + tracepoint_id = conf->id; + } else { + tracepoint_id = + ddprof::tracepoint_get_id(conf->eventname, conf->groupname); + } + // At this point we needed to find a valid tracepoint id + if (tracepoint_id == kIgnoredWatcherID) { + return false; + } watcher->config = tracepoint_id; } @@ -161,13 +121,20 @@ bool watcher_from_str(const char *str, PerfWatcher *watcher) { // The output mode isn't set as part of the configuration templates; we // always default to callgraph mode - watcher->output_mode = EventConfMode::kCallgraph; - if (EventConfMode::kAll <= conf->mode) { + if (conf->mode != EventConfMode::kDisabled) { watcher->output_mode = conf->mode; + } else { + watcher->output_mode = EventConfMode::kCallgraph; } watcher->tracepoint_event = conf->eventname; watcher->tracepoint_group = conf->groupname; watcher->tracepoint_label = conf->label; + + // Allocation watcher, has an extra field to ensure we capture address + if (watcher->config == kDDPROF_COUNT_ALLOCATIONS) { + watcher->sample_type |= PERF_SAMPLE_ADDR; + } + return true; } diff --git a/src/ddprof_context_lib.cc b/src/ddprof_context_lib.cc index 697be345d..37b64c335 100644 --- a/src/ddprof_context_lib.cc +++ b/src/ddprof_context_lib.cc @@ -56,6 +56,7 @@ DDRes add_preset(DDProfContext *ctx, const char *preset, {"default-pid", {"sCPU"}}, {"cpu_only", {"sCPU"}}, {"alloc_only", {"sALLOC"}}, + {"cpu_live_heap", {"sCPU", "sALLOC mode=l"}}, }; if (preset == "default"sv && pid_or_global_mode) { @@ -99,40 +100,6 @@ DDRes add_preset(DDProfContext *ctx, const char *preset, return {}; } -static void log_watcher(const PerfWatcher *w, int idx) { - PRINT_NFO(" ID: %s, Pos: %d, Index: %lu", w->desc.c_str(), idx, w->config); - switch (w->value_source) { - case EventConfValueSource::kSample: - PRINT_NFO(" Location: Sample"); - break; - case EventConfValueSource::kRegister: - PRINT_NFO(" Location: Register, regno: %d", w->regno); - break; - case EventConfValueSource::kRaw: - PRINT_NFO(" Location: Raw event, offset: %d, size: %d", w->raw_off, - w->raw_sz); - break; - default: - PRINT_NFO(" ILLEGAL LOCATION"); - break; - } - - PRINT_NFO(" Category: %s, EventName: %s, GroupName: %s, Label: %s", - sample_type_name_from_idx(w->sample_type_id), - w->tracepoint_event.c_str(), w->tracepoint_group.c_str(), - w->tracepoint_label.c_str()); - - if (w->options.is_freq) - PRINT_NFO(" Cadence: Freq, Freq: %lu", w->sample_frequency); - else - PRINT_NFO(" Cadence: Period, Period: %lu", w->sample_period); - - if (EventConfMode::kCallgraph <= w->output_mode) - PRINT_NFO(" Outputting to callgraph (flamegraph)"); - if (EventConfMode::kMetric <= w->output_mode) - PRINT_NFO(" Outputting to metric"); -} - /**************************** Argument Processor ***************************/ DDRes ddprof_context_set(DDProfInput *input, DDProfContext *ctx) { *ctx = {}; @@ -152,6 +119,7 @@ DDRes ddprof_context_set(DDProfInput *input, DDProfContext *ctx) { ctx->watchers[nwatchers] = input->watchers[nwatchers]; } ctx->num_watchers = nwatchers; + // Set defaults ctx->params.upload_period = 60.0; diff --git a/src/ddprof_input.cc b/src/ddprof_input.cc index 4aefcf698..223cb5693 100644 --- a/src/ddprof_input.cc +++ b/src/ddprof_input.cc @@ -146,7 +146,7 @@ const char* help_str[DD_KLEN] = { [DD_PROFILING_NATIVE_SHOW_SAMPLES] = STR_UNDF, [DD_PROFILING_NATIVE_CPU_AFFINITY] = STR_UNDF, [DD_PROFILING_NATIVE_SWITCH_USER] = - " Run the target process under the given user.\n", + " Run the target process under the given user.\n" }; // clang-format on diff --git a/src/ddprof_worker.cc b/src/ddprof_worker.cc index 497411bf7..b56ccde53 100644 --- a/src/ddprof_worker.cc +++ b/src/ddprof_worker.cc @@ -6,6 +6,7 @@ #include "ddprof_worker.hpp" #include "ddprof_context.hpp" +#include "ddprof_perf_event.hpp" #include "ddprof_stats.hpp" #include "dso_hdr.hpp" #include "dwfl_hdr.hpp" @@ -41,6 +42,11 @@ static const DDPROF_STATS s_cycled_stats[] = { static const long k_clock_ticks_per_sec = sysconf(_SC_CLK_TCK); +/// Remove all structures related to +static DDRes worker_pid_free(DDProfContext *ctx, pid_t el); + +static DDRes clear_unvisited_pids(DDProfContext *ctx); + /// Human readable runtime information static void print_diagnostics(const DsoHdr &dso_hdr) { LG_NFO("Printing internal diagnostics"); @@ -60,7 +66,7 @@ static DDRes report_lost_events(DDProfContext *ctx) { if (ctx->worker_ctx.lost_events_per_watcher[watcher_idx] > 0) { PerfWatcher *watcher = &ctx->watchers[watcher_idx]; UnwindState *us = ctx->worker_ctx.us; - uw_output_clear(&us->output); + us->output.clear(); add_common_frame(us, SymbolErrors::lost_event); LG_WRN("Reporting #%lu -> [%lu] lost samples for watcher #%d", ctx->worker_ctx.lost_events_per_watcher[watcher_idx], @@ -68,7 +74,7 @@ static DDRes report_lost_events(DDProfContext *ctx) { watcher->sample_period, watcher_idx); DDRES_CHECK_FWD(pprof_aggregate( - &us->output, &us->symbol_hdr, watcher->sample_period, + &us->output, us->symbol_hdr, watcher->sample_period, ctx->worker_ctx.lost_events_per_watcher[watcher_idx], watcher, ctx->worker_ctx.pprof[ctx->worker_ctx.i_current_pprof])); ctx->worker_ctx.lost_events_per_watcher[watcher_idx] = 0; @@ -221,19 +227,14 @@ static DDRes worker_update_stats(ProcStatus *procstat, const UnwindState &us, return ddres_init(); } -/************************* perf_event_open() helpers **************************/ -/// Entry point for sample aggregation -DDRes ddprof_pr_sample(DDProfContext *ctx, perf_event_sample *sample, - int watcher_pos) { - if (!sample) - return ddres_warn(DD_WHAT_PERFSAMP); +static DDRes ddprof_unwind_sample(DDProfContext *ctx, perf_event_sample *sample, + int watcher_pos) { struct UnwindState *us = ctx->worker_ctx.us; PerfWatcher *watcher = &ctx->watchers[watcher_pos]; ddprof_stats_add(STATS_SAMPLE_COUNT, 1, NULL); ddprof_stats_add(STATS_UNWIND_AVG_STACK_SIZE, sample->size_stack, nullptr); - auto ticks0 = ddprof::get_tsc_cycles(); // copy the sample context into the unwind structure unwind_init_sample(us, sample->regs, sample->pid, sample->size_stack, sample->data_stack); @@ -248,7 +249,7 @@ DDRes ddprof_pr_sample(DDProfContext *ctx, perf_event_sample *sample, // Attempt to fully unwind if the watcher has a callgraph type DDRes res = {}; - if (EventConfMode::kCallgraph <= watcher->output_mode) + if (AnyCallgraph(watcher->output_mode)) res = unwindstate__unwind(us); /* This test is not 100% accurate: @@ -275,40 +276,72 @@ DDRes ddprof_pr_sample(DDProfContext *ctx, perf_event_sample *sample, ddprof_stats_add(STATS_UNWIND_TRUNCATED_INPUT, 1, nullptr); } + if (us->_dwfl_wrapper->_inconsistent) { + // Loaded modules were inconsistend, assume we should flush everything. + LG_WRN("(Inconsistent DWFL/DSOs)%d - Free associated objects", us->pid); + DDRES_CHECK_FWD(worker_pid_free(ctx, us->pid)); + } + return res; +} + +/************************* perf_event_open() helpers **************************/ +/// Entry point for sample aggregation +DDRes ddprof_pr_sample(DDProfContext *ctx, perf_event_sample *sample, + int watcher_pos) { + if (!sample) + return ddres_warn(DD_WHAT_PERFSAMP); + + // If this is a SW_TASK_CLOCK-type event, then aggregate the time + if (ctx->watchers[watcher_pos].config == PERF_COUNT_SW_TASK_CLOCK) + ddprof_stats_add(STATS_TARGET_CPU_USAGE, sample->period, NULL); + + auto ticks0 = ddprof::get_tsc_cycles(); + DDRes res = ddprof_unwind_sample(ctx, sample, watcher_pos); auto unwind_ticks = ddprof::get_tsc_cycles(); - DDRES_CHECK_FWD( - ddprof_stats_add(STATS_UNWIND_AVG_TIME, unwind_ticks - ticks0, NULL)); + ddprof_stats_add(STATS_UNWIND_AVG_TIME, unwind_ticks - ticks0, NULL); + + // Usually we want to send the sample_val, but sometimes we need to process + // the event to get the desired value + PerfWatcher *watcher = &ctx->watchers[watcher_pos]; // Aggregate if unwinding went well (todo : fatal error propagation) - if (!IsDDResFatal(res) && EventConfMode::kCallgraph <= watcher->output_mode) { + if (!IsDDResFatal(res)) { + struct UnwindState *us = ctx->worker_ctx.us; + if (Any(EventConfMode::kLiveCallgraph & watcher->output_mode)) { + // Live callgraph mode + // for now we hard code the live aggregation mode + ctx->worker_ctx.live_allocation.register_allocation( + us->output, sample->addr, sample->period, watcher_pos, sample->pid); + } else if (Any(EventConfMode::kCallgraph & watcher->output_mode)) { #ifndef DDPROF_NATIVE_LIB - // Depending on the type of watcher, compute a value for sample - uint64_t sample_val = perf_value_from_sample(watcher, sample); - - // in lib mode we don't aggregate (protect to avoid link failures) - int i_export = ctx->worker_ctx.i_current_pprof; - DDProfPProf *pprof = ctx->worker_ctx.pprof[i_export]; - DDRES_CHECK_FWD(pprof_aggregate(&us->output, &us->symbol_hdr, sample_val, 1, - watcher, pprof)); - if (ctx->params.show_samples) { - ddprof_print_sample(us->output, us->symbol_hdr, sample->period, *watcher); - } + // Depending on the type of watcher, compute a value for sample + uint64_t sample_val = perf_value_from_sample(watcher, sample); + + // in lib mode we don't aggregate (protect to avoid link failures) + int i_export = ctx->worker_ctx.i_current_pprof; + DDProfPProf *pprof = ctx->worker_ctx.pprof[i_export]; + DDRES_CHECK_FWD(pprof_aggregate(&us->output, us->symbol_hdr, sample_val, + 1, watcher, pprof)); + if (ctx->params.show_samples) { + ddprof_print_sample(us->output, us->symbol_hdr, sample->period, + *watcher); + } #else - // Call the user's stack handler - if (ctx->stack_handler) { - if (!ctx->stack_handler->apply(&us->output, ctx, - ctx->stack_handler->callback_ctx, - watcher_pos)) { - DDRES_RETURN_ERROR_LOG(DD_WHAT_STACK_HANDLE, - "Stack handler returning errors"); + // Call the user's stack handler + if (ctx->stack_handler) { + if (!ctx->stack_handler->apply(&us->output, ctx, + ctx->stack_handler->callback_ctx, + watcher_pos)) { + DDRES_RETURN_ERROR_LOG(DD_WHAT_STACK_HANDLE, + "Stack handler returning errors"); + } } - } #endif + } } - DDRES_CHECK_FWD(ddprof_stats_add(STATS_AGGREGATION_AVG_TIME, - ddprof::get_tsc_cycles() - unwind_ticks, - NULL)); + ddprof_stats_add(STATS_AGGREGATION_AVG_TIME, + ddprof::get_tsc_cycles() - unwind_ticks, NULL); return {}; } @@ -340,10 +373,100 @@ void *ddprof_worker_export_thread(void *arg) { } #endif +#ifndef DDPROF_NATIVE_LIB +static DDRes aggregate_livealloc_stack( + const LiveAllocation::PprofStacks::value_type &alloc_info, + DDProfContext *ctx, const PerfWatcher *watcher, DDProfPProf *pprof, + const SymbolHdr &symbol_hdr) { + DDRES_CHECK_FWD(pprof_aggregate(&alloc_info.first, symbol_hdr, + alloc_info.second._value, + alloc_info.second._count, watcher, pprof)); + if (ctx->params.show_samples) { + ddprof_print_sample(alloc_info.first, symbol_hdr, alloc_info.second._value, + *watcher); + } + return ddres_init(); +} + +static DDRes aggregate_live_allocations_for_pid(DDProfContext *ctx, pid_t pid) { + struct UnwindState *us = ctx->worker_ctx.us; + int i_export = ctx->worker_ctx.i_current_pprof; + DDProfPProf *pprof = ctx->worker_ctx.pprof[i_export]; + const SymbolHdr &symbol_hdr = us->symbol_hdr; + LiveAllocation &live_allocations = ctx->worker_ctx.live_allocation; + for (unsigned watcher_pos = 0; + watcher_pos < live_allocations._watcher_vector.size(); ++watcher_pos) { + auto &pid_map = live_allocations._watcher_vector[watcher_pos]; + const PerfWatcher *watcher = &ctx->watchers[watcher_pos]; + auto &pid_stacks = pid_map[pid]; + for (const auto &alloc_info : pid_stacks._unique_stacks) { + DDRES_CHECK_FWD(aggregate_livealloc_stack(alloc_info, ctx, watcher, pprof, + symbol_hdr)); + } + } + return ddres_init(); +} + +static DDRes aggregate_live_allocations(DDProfContext *ctx) { + // this would be more efficient if we could reuse the same stacks in + // libdatadog + struct UnwindState *us = ctx->worker_ctx.us; + int i_export = ctx->worker_ctx.i_current_pprof; + DDProfPProf *pprof = ctx->worker_ctx.pprof[i_export]; + const SymbolHdr &symbol_hdr = us->symbol_hdr; + const LiveAllocation &live_allocations = ctx->worker_ctx.live_allocation; + for (unsigned watcher_pos = 0; + watcher_pos < live_allocations._watcher_vector.size(); ++watcher_pos) { + const auto &pid_map = live_allocations._watcher_vector[watcher_pos]; + const PerfWatcher *watcher = &ctx->watchers[watcher_pos]; + for (const auto &pid_vt : pid_map) { + for (const auto &alloc_info : pid_vt.second._unique_stacks) { + DDRES_CHECK_FWD(aggregate_livealloc_stack(alloc_info, ctx, watcher, + pprof, symbol_hdr)); + } + LG_NTC("<%u> Number of Live allocations for PID%d=%lu, Unique stacks=%lu", + watcher_pos, pid_vt.first, pid_vt.second._address_map.size(), + pid_vt.second._unique_stacks.size()); + } + } + return ddres_init(); +} + +static DDRes worker_pid_free(DDProfContext *ctx, pid_t el) { + DDRES_CHECK_FWD(aggregate_live_allocations_for_pid(ctx, el)); + UnwindState *us = ctx->worker_ctx.us; + unwind_pid_free(us, el); + ctx->worker_ctx.live_allocation.clear_pid(el); + return ddres_init(); +} +#else +static DDRes worker_pid_free(DDProfContext *ctx, pid_t el) { + UnwindState *us = ctx->worker_ctx.us; + unwind_pid_free(us, el); + ctx->worker_ctx.live_allocation.clear_pid(el); + return ddres_init(); +} +#endif + +static DDRes clear_unvisited_pids(DDProfContext *ctx) { + UnwindState *us = ctx->worker_ctx.us; + const std::vector pids_remove = us->dwfl_hdr.get_unvisited(); + for (pid_t el : pids_remove) { + DDRES_CHECK_FWD(worker_pid_free(ctx, el)); + } + us->dwfl_hdr.reset_unvisited(); + return ddres_init(); +} + /// Cycle operations : export, sync metrics, update counters DDRes ddprof_worker_cycle(DDProfContext *ctx, int64_t now, [[maybe_unused]] bool synchronous_export) { + + // Clearing unused PIDs will ensure we don't report them at next cycle + DDRES_CHECK_FWD(clear_unvisited_pids(ctx)); #ifndef DDPROF_NATIVE_LIB + DDRES_CHECK_FWD(aggregate_live_allocations(ctx)); + // Take the current pprof contents and ship them to the backend. This also // clears the pprof for reuse // Dispatch happens in a thread, with the underlying data structure for @@ -449,22 +572,24 @@ void ddprof_pr_lost(DDProfContext *ctx, const perf_event_lost *lost, ctx->worker_ctx.lost_events_per_watcher[watcher_pos] += lost->lost; } -void ddprof_pr_comm(DDProfContext *ctx, const perf_event_comm *comm, - int watcher_pos) { +DDRes ddprof_pr_comm(DDProfContext *ctx, const perf_event_comm *comm, + int watcher_pos) { // Change in process name (assuming exec) : clear all associated dso if (comm->header.misc & PERF_RECORD_MISC_COMM_EXEC) { LG_DBG("<%d>(COMM)%d -> %s", watcher_pos, comm->pid, comm->comm); - unwind_pid_free(ctx->worker_ctx.us, comm->pid); + DDRES_CHECK_FWD(worker_pid_free(ctx, comm->pid)); } + return ddres_init(); } -void ddprof_pr_fork(DDProfContext *ctx, const perf_event_fork *frk, - int watcher_pos) { +DDRes ddprof_pr_fork(DDProfContext *ctx, const perf_event_fork *frk, + int watcher_pos) { LG_DBG("<%d>(FORK)%d -> %d/%d", watcher_pos, frk->ppid, frk->pid, frk->tid); if (frk->ppid != frk->pid) { // Clear everything and populate at next error or with coming samples - unwind_pid_free(ctx->worker_ctx.us, frk->pid); + DDRES_CHECK_FWD(worker_pid_free(ctx, frk->pid)); } + return ddres_init(); } void ddprof_pr_exit(DDProfContext *ctx, const perf_event_exit *ext, @@ -482,6 +607,20 @@ void ddprof_pr_exit(DDProfContext *ctx, const perf_event_exit *ext, } } +void ddprof_pr_clear_live_allocation(DDProfContext *ctx, + const ClearLiveAllocationEvent *event, + int watcher_pos) { + LG_DBG("<%d>(CLEAR LIVE)%d", watcher_pos, event->sample_id.pid); + ctx->worker_ctx.live_allocation.clear_pid_for_watcher(watcher_pos, + event->sample_id.pid); +} + +void ddprof_pr_deallocation(DDProfContext *ctx, const DeallocationEvent *event, + int watcher_pos) { + ctx->worker_ctx.live_allocation.register_deallocation(event->ptr, watcher_pos, + event->sample_id.pid); +} + /********************************** callbacks *********************************/ DDRes ddprof_worker_maybe_export(DDProfContext *ctx, int64_t now_ns) { try { @@ -584,11 +723,12 @@ DDRes ddprof_worker_process_event(const perf_event_header *hdr, int watcher_pos, ddprof_stats_add(STATS_EVENT_COUNT, 1, NULL); const perf_event_hdr_wpid *wpid = static_cast(hdr); + PerfWatcher *watcher = &ctx->watchers[watcher_pos]; switch (hdr->type) { /* Cases where the target type has a PID */ case PERF_RECORD_SAMPLE: if (wpid->pid) { - uint64_t mask = ctx->watchers[watcher_pos].sample_type; + uint64_t mask = watcher->sample_type; perf_event_sample *sample = hdr2samp(hdr, mask); if (sample) { DDRES_CHECK_FWD(ddprof_pr_sample(ctx, sample, watcher_pos)); @@ -602,8 +742,8 @@ DDRes ddprof_worker_process_event(const perf_event_header *hdr, int watcher_pos, break; case PERF_RECORD_COMM: if (wpid->pid) - ddprof_pr_comm(ctx, reinterpret_cast(hdr), - watcher_pos); + DDRES_CHECK_FWD(ddprof_pr_comm( + ctx, reinterpret_cast(hdr), watcher_pos)); break; case PERF_RECORD_EXIT: if (wpid->pid) @@ -612,8 +752,9 @@ DDRes ddprof_worker_process_event(const perf_event_header *hdr, int watcher_pos, break; case PERF_RECORD_FORK: if (wpid->pid) - ddprof_pr_fork(ctx, reinterpret_cast(hdr), - watcher_pos); + DDRES_CHECK_FWD(ddprof_pr_fork( + ctx, reinterpret_cast(hdr), watcher_pos)); + break; /* Cases where the target type might not have a PID */ @@ -621,6 +762,19 @@ DDRes ddprof_worker_process_event(const perf_event_header *hdr, int watcher_pos, ddprof_pr_lost(ctx, reinterpret_cast(hdr), watcher_pos); break; + case PERF_CUSTOM_EVENT_DEALLOCATION: + ddprof_pr_deallocation( + ctx, reinterpret_cast(hdr), watcher_pos); + break; + case PERF_CUSTOM_EVENT_CLEAR_LIVE_ALLOCATION: { + const ClearLiveAllocationEvent *event = + reinterpret_cast(hdr); +#ifndef DDPROF_NATIVE_LIB + DDRES_CHECK_FWD( + aggregate_live_allocations_for_pid(ctx, event->sample_id.pid)); +#endif + ddprof_pr_clear_live_allocation(ctx, event, watcher_pos); + } break; default: break; } diff --git a/src/dwfl_hdr.cc b/src/dwfl_hdr.cc index 67ff4628b..ee93c4dea 100644 --- a/src/dwfl_hdr.cc +++ b/src/dwfl_hdr.cc @@ -81,7 +81,7 @@ DDProfMod *DwflWrapper::register_mod(ProcessAddress_t pc, const Dso &dso, .first->second; } -void DwflHdr::clear_unvisited() { +std::vector DwflHdr::get_unvisited() const { std::vector pids_remove; for (auto &el : _dwfl_map) { if (_visited_pid.find(el.first) == _visited_pid.end()) { @@ -89,11 +89,10 @@ void DwflHdr::clear_unvisited() { pids_remove.push_back(el.first); } } - for (pid_t el : pids_remove) { - _dwfl_map.erase(el); - LG_NFO("[DWFL] DWFL Map Clearing PID%d", el); - } + return pids_remove; +} +void DwflHdr::reset_unvisited() { // clear the list of visited for next cycle _visited_pid.clear(); } diff --git a/src/event_parser/event_parser.y b/src/event_parser/event_parser.y index 5402225bd..0fa30b8e5 100644 --- a/src/event_parser/event_parser.y +++ b/src/event_parser/event_parser.y @@ -38,18 +38,24 @@ EventConfMode mode_from_str(const std::string &str) { EventConfMode mode = EventConfMode::kDisabled; if (str.empty()) return mode; - - const std::string m_str{"Mm"}; - const std::string g_str{"Gg"}; const std::string a_str{"Aa*"}; + const std::string l_str{"Ll"}; + const std::string g_str{"Gg"}; + const std::string m_str{"Mm"}; for (const char &c : str) { - if (m_str.find(c) != std::string::npos) + if (m_str.find(c) != std::string::npos) { mode |= EventConfMode::kMetric; - if (g_str.find(c) != std::string::npos) + } + if (g_str.find(c) != std::string::npos) { mode |= EventConfMode::kCallgraph; - if (a_str.find(c) != std::string::npos) + } + if (l_str.find(c) != std::string::npos) { + mode |= EventConfMode::kLiveCallgraph; + } + if (a_str.find(c) != std::string::npos) { mode |= EventConfMode::kAll; + } } return mode; } @@ -89,7 +95,7 @@ void conf_print(const EventConf *tp) { else printf(" label: \n"); - const char *modenames[] = {"ILLEGAL", "callgraph", "metric", "metric and callgraph"}; + const char *modenames[] = {"ILLEGAL", "callgraph", "metric", "live callgraph", "metric and callgraph"}; printf(" type: %s\n", modenames[static_cast(tp->mode)]); diff --git a/src/exe/main.cc b/src/exe/main.cc index 828b0f766..3f827c76b 100644 --- a/src/exe/main.cc +++ b/src/exe/main.cc @@ -356,6 +356,11 @@ static int start_profiler_internal(DDProfContext *ctx, bool &is_profiler) { static_cast(event_it->ring_buffer_type); reply.allocation_profiling_rate = ctx->watchers[alloc_watcher_idx].sample_period; + + if (ctx->watchers[alloc_watcher_idx].output_mode == + EventConfMode::kLiveCallgraph) { + reply.allocation_flags |= (1 << ddprof::ReplyMessage::kLiveCallgraph); + } } } diff --git a/src/ipc.cc b/src/ipc.cc index 10425bbcb..5bf0199b2 100644 --- a/src/ipc.cc +++ b/src/ipc.cc @@ -24,6 +24,7 @@ struct InternalResponseMessage { int64_t mem_size; int64_t allocation_profiling_rate; int32_t ring_buffer_type; + int32_t allocation_flags; }; struct timeval to_timeval(std::chrono::microseconds duration) noexcept { @@ -257,7 +258,8 @@ DDRes send(UnixSocket &socket, const ReplyMessage &msg) { .pid = msg.pid, .mem_size = msg.ring_buffer.mem_size, .allocation_profiling_rate = msg.allocation_profiling_rate, - .ring_buffer_type = msg.ring_buffer.ring_buffer_type}; + .ring_buffer_type = msg.ring_buffer.ring_buffer_type, + .allocation_flags = msg.allocation_flags}; socket.send(to_byte_span(&data), fd_span, ec); DDRES_CHECK_ERRORCODE(ec, DD_WHAT_SOCKET, "Unable to send response message"); return {}; @@ -291,6 +293,7 @@ DDRes receive(UnixSocket &socket, ReplyMessage &msg) { msg.ring_buffer.ring_buffer_type = data.ring_buffer_type; msg.ring_buffer.ring_fd = fds[0]; msg.ring_buffer.event_fd = fds[1]; + msg.allocation_flags = data.allocation_flags; return {}; } diff --git a/src/lib/allocation_tracker.cc b/src/lib/allocation_tracker.cc index ea4532f5d..3a9ee70e2 100644 --- a/src/lib/allocation_tracker.cc +++ b/src/lib/allocation_tracker.cc @@ -5,9 +5,11 @@ #include "allocation_tracker.hpp" +#include "ddprof_perf_event.hpp" #include "ddres.hpp" #include "defer.hpp" #include "ipc.hpp" +#include "live_allocation-c.hpp" #include "perf.hpp" #include "pevent_lib.hpp" #include "ringbuffer_utils.hpp" @@ -27,10 +29,10 @@ namespace ddprof { struct AllocationEvent { perf_event_header hdr; struct sample_id sample_id; + uint64_t addr; /* if PERF_SAMPLE_ADDR */ uint64_t period; - uint64_t abi; /* if PERF_SAMPLE_REGS_USER */ - uint64_t regs[PERF_REGS_COUNT]; - /* if PERF_SAMPLE_REGS_USER */ + uint64_t abi; /* if PERF_SAMPLE_REGS_USER */ + uint64_t regs[PERF_REGS_COUNT]; /* if PERF_SAMPLE_REGS_USER */ uint64_t size; /* if PERF_SAMPLE_STACK_USER */ std::byte data[PERF_SAMPLE_STACK_SIZE]; /* if PERF_SAMPLE_STACK_USER */ uint64_t dyn_size; /* if PERF_SAMPLE_STACK_USER && @@ -96,8 +98,8 @@ DDRes AllocationTracker::allocation_tracking_init( DDRES_CHECK_FWD(instance->init(allocation_profiling_rate, flags & kDeterministicSampling, ring_buffer)); _instance = instance; - state.track_allocations = true; - state.track_deallocations = flags & kTrackDeallocations; + + state.init(true, flags & kTrackDeallocations); return {}; } @@ -139,7 +141,21 @@ void AllocationTracker::allocation_tracking_free() { instance->free(); } -void AllocationTracker::track_allocation(uintptr_t, size_t size, +void AllocationTracker::free_on_consecutive_failures(bool success) { + if (!success) { + ++_state.failure_count; + if (_state.failure_count >= k_max_consecutive_failures) { + // Too many errors during ring buffer operation: stop allocation profiling + free(); + } + } else { + if (_state.failure_count.load(std::memory_order_relaxed) > 0) { + _state.failure_count = 0; + } + } +} + +void AllocationTracker::track_allocation(uintptr_t addr, size_t size, TrackerThreadLocalState &tl_state) { // Prevent reentrancy to avoid dead lock on mutex ReentryGuard guard(&tl_state.reentry_guard); @@ -184,19 +200,48 @@ void AllocationTracker::track_allocation(uintptr_t, size_t size, tl_state.remaining_bytes = remaining_bytes; uint64_t total_size = nsamples * sampling_interval; - if (!IsDDResOK(push_sample(total_size, tl_state))) { - ++_state.failure_count; - if (_state.failure_count >= k_max_consecutive_failures) { - // Too many errors during ring buffer operation: stop allocation profiling - free(); - } - } else { - if (_state.failure_count.load(std::memory_order_relaxed) > 0) { - _state.failure_count = 0; + bool success = IsDDResOK(push_alloc_sample(addr, total_size, tl_state)); + free_on_consecutive_failures(success); + + if (success && _state.track_deallocations) { + // ensure we track this dealloc if it occurs + _address_set.insert(addr); + if (unlikely(_address_set.size() > ddprof::liveallocation::kMaxTracked)) { + if (IsDDResOK(push_clear_live_allocation(tl_state))) { + _address_set.clear(); + } else { + fprintf( + stderr, + "Stop allocation profiling. Unable to clear live allocation \n"); + free(); + } } } } +void AllocationTracker::track_deallocation(uintptr_t addr, + TrackerThreadLocalState &tl_state) { + // Prevent reentrancy to avoid dead lock on mutex + ReentryGuard guard(&tl_state.reentry_guard); + + if (!guard) { + // This is an internal dealloc, so we don't need to keep track of this + return; + } + std::lock_guard lock{_state.mutex}; + + // recheck if profiling is enabled + if (!_state.track_deallocations) { + return; + } + + // Inserting / Erasing addresses is done within the lock + if (_address_set.erase(addr)) { + bool success = IsDDResOK(push_dealloc_sample(addr, tl_state)); + free_on_consecutive_failures(success); + } +} + DDRes AllocationTracker::push_lost_sample(MPSCRingBufferWriter &writer, bool ¬ify_needed) { auto lost_count = _state.lost_count.exchange(0, std::memory_order_acq_rel); @@ -225,8 +270,103 @@ DDRes AllocationTracker::push_lost_sample(MPSCRingBufferWriter &writer, return {}; } -DDRes AllocationTracker::push_sample(uint64_t allocated_size, - TrackerThreadLocalState &tl_state) { +// Return true if consumer should be notified +DDRes AllocationTracker::push_clear_live_allocation( + TrackerThreadLocalState &tl_state) { + MPSCRingBufferWriter writer{_pevent.rb}; + bool timeout = false; + + auto buffer = writer.reserve(sizeof(ClearLiveAllocationEvent), &timeout); + if (buffer.empty()) { + // unable to push a clear is an error (we don't want to grow too much) + // No use pushing a lost event. As this is a sync mechanism. + DDRES_RETURN_ERROR_LOG(DD_WHAT_PERFRB, + "Unable to get write lock on ring buffer"); + } + + ClearLiveAllocationEvent *event = + reinterpret_cast(buffer.data()); + event->hdr.misc = 0; + event->hdr.size = sizeof(ClearLiveAllocationEvent); + event->hdr.type = PERF_CUSTOM_EVENT_CLEAR_LIVE_ALLOCATION; + event->sample_id.time = 0; + if (_state.pid == 0) { + _state.pid = getpid(); + } + if (tl_state.tid == 0) { + tl_state.tid = ddprof::gettid(); + } + event->sample_id.pid = _state.pid; + event->sample_id.tid = tl_state.tid; + + if (writer.commit(buffer)) { + uint64_t count = 1; + if (write(_pevent.fd, &count, sizeof(count)) != sizeof(count)) { + DDRES_RETURN_ERROR_LOG(DD_WHAT_PERFRB, + "Error writing to memory allocation eventfd (%s)", + strerror(errno)); + } + } + + return {}; +} + +DDRes AllocationTracker::push_dealloc_sample( + uintptr_t addr, TrackerThreadLocalState &tl_state) { + MPSCRingBufferWriter writer{_pevent.rb}; + bool notify_consumer{false}; + + bool timeout = false; + if (unlikely(_state.lost_count.load(std::memory_order_relaxed))) { + DDRES_CHECK_FWD(push_lost_sample(writer, notify_consumer)); + } + + auto buffer = writer.reserve(sizeof(DeallocationEvent), &timeout); + if (buffer.empty()) { + // ring buffer is full, increase lost count + _state.lost_count.fetch_add(1, std::memory_order_acq_rel); + + if (timeout) { + DDRES_RETURN_ERROR_LOG(DD_WHAT_PERFRB, + "Unable to get write lock on ring buffer"); + } + // not an error + return {}; + } + + DeallocationEvent *event = + reinterpret_cast(buffer.data()); + event->hdr.misc = 0; + event->hdr.size = sizeof(DeallocationEvent); + event->hdr.type = PERF_CUSTOM_EVENT_DEALLOCATION; + event->sample_id.time = 0; + + if (_state.pid == 0) { + _state.pid = getpid(); + } + if (tl_state.tid == 0) { + tl_state.tid = ddprof::gettid(); + } + event->sample_id.pid = _state.pid; + event->sample_id.tid = tl_state.tid; + + // address of dealloc + event->ptr = addr; + + if (writer.commit(buffer) || notify_consumer) { + uint64_t count = 1; + if (write(_pevent.fd, &count, sizeof(count)) != sizeof(count)) { + DDRES_RETURN_ERROR_LOG(DD_WHAT_PERFRB, + "Error writing to memory allocation eventfd (%s)", + strerror(errno)); + } + } + return {}; +} + +DDRes AllocationTracker::push_alloc_sample(uintptr_t addr, + uint64_t allocated_size, + TrackerThreadLocalState &tl_state) { MPSCRingBufferWriter writer{_pevent.rb}; bool notify_consumer{false}; @@ -255,7 +395,7 @@ DDRes AllocationTracker::push_sample(uint64_t allocated_size, event->hdr.type = PERF_RECORD_SAMPLE; event->abi = PERF_SAMPLE_REGS_ABI_64; event->sample_id.time = 0; - + event->addr = addr; if (_state.pid == 0) { _state.pid = getpid(); } diff --git a/src/lib/dd_profiling.cc b/src/lib/dd_profiling.cc index cb5493444..a38dbe0ee 100644 --- a/src/lib/dd_profiling.cc +++ b/src/lib/dd_profiling.cc @@ -249,6 +249,12 @@ int ddprof_start_profiling_internal() { flags |= ddprof::AllocationTracker::kDeterministicSampling; info.allocation_profiling_rate = -info.allocation_profiling_rate; } + + if (info.allocation_flags & (1 << ddprof::ReplyMessage::kLiveCallgraph)) { + // tracking deallocations to allow a live view + flags |= ddprof::AllocationTracker::kTrackDeallocations; + } + if (IsDDResOK(ddprof::AllocationTracker::allocation_tracking_init( info.allocation_profiling_rate, flags, info.ring_buffer))) { // \fixme{nsavoire} pthread_create should probably be overridden diff --git a/src/lib/symbol_overrides.cc b/src/lib/symbol_overrides.cc index c30d3b478..991a6ac92 100644 --- a/src/lib/symbol_overrides.cc +++ b/src/lib/symbol_overrides.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #if defined(__GNUC__) && !defined(__clang__) @@ -43,6 +44,27 @@ timer_t g_timerid; int g_timer_sig = -1; int g_nb_loaded_libraries = -1; +// \fixme{nsavoire} The goal of this flag is to avoid double-counting +// mmaps that done inside malloc, not sure if it desirable or not. +// We should probably merge this TL state with AllocationTracker::tl_state +// to have a single TL state since accessing it is costly (call to tls_get_addr) +thread_local bool g_in_allocator_guard = false; + +class Guard { +public: + explicit Guard(bool *guard) : _guard(guard), _ok(!*guard) { *_guard = true; } + ~Guard() { + if (_ok) { + *_guard = false; + } + } + explicit operator bool() const { return _ok; } + +private: + bool *_guard; + bool _ok; +}; + DDPROF_NOINLINE bool loaded_libraries_have_changed() { int nb = ddprof::count_loaded_libraries(); if (nb != g_nb_loaded_libraries) { @@ -70,6 +92,7 @@ struct malloc { static void *hook(size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size); @@ -101,6 +124,7 @@ struct calloc { static void *hook(size_t nmemb, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(nmemb, size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size * nmemb); @@ -116,6 +140,7 @@ struct realloc { static void *hook(void *ptr, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); if (likely(ptr)) { ddprof::AllocationTracker::track_deallocation( reinterpret_cast(ptr)); @@ -138,6 +163,7 @@ struct posix_memalign { static int hook(void **memptr, size_t alignment, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ret = ref(memptr, alignment, size); if (likely(!ret)) { ddprof::AllocationTracker::track_allocation( @@ -154,6 +180,7 @@ struct aligned_alloc { static void *hook(size_t alignment, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(alignment, size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size); @@ -169,6 +196,7 @@ struct memalign { static void *hook(size_t alignment, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(alignment, size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size); @@ -184,6 +212,7 @@ struct pvalloc { static void *hook(size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size); @@ -199,6 +228,7 @@ struct valloc { static void *hook(size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); auto ptr = ref(size); ddprof::AllocationTracker::track_allocation( reinterpret_cast(ptr), size); @@ -214,6 +244,7 @@ struct reallocarray { static void *hook(void *ptr, size_t nmemb, size_t size) noexcept { check_libraries(); + Guard guard(&g_in_allocator_guard); if (ptr) { ddprof::AllocationTracker::track_deallocation( reinterpret_cast(ptr)); @@ -277,6 +308,88 @@ struct pthread_create { } }; +struct mmap { + static constexpr auto name = "mmap"; + static inline auto ref = &::mmap; + static inline bool ref_checked = false; + + static void *hook(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) noexcept { + void *ptr = ref(addr, length, prot, flags, fd, offset); + + if (addr == nullptr && fd == -1 && ptr != nullptr && + !g_in_allocator_guard) { + ddprof::AllocationTracker::track_allocation( + reinterpret_cast(ptr), length); + } + return ptr; + } +}; + +struct mmap_ { + static constexpr auto name = "__mmap"; + static inline auto ref = &::mmap; + static inline bool ref_checked = false; + + static void *hook(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) noexcept { + void *ptr = ref(addr, length, prot, flags, fd, offset); + + if (addr == nullptr && fd == -1 && ptr != nullptr && + !g_in_allocator_guard) { + ddprof::AllocationTracker::track_allocation( + reinterpret_cast(ptr), length); + } + return ptr; + } +}; + +struct mmap64_ { + static constexpr auto name = "mmap64"; + static inline auto ref = &::mmap64; + static inline bool ref_checked = false; + + static void *hook(void *addr, size_t length, int prot, int flags, int fd, + off_t offset) noexcept { + void *ptr = ref(addr, length, prot, flags, fd, offset); + + if (addr == nullptr && fd == -1 && ptr != nullptr && + !g_in_allocator_guard) { + ddprof::AllocationTracker::track_allocation( + reinterpret_cast(ptr), length); + } + return ptr; + } +}; + +struct munmap { + static constexpr auto name = "munmap"; + static inline auto ref = &::munmap; + static inline bool ref_checked = false; + + static int hook(void *addr, size_t length) noexcept { + if (!g_in_allocator_guard) { + ddprof::AllocationTracker::track_deallocation( + reinterpret_cast(addr)); + } + return ref(addr, length); + } +}; + +struct munmap_ { + static constexpr auto name = "__munmap"; + static inline auto ref = &::munmap; + static inline bool ref_checked = false; + + static int hook(void *addr, size_t length) noexcept { + if (!g_in_allocator_guard) { + ddprof::AllocationTracker::track_deallocation( + reinterpret_cast(addr)); + } + return ref(addr, length); + } +}; + template void install_hook(bool restore) { // On ubuntu 16, some symbols might be bound to @plt symbols // in exe and since we override the symbols in the exe, this would cause @@ -375,6 +488,12 @@ void setup_hooks(bool restore) { install_hook(restore); install_hook(restore); + install_hook(restore); + install_hook(restore); + install_hook(restore); + install_hook(restore); + install_hook(restore); + if (reallocarray::ref) { install_hook(restore); } diff --git a/src/live_allocation.cc b/src/live_allocation.cc new file mode 100644 index 000000000..b2822b9ba --- /dev/null +++ b/src/live_allocation.cc @@ -0,0 +1,86 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#include "live_allocation.hpp" + +#include "logger.hpp" + +namespace ddprof { + +void LiveAllocation::register_deallocation(uintptr_t address, + PprofStacks &stacks, + AddressMap &address_map) { + // Find the ValuePerAddress object corresponding to the address + auto map_iter = address_map.find(address); + if (map_iter == address_map.end()) { + // No element found, nothing to do + // This means we lost previous events, leading to de-sync between + // the state of the profiler and the state of the library. + LG_DBG("Unmatched de-allocation at %lx", address); + return; + } + ValuePerAddress &v = map_iter->second; + + // Decrement count and value of the corresponding PprofStacks::value_type + // object + if (v._unique_stack) { + v._unique_stack->second._value -= v._value; + if (v._unique_stack->second._count) { + --(v._unique_stack->second._count); + } + if (!v._unique_stack->second._count) { + // If count reaches 0, remove the UnwindOutput from stacks + stacks.erase(v._unique_stack->first); + } + } + + // Remove the element from the address map + address_map.erase(map_iter); +} + +void LiveAllocation::register_allocation(const UnwindOutput &uo, + uintptr_t address, int64_t value, + PprofStacks &stacks, + AddressMap &address_map) { + if (!uo.locs.size()) { + // avoid sending empty stacks + LG_DBG("(LIVE_ALLOC) Avoid registering empty stack"); + return; + } + // Find or create the PprofStacks::value_type object corresponding to the + // UnwindOutput + auto iter = stacks.find(uo); + if (iter == stacks.end()) { + iter = stacks.emplace(uo, ValueAndCount{}).first; + } + PprofStacks::value_type &unique_stack = *iter; + + // Add the value to the address map + ValuePerAddress &v = address_map[address]; + if (v._value) { + // unexpected, we already have an allocation here + // This means we missed a previous free + LG_DBG("Existing allocation: %lx (cleaning up)", address); + if (v._unique_stack) { + // we should decrement count / value + v._unique_stack->second._value -= v._value; + if (v._unique_stack->second._count) { + --(v._unique_stack->second._count); + } + // Should we erase the element here ? + // only if we are sure it is not the same as the one we are inserting. + if (v._unique_stack != &unique_stack && !v._unique_stack->second._count) { + stacks.erase(v._unique_stack->first); + } + } + } + + v._value = value; + v._unique_stack = &unique_stack; + v._unique_stack->second._value += value; + ++(v._unique_stack->second._count); +} + +} // namespace ddprof diff --git a/src/perf_watcher.cc b/src/perf_watcher.cc index cad7562bc..715036325 100644 --- a/src/perf_watcher.cc +++ b/src/perf_watcher.cc @@ -5,6 +5,7 @@ #include "perf_watcher.hpp" +#include "logger.hpp" #include "perf.hpp" #include @@ -94,6 +95,7 @@ const PerfWatcher *tracepoint_default_watcher() { .type = PERF_TYPE_TRACEPOINT, .sample_period = 1, .sample_type_id = DDPROF_PWT_TRACEPOINT, + .sample_stack_size = PERF_SAMPLE_STACK_SIZE, .options = {.use_kernel = PerfWatcherUseKernel::kRequired}, .value_scale = 1.0, }; @@ -103,3 +105,38 @@ const PerfWatcher *tracepoint_default_watcher() { bool watcher_has_tracepoint(const PerfWatcher *watcher) { return DDPROF_PWT_TRACEPOINT == watcher->sample_type_id; } + +void log_watcher(const PerfWatcher *w, int idx) { + PRINT_NFO(" ID: %s, Pos: %d, Index: %lu", w->desc.c_str(), idx, w->config); + switch (w->value_source) { + case EventConfValueSource::kSample: + PRINT_NFO(" Location: Sample"); + break; + case EventConfValueSource::kRegister: + PRINT_NFO(" Location: Register, regno: %d", w->regno); + break; + case EventConfValueSource::kRaw: + PRINT_NFO(" Location: Raw event, offset: %d, size: %d", w->raw_off, + w->raw_sz); + break; + default: + PRINT_NFO(" ILLEGAL LOCATION"); + break; + } + + PRINT_NFO(" Category: %s, EventName: %s, GroupName: %s, Label: %s", + sample_type_name_from_idx(w->sample_type_id), + w->tracepoint_event.c_str(), w->tracepoint_group.c_str(), + w->tracepoint_label.c_str()); + + if (w->options.is_freq) + PRINT_NFO(" Cadence: Freq, Freq: %lu", w->sample_frequency); + else + PRINT_NFO(" Cadence: Period, Period: %lu", w->sample_period); + if (Any(EventConfMode::kCallgraph & w->output_mode)) + PRINT_NFO(" Outputting to callgraph (flamegraph)"); + if (Any(EventConfMode::kMetric & w->output_mode)) + PRINT_NFO(" Outputting to metric"); + if (Any(EventConfMode::kLiveCallgraph & w->output_mode)) + PRINT_NFO(" Outputting to live callgraph"); +} diff --git a/src/pevent_lib.cc b/src/pevent_lib.cc index 07997aff5..b164c3742 100644 --- a/src/pevent_lib.cc +++ b/src/pevent_lib.cc @@ -5,12 +5,14 @@ #include "pevent_lib.hpp" +#include "ddprof_cmdline.hpp" #include "ddres.hpp" #include "defer.hpp" #include "perf.hpp" #include "ringbuffer_utils.hpp" #include "sys_utils.hpp" #include "syscalls.hpp" +#include "tracepoint_config.hpp" #include "user_override.hpp" #include @@ -131,7 +133,8 @@ DDRes pevent_open(DDProfContext *ctx, pid_t pid, int num_cpu, PEventHdr *pevent_hdr) { assert(pevent_hdr->size == 0); // check for previous init for (int watcher_idx = 0; watcher_idx < ctx->num_watchers; ++watcher_idx) { - if (ctx->watchers[watcher_idx].type < kDDPROF_TYPE_CUSTOM) { + PerfWatcher *watcher = &ctx->watchers[watcher_idx]; + if (watcher->type < kDDPROF_TYPE_CUSTOM) { DDRES_CHECK_FWD(pevent_open_all_cpus( &ctx->watchers[watcher_idx], watcher_idx, pid, num_cpu, pevent_hdr)); } else { @@ -201,6 +204,23 @@ DDRes pevent_setup(DDProfContext *ctx, pid_t pid, int num_cpu, LG_NTC("Retrying attachment without user override"); DDRES_CHECK_FWD(pevent_mmap(pevent_hdr, false)); } + + // If any watchers have self-instrumentation, then they may have set up child + // fds which now need to be consolidated via ioctl. These fds cannot be + // closed until profiling is completed. + for (unsigned i = 0; i < pevent_hdr->size; i++) { + PEvent *pes = &pevent_hdr->pes[i]; + if (ctx->watchers[pes->watcher_pos].instrument_self) { + int fd = pes->fd; + for (int j = 0; j < pes->current_child_fd; ++j) { + int child_fd = pes->child_fds[j]; + if (ioctl(child_fd, PERF_EVENT_IOC_SET_OUTPUT, fd)) { + DDRES_RETURN_ERROR_LOG(DD_WHAT_PERFOPEN, + "Could not ioctl() tALLOCSYS1"); + } + } + } + } return ddres_init(); } diff --git a/src/pprof/ddprof_pprof.cc b/src/pprof/ddprof_pprof.cc index d2306f3eb..995dd5cc1 100644 --- a/src/pprof/ddprof_pprof.cc +++ b/src/pprof/ddprof_pprof.cc @@ -169,12 +169,12 @@ static void write_line(const ddprof::Symbol &symbol, ddog_prof_Line *ffi_line) { // Assumption of API is that sample is valid in a single type DDRes pprof_aggregate(const UnwindOutput *uw_output, - const SymbolHdr *symbol_hdr, uint64_t value, + const SymbolHdr &symbol_hdr, uint64_t value, uint64_t count, const PerfWatcher *watcher, DDProfPProf *pprof) { - const ddprof::SymbolTable &symbol_table = symbol_hdr->_symbol_table; - const ddprof::MapInfoTable &mapinfo_table = symbol_hdr->_mapinfo_table; + const ddprof::SymbolTable &symbol_table = symbol_hdr._symbol_table; + const ddprof::MapInfoTable &mapinfo_table = symbol_hdr._mapinfo_table; ddog_prof_Profile *profile = pprof->_profile; int64_t values[DDPROF_PWT_LENGTH] = {}; @@ -187,7 +187,7 @@ DDRes pprof_aggregate(const UnwindOutput *uw_output, // assumption of single line per loc for now ddog_prof_Line line_buff[DD_MAX_STACK_DEPTH]; - ddprof::span locs{uw_output->locs, uw_output->nb_locs}; + ddprof::span locs{uw_output->locs}; if (watcher->options.nb_frames_to_skip < locs.size()) { locs = locs.subspan(watcher->options.nb_frames_to_skip); @@ -275,7 +275,7 @@ void ddprof_print_sample(const UnwindOutput &uw_output, const PerfWatcher &watcher) { auto &symbol_table = symbol_hdr._symbol_table; - ddprof::span locs{uw_output.locs, uw_output.nb_locs}; + ddprof::span locs{uw_output.locs}; const char *sample_name = sample_type_name_from_idx( sample_type_id_to_count_sample_type_id(watcher.sample_type_id)); diff --git a/src/tracepoint_config.cc b/src/tracepoint_config.cc new file mode 100644 index 000000000..209c9bb2d --- /dev/null +++ b/src/tracepoint_config.cc @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#include +#include +#include +#include + +namespace ddprof { +// Returns the ID of the given Linux tracepoint, or -1 if an error occurs. +int64_t tracepoint_get_id(std::string_view global_name, + std::string_view tracepoint_name) { + if (global_name.empty() || tracepoint_name.empty()) { + return -1; + } + + // todo: should we even consider the debug path ? (is it not deprecated?) + std::string fs_path; + struct stat sb; + if (stat("/sys/kernel/tracing/events", &sb) == 0) { + fs_path = "/sys/kernel/tracing/events"; + } else if (stat("/sys/kernel/debug/tracing/events", &sb) == 0) { + fs_path = "/sys/kernel/debug/tracing/events"; + } else { + return -1; // Neither debugfs nor tracefs is available. + } + + // todo this path could change (from user overrides) + std::stringstream id_path; + id_path << fs_path << "/" << global_name << "/" << tracepoint_name << "/id"; + + // Read the ID from the file. + std::ifstream id_file(id_path.str()); + if (!id_file) { + return -1; + } + long trace_id; + if (!(id_file >> trace_id)) { + return -1; + } + return trace_id; +} +} // namespace ddprof diff --git a/src/unwind.cc b/src/unwind.cc index 2fd058bca..3f4c42031 100644 --- a/src/unwind.cc +++ b/src/unwind.cc @@ -36,7 +36,7 @@ static void find_dso_add_error_frame(UnwindState *us) { void unwind_init_sample(UnwindState *us, uint64_t *sample_regs, pid_t sample_pid, uint64_t sample_size_stack, char *sample_data_stack) { - uw_output_clear(&us->output); + us->output.clear(); memcpy(&us->initial_regs.regs[0], sample_regs, K_NB_REGS_UNWIND * sizeof(uint64_t)); us->current_ip = us->initial_regs.regs[REGNAME(PC)]; @@ -56,11 +56,11 @@ static bool is_stack_complete(UnwindState *us) { static constexpr std::array s_expected_root_frames{"_start"sv, "__clone"sv, "_exit"sv}; - if (us->output.nb_locs == 0) { + if (us->output.locs.size() == 0) { return false; } - const auto &root_loc = us->output.locs[us->output.nb_locs - 1]; + const auto &root_loc = us->output.locs.back(); const auto &root_mapping = us->symbol_hdr._mapinfo_table[root_loc._map_info_idx]; @@ -96,15 +96,11 @@ DDRes unwindstate__unwind(UnwindState *us) { } else { us->output.is_incomplete = false; } - ddprof_stats_add(STATS_UNWIND_AVG_STACK_DEPTH, us->output.nb_locs, nullptr); + ddprof_stats_add(STATS_UNWIND_AVG_STACK_DEPTH, us->output.locs.size(), + nullptr); // Add a frame that identifies executable to which these belong add_virtual_base_frame(us); - if (us->_dwfl_wrapper->_inconsistent) { - // error detected on this pid - LG_WRN("(Inconsistent DWFL/DSOs)%d - Free associated objects", us->pid); - unwind_pid_free(us, us->pid); - } return res; } @@ -119,7 +115,6 @@ void unwind_cycle(UnwindState *us) { us->symbol_hdr.cycle(); // clean up pids that we did not see recently us->dwfl_hdr.display_stats(); - us->dwfl_hdr.clear_unvisited(); us->dso_hdr._stats.reset(); unwind_metrics_reset(); diff --git a/src/unwind_dwfl.cc b/src/unwind_dwfl.cc index 4a176194a..12a61c852 100644 --- a/src/unwind_dwfl.cc +++ b/src/unwind_dwfl.cc @@ -77,7 +77,8 @@ static void trace_unwinding_end(UnwindState *us) { if (LL_DEBUG <= LOG_getlevel()) { DsoHdr::DsoFindRes find_res = us->dso_hdr.dso_find_closest(us->pid, us->current_ip); - SymbolIdx_t symIdx = us->output.locs[us->output.nb_locs - 1]._symbol_idx; + SymbolIdx_t symIdx = + us->output.locs[us->output.locs.size() - 1]._symbol_idx; if (find_res.second) { const std::string &last_func = us->symbol_hdr._symbol_table[symIdx]._symname; @@ -102,7 +103,7 @@ static DDRes add_runtime_symbol_frame(UnwindState *us, const Dso &dso, static DDRes add_symbol(Dwfl_Frame *dwfl_frame, UnwindState *us) { if (is_max_stack_depth_reached(*us)) { add_common_frame(us, SymbolErrors::truncated_stack); - LG_DBG("Max stack depth reached (depth#%lu)", us->output.nb_locs); + LG_DBG("Max stack depth reached (depth#%lu)", us->output.locs.size()); ddprof_stats_add(STATS_UNWIND_TRUNCATED_OUTPUT, 1, nullptr); return ddres_warn(DD_WHAT_UW_MAX_DEPTH); } @@ -110,7 +111,7 @@ static DDRes add_symbol(Dwfl_Frame *dwfl_frame, UnwindState *us) { Dwarf_Addr pc = 0; if (!dwfl_frame_pc(dwfl_frame, &pc, nullptr)) { LG_DBG("Failure to compute frame PC: %s (depth#%lu)", dwfl_errmsg(-1), - us->output.nb_locs); + us->output.locs.size()); add_error_frame(nullptr, us, pc, SymbolErrors::dwfl_frame); return ddres_init(); // invalid pc : do not add frame } @@ -127,7 +128,7 @@ static DDRes add_symbol(Dwfl_Frame *dwfl_frame, UnwindState *us) { if (!find_res.second) { // no matching file was found LG_DBG("[UW] (PID%d) DSO not found at 0x%lx (depth#%lu)", us->pid, pc, - us->output.nb_locs); + us->output.locs.size()); add_error_frame(nullptr, us, pc, SymbolErrors::unknown_dso); return ddres_init(); } @@ -173,7 +174,7 @@ static DDRes add_symbol(Dwfl_Frame *dwfl_frame, UnwindState *us) { if (!dwfl_frame_pc(dwfl_frame, &pc, &isactivation)) { LG_DBG("Failure to compute frame PC: %s (depth#%lu)", dwfl_errmsg(-1), - us->output.nb_locs); + us->output.locs.size()); add_error_frame(nullptr, us, pc, SymbolErrors::dwfl_frame); return ddres_init(); // invalid pc : do not add frame } @@ -190,7 +191,7 @@ static DDRes add_symbol(Dwfl_Frame *dwfl_frame, UnwindState *us) { bool is_infinite_loop(UnwindState *us) { UnwindOutput &output = us->output; - uint64_t nb_locs = output.nb_locs; + uint64_t nb_locs = output.locs.size(); unsigned nb_frames_to_check = 3; if (nb_locs <= nb_frames_to_check) { return false; @@ -209,7 +210,7 @@ bool is_infinite_loop(UnwindState *us) { static int frame_cb(Dwfl_Frame *dwfl_frame, void *arg) { UnwindState *us = (UnwindState *)arg; #ifdef DEBUG - LG_NFO("Beging depth %lu", us->output.nb_locs); + LG_NFO("Beging depth %lu", us->output.locs.size()); #endif int dwfl_error_value = dwfl_errno(); if (dwfl_error_value) { @@ -222,8 +223,9 @@ static int frame_cb(Dwfl_Frame *dwfl_frame, void *arg) { #ifdef DEBUG // We often fallback to frame pointer unwinding (which logs an error) if (dwfl_error_value) { - LG_DBG("Error flagged at depth = %lu -- %d Error:%s ", us->output.nb_locs, - dwfl_error_value, dwfl_errmsg(dwfl_error_value)); + LG_DBG("Error flagged at depth = %lu -- %d Error:%s ", + us->output.locs.size(), dwfl_error_value, + dwfl_errmsg(dwfl_error_value)); } #endif // Before we potentially exit, record the fact that we're processing a frame @@ -248,8 +250,8 @@ DDRes unwind_dwfl(UnwindState *us) { 0) { trace_unwinding_end(us); } - res = us->output.nb_locs > 0 ? ddres_init() - : ddres_warn(DD_WHAT_DWFL_LIB_ERROR); + res = us->output.locs.size() > 0 ? ddres_init() + : ddres_warn(DD_WHAT_DWFL_LIB_ERROR); return res; } diff --git a/src/unwind_helpers.cc b/src/unwind_helpers.cc index 1f90679b3..270b6ad03 100644 --- a/src/unwind_helpers.cc +++ b/src/unwind_helpers.cc @@ -15,35 +15,33 @@ namespace ddprof { bool is_max_stack_depth_reached(const UnwindState &us) { // +2 to keep room for common base frame - return us.output.nb_locs + 2 >= DD_MAX_STACK_DEPTH; + return us.output.locs.size() + 2 >= DD_MAX_STACK_DEPTH; } DDRes add_frame(SymbolIdx_t symbol_idx, MapInfoIdx_t map_idx, ElfAddress_t pc, UnwindState *us) { UnwindOutput *output = &us->output; - int64_t current_loc_idx = output->nb_locs; - if (output->nb_locs >= DD_MAX_STACK_DEPTH) { + if (output->locs.size() >= DD_MAX_STACK_DEPTH) { DDRES_RETURN_WARN_LOG(DD_WHAT_UW_MAX_DEPTH, "Max stack depth reached"); // avoid overflow } - - output->locs[current_loc_idx]._symbol_idx = symbol_idx; - output->locs[current_loc_idx].ip = pc; + FunLoc current; + current._symbol_idx = symbol_idx; + current.ip = pc; if (map_idx == -1) { // just add an empty element for mapping info - output->locs[current_loc_idx]._map_info_idx = - us->symbol_hdr._common_mapinfo_lookup.get_or_insert( - CommonMapInfoLookup::MappingErrors::empty, - us->symbol_hdr._mapinfo_table); + current._map_info_idx = us->symbol_hdr._common_mapinfo_lookup.get_or_insert( + CommonMapInfoLookup::MappingErrors::empty, + us->symbol_hdr._mapinfo_table); } else { - output->locs[current_loc_idx]._map_info_idx = map_idx; + current._map_info_idx = map_idx; } #ifdef DEBUG LG_NTC("Considering frame with IP : %lx / %s ", pc, - us->symbol_hdr._symbol_table[output->locs[current_loc_idx]._symbol_idx] - ._symname.c_str()); + us->symbol_hdr._symbol_table[current._symbol_idx]._symname.c_str()); #endif - output->nb_locs++; + output->locs.push_back(current); + return ddres_init(); } @@ -178,7 +176,7 @@ bool memory_read(ProcessAddress_t addr, ElfWord_t *result, int regno, // requested when unwinding the leaf function for a register, we simply // return the initial register value. constexpr uint64_t k_red_zone_size = 128; - if (us->output.nb_locs <= 1 && regno != -1 && + if (us->output.locs.size() <= 1 && regno != -1 && addr >= sp_start - k_red_zone_size) { *result = us->initial_regs.regs[regno]; return true; @@ -232,6 +230,6 @@ void add_error_frame(const Dso *dso, UnwindState *us, } else { add_common_frame(us, error_case); } - LG_DBG("Error frame (depth#%lu)", us->output.nb_locs); + LG_DBG("Error frame (depth#%lu)", us->output.locs.size()); } } // namespace ddprof diff --git a/src/unwind_output.cc b/src/unwind_output.cc deleted file mode 100644 index 34a927dbb..000000000 --- a/src/unwind_output.cc +++ /dev/null @@ -1,19 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. This product includes software -// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present -// Datadog, Inc. - -#include -#include - -#include "unwind_output.hpp" - -static void FunLoc_clear(FunLoc *locs) { - memset(locs, 0, sizeof(*locs) * DD_MAX_STACK_DEPTH); -} - -void uw_output_clear(UnwindOutput *output) { - FunLoc_clear(output->locs); - output->nb_locs = 0; - output->is_incomplete = true; -} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 200091049..4b35fe2f5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -87,8 +87,8 @@ endfunction() # Definition of unit tests add_compile_definitions("UNIT_TEST_DATA=\"${CMAKE_CURRENT_SOURCE_DIR}/data\"") -add_unit_test(ddprofcmdline-ut ../src/ddprof_cmdline.cc ../src/perf_watcher.cc ddprofcmdline-ut.cc - LIBRARIES DDProf::Parser) +add_unit_test(ddprofcmdline-ut ../src/ddprof_cmdline.cc ../src/perf_watcher.cc + ../src/tracepoint_config.cc ddprofcmdline-ut.cc LIBRARIES DDProf::Parser) add_unit_test(logger-ut logger-ut.cc) @@ -131,15 +131,19 @@ add_unit_test( ../src/ddprof_cpumask.cc ../src/logger_setup.cc ../src/perf_watcher.cc + ../src/tracepoint_config.cc ddprof_input-ut.cc LIBRARIES DDProf::Parser DEFINITIONS MYNAME="ddprof_input-ut") -add_unit_test(perf_ringbuffer-ut ../src/perf.cc ../src/perf_watcher.cc ../src/perf_ringbuffer.cc - perf_ringbuffer-ut.cc DEFINITIONS MYNAME="perf_ringbuffer-ut") +add_unit_test( + perf_ringbuffer-ut ../src/perf.cc ../src/perf_watcher.cc ../src/perf_ringbuffer.cc + ../src/tracepoint_config.cc perf_ringbuffer-ut.cc DEFINITIONS MYNAME="perf_ringbuffer-ut") add_unit_test( pevent-ut + ../src/ddprof_cmdline.cc + ../src/tracepoint_config.cc ../src/pevent_lib.cc ../src/user_override.cc ../src/perf.cc @@ -148,11 +152,11 @@ add_unit_test( ../src/ringbuffer_utils.cc ../src/sys_utils.cc pevent-ut.cc + LIBRARIES DDProf::Parser DEFINITIONS MYNAME="pevent-ut") add_unit_test( - ddprof_pprof-ut ../src/pprof/ddprof_pprof.cc ../src/unwind_output.cc ../src/perf_watcher.cc - ddprof_pprof-ut.cc + ddprof_pprof-ut ../src/pprof/ddprof_pprof.cc ../src/perf_watcher.cc ddprof_pprof-ut.cc LIBRARIES Datadog::Profiling DDProf::Parser DEFINITIONS MYNAME="ddprof_pprof-ut") @@ -160,8 +164,8 @@ add_unit_test( ddprof_exporter-ut ../src/exporter/ddprof_exporter.cc ../src/ddprof_cmdline.cc + ../src/tracepoint_config.cc ../src/pprof/ddprof_pprof.cc - ../src/unwind_output.cc ../src/perf_watcher.cc ../src/tags.cc ddprof_exporter-ut.cc @@ -240,7 +244,6 @@ add_unit_test( ../src/unwind_dwfl.cc ../src/unwind_helpers.cc ../src/unwind_metrics.cc - ../src/unwind_output.cc ../src/user_override.cc LIBRARIES ${ELFUTILS_LIBRARIES} llvm-demangle DEFINITIONS MYNAME="savecontext-ut") @@ -253,6 +256,7 @@ add_unit_test( ../src/build_id.cc ../src/common_mapinfo_lookup.cc ../src/common_symbol_lookup.cc + ../src/ddprof_cmdline.cc ../src/ddprof_file_info.cc ../src/ddprof_stats.cc ../src/dso.cc @@ -280,26 +284,30 @@ add_unit_test( ../src/signal_helper.cc ../src/statsd.cc ../src/sys_utils.cc + ../src/tracepoint_config.cc ../src/user_override.cc ../src/unwind.cc ../src/unwind_dwfl.cc ../src/unwind_helpers.cc ../src/unwind_metrics.cc - ../src/unwind_output.cc - LIBRARIES ${ELFUTILS_LIBRARIES} llvm-demangle - DEFINITIONS ${DDPROF_DEFINITION_LIST}) + LIBRARIES ${ELFUTILS_LIBRARIES} llvm-demangle DDProf::Parser + DEFINITIONS ${DDPROF_DEFINITION_LIST} KMAX_TRACKED_ALLOCATIONS=100) add_unit_test(sys_utils-ut sys_utils-ut.cc ../src/sys_utils.cc) add_unit_test( ringbuffer-ut ringbuffer-ut.cc + ../src/ddprof_cmdline.cc + ../src/tracepoint_config.cc ../src/perf.cc ../src/perf_ringbuffer.cc + ../src/perf_watcher.cc ../src/pevent_lib.cc ../src/ringbuffer_utils.cc ../src/sys_utils.cc - ../src/user_override.cc) + ../src/user_override.cc + LIBRARIES DDProf::Parser) add_unit_test(timer-ut timer-ut.cc ../src/timer.cc ../src/perf.cc) @@ -316,6 +324,10 @@ add_unit_test(build_id-ut build_id-ut.cc ../src/build_id.cc) add_unit_test(jitdump-ut jitdump-ut.cc ../src/jit/jitdump.cc) +add_unit_test(tracepoint_config-ut tracepoint_config-ut.cc ../src/tracepoint_config.cc) + +add_unit_test(live_allocation-ut live_allocation-ut.cc ../src/live_allocation.cc) + add_benchmark(savecontext-bench savecontext-bench.cc ../src/lib/savecontext.cc ../src/lib/saveregisters.cc) diff --git a/test/allocation_tracker-ut.cc b/test/allocation_tracker-ut.cc index e8307486f..ba646ccc8 100644 --- a/test/allocation_tracker-ut.cc +++ b/test/allocation_tracker-ut.cc @@ -5,7 +5,10 @@ #include "allocation_tracker.hpp" #include "ddprof_base.hpp" +#include "ddprof_perf_event.hpp" #include "ipc.hpp" +#include "live_allocation-c.hpp" +#include "loghandle.hpp" #include "perf_watcher.hpp" #include "pevent_lib.hpp" #include "ringbuffer_holder.hpp" @@ -18,8 +21,14 @@ #include #include -DDPROF_NOINLINE void my_malloc(size_t size) { - ddprof::AllocationTracker::track_allocation(0xdeadbeef, size); +DDPROF_NOINLINE void my_malloc(size_t size, uintptr_t addr = 0xdeadbeef) { + ddprof::AllocationTracker::track_allocation(addr, size); + // prevent tail call optimization + getpid(); +} + +DDPROF_NOINLINE void my_free(uintptr_t addr) { + ddprof::AllocationTracker::track_deallocation(addr); // prevent tail call optimization getpid(); } @@ -38,49 +47,76 @@ TEST(allocation_tracker, start_stop) { ddprof::RingBufferHolder ring_buffer{buf_size_order, RingBufferType::kMPSCRingBuffer}; ddprof::AllocationTracker::allocation_tracking_init( - rate, ddprof::AllocationTracker::kDeterministicSampling, + rate, + ddprof::AllocationTracker::kDeterministicSampling | + ddprof::AllocationTracker::kTrackDeallocations, ring_buffer.get_buffer_info()); ASSERT_TRUE(ddprof::AllocationTracker::is_active()); my_func_calling_malloc(1); - - ddprof::MPSCRingBufferReader reader{ring_buffer.get_ring_buffer()}; - ASSERT_GT(reader.available_size(), 0); - - auto buf = reader.read_sample(); - ASSERT_FALSE(buf.empty()); - const perf_event_header *hdr = - reinterpret_cast(buf.data()); - ASSERT_EQ(hdr->type, PERF_RECORD_SAMPLE); - - perf_event_sample *sample = hdr2samp(hdr, perf_event_default_sample_type()); - - ASSERT_EQ(sample->period, 1); - ASSERT_EQ(sample->pid, getpid()); - ASSERT_EQ(sample->tid, ddprof::gettid()); - - UnwindState state; - ddprof::unwind_init_sample(&state, sample->regs, sample->pid, - sample->size_stack, sample->data_stack); - ddprof::unwindstate__unwind(&state); - - const auto &symbol_table = state.symbol_hdr._symbol_table; - ASSERT_GT(state.output.nb_locs, NB_FRAMES_TO_SKIP); - const auto &symbol = - symbol_table[state.output.locs[NB_FRAMES_TO_SKIP]._symbol_idx]; - ASSERT_EQ(symbol._symname, "my_func_calling_malloc"); - + { // check that we get the relevant info for this allocation + ddprof::MPSCRingBufferReader reader{ring_buffer.get_ring_buffer()}; + ASSERT_GT(reader.available_size(), 0); + + auto buf = reader.read_sample(); + ASSERT_FALSE(buf.empty()); + const perf_event_header *hdr = + reinterpret_cast(buf.data()); + ASSERT_EQ(hdr->type, PERF_RECORD_SAMPLE); + + perf_event_sample *sample = + hdr2samp(hdr, perf_event_default_sample_type() | PERF_SAMPLE_ADDR); + + ASSERT_EQ(sample->period, 1); + ASSERT_EQ(sample->pid, getpid()); + ASSERT_EQ(sample->tid, ddprof::gettid()); + ASSERT_EQ(sample->addr, 0xdeadbeef); + + UnwindState state; + ddprof::unwind_init_sample(&state, sample->regs, sample->pid, + sample->size_stack, sample->data_stack); + ddprof::unwindstate__unwind(&state); + + const auto &symbol_table = state.symbol_hdr._symbol_table; + ASSERT_GT(state.output.locs.size(), NB_FRAMES_TO_SKIP); + const auto &symbol = + symbol_table[state.output.locs[NB_FRAMES_TO_SKIP]._symbol_idx]; + ASSERT_EQ(symbol._symname, "my_func_calling_malloc"); + } + my_free(0xdeadbeef); + // ensure we get a deallocation event + { + ddprof::MPSCRingBufferReader reader{ring_buffer.get_ring_buffer()}; + ASSERT_GT(reader.available_size(), 0); + + auto buf = reader.read_sample(); + ASSERT_FALSE(buf.empty()); + const perf_event_header *hdr = + reinterpret_cast(buf.data()); + ASSERT_EQ(hdr->type, PERF_CUSTOM_EVENT_DEALLOCATION); + const ddprof::DeallocationEvent *sample = + reinterpret_cast(hdr); + ASSERT_EQ(sample->ptr, 0xdeadbeef); + } + my_free(0xcafebabe); + { + ddprof::MPSCRingBufferReader reader{ring_buffer.get_ring_buffer()}; + ASSERT_EQ(reader.available_size(), 0); + } ddprof::AllocationTracker::allocation_tracking_free(); ASSERT_FALSE(ddprof::AllocationTracker::is_active()); } TEST(allocation_tracker, stale_lock) { + LogHandle log_handle; const uint64_t rate = 1; const size_t buf_size_order = 5; ddprof::RingBufferHolder ring_buffer{buf_size_order, RingBufferType::kMPSCRingBuffer}; ddprof::AllocationTracker::allocation_tracking_init( - rate, ddprof::AllocationTracker::kDeterministicSampling, + rate, + ddprof::AllocationTracker::kDeterministicSampling | + ddprof::AllocationTracker::kTrackDeallocations, ring_buffer.get_buffer_info()); // simulate stale lock @@ -92,4 +128,58 @@ TEST(allocation_tracker, stale_lock) { } ASSERT_FALSE(ddprof::AllocationTracker::is_active()); ddprof::AllocationTracker::allocation_tracking_free(); -} \ No newline at end of file +} + +TEST(allocation_tracker, max_tracked_allocs) { + const uint64_t rate = 1; + const size_t buf_size_order = 9; + ddprof::RingBufferHolder ring_buffer{buf_size_order, + RingBufferType::kMPSCRingBuffer}; + ddprof::AllocationTracker::allocation_tracking_init( + rate, + ddprof::AllocationTracker::kDeterministicSampling | + ddprof::AllocationTracker::kTrackDeallocations, + ring_buffer.get_buffer_info()); + + ASSERT_TRUE(ddprof::AllocationTracker::is_active()); + + for (int i = 0; i <= ddprof::liveallocation::kMaxTracked + 1; ++i) { + my_malloc(1, 0x1000 + i); + ddprof::MPSCRingBufferReader reader{ring_buffer.get_ring_buffer()}; + if (i <= + ddprof::liveallocation::kMaxTracked) { // check that we get the relevant + // info for this allocation + ASSERT_GT(reader.available_size(), 0); + auto buf = reader.read_sample(); + ASSERT_FALSE(buf.empty()); + const perf_event_header *hdr = + reinterpret_cast(buf.data()); + ASSERT_EQ(hdr->type, PERF_RECORD_SAMPLE); + + perf_event_sample *sample = + hdr2samp(hdr, perf_event_default_sample_type() | PERF_SAMPLE_ADDR); + + ASSERT_EQ(sample->period, 1); + ASSERT_EQ(sample->pid, getpid()); + ASSERT_EQ(sample->tid, ddprof::gettid()); + ASSERT_EQ(sample->addr, 0x1000 + i); + } else { + bool clear_found = false; + int nb_read = 0; + ddprof::ConstBuffer buf; + do { + buf = reader.read_sample(); + ++nb_read; + if (buf.empty()) + break; + const perf_event_header *hdr = + reinterpret_cast(buf.data()); + if (hdr->type == PERF_CUSTOM_EVENT_CLEAR_LIVE_ALLOCATION) { + clear_found = true; + } + } while (true); + EXPECT_EQ(nb_read, 3); + EXPECT_TRUE(clear_found); + } + } +} diff --git a/test/ddprof_exporter-ut.cc b/test/ddprof_exporter-ut.cc index 3e3e94d71..9c12177a4 100644 --- a/test/ddprof_exporter-ut.cc +++ b/test/ddprof_exporter-ut.cc @@ -160,7 +160,7 @@ TEST(DDProfExporter, simple) { res = pprof_create_profile(&pprofs, &ctx); EXPECT_TRUE(IsDDResOK(res)); - res = pprof_aggregate(&mock_output, &symbol_hdr, 1000, 1, &ctx.watchers[0], + res = pprof_aggregate(&mock_output, symbol_hdr, 1000, 1, &ctx.watchers[0], &pprofs); EXPECT_TRUE(IsDDResOK(res)); } diff --git a/test/ddprof_input-ut.cc b/test/ddprof_input-ut.cc index 5f64b5254..0b6f6637e 100644 --- a/test/ddprof_input-ut.cc +++ b/test/ddprof_input-ut.cc @@ -6,6 +6,7 @@ #include "ddprof_input.hpp" #include "constants.hpp" +#include "ddprof_cmdline.hpp" #include "ddprof_context.hpp" #include "ddprof_context_lib.hpp" #include "defer.hpp" @@ -30,6 +31,15 @@ bool s_version_called = false; void print_version() { s_version_called = true; } string_view str_version() { return STRING_VIEW_LITERAL("1.2.3"); } +TEST_F(InputTest, watcher_from_str) { + LogHandle handle; + const char *str_event = "sALLOC mode=l"; + PerfWatcher watcher; + bool ret = watcher_from_str(str_event, &watcher); + ASSERT_TRUE(ret); + log_watcher(&watcher, 0); +} + TEST_F(InputTest, default_values) { DDProfInput input; DDRes res = ddprof_input_default(&input); @@ -229,6 +239,7 @@ TEST_F(InputTest, duplicate_events) { } TEST_F(InputTest, presets) { + LogHandle handle; { // Default preset should be CPU + ALLOC DDProfInput input; @@ -262,6 +273,7 @@ TEST_F(InputTest, presets) { ddprof_input_free(&input); ddprof_context_free(&ctx); } + { // Default preset for PID mode should be CPU DDProfInput input; @@ -329,6 +341,55 @@ TEST_F(InputTest, presets) { ddprof_input_free(&input); ddprof_context_free(&ctx); } + { + // Check manual setting of live allocation + DDProfInput input; + bool contine_exec = true; + const char *input_values[] = {MYNAME, "-e", "sALLOC mode=l", "my_program"}; + DDRes res = ddprof_input_parse( + std::size(input_values), (char **)input_values, &input, &contine_exec); + + EXPECT_TRUE(IsDDResOK(res)); + EXPECT_TRUE(contine_exec); + + DDProfContext ctx; + res = ddprof_context_set(&input, &ctx); + EXPECT_TRUE(IsDDResOK(res)); + + EXPECT_EQ(ctx.num_watchers, 2); + EXPECT_EQ(ctx.watchers[1].ddprof_event_type, DDPROF_PWE_sALLOC); + EXPECT_EQ(ctx.watchers[1].output_mode, EventConfMode::kLiveCallgraph); + log_watcher(&ctx.watchers[0], 0); + log_watcher(&ctx.watchers[1], 1); + + ddprof_input_free(&input); + ddprof_context_free(&ctx); + } + { + // Check cpu_live_heap preset + DDProfInput input; + bool contine_exec = true; + const char *input_values[] = {MYNAME, "--preset", "cpu_live_heap", + "my_program"}; + DDRes res = ddprof_input_parse( + std::size(input_values), (char **)input_values, &input, &contine_exec); + + EXPECT_TRUE(IsDDResOK(res)); + EXPECT_TRUE(contine_exec); + + DDProfContext ctx; + res = ddprof_context_set(&input, &ctx); + EXPECT_TRUE(IsDDResOK(res)); + + EXPECT_EQ(ctx.num_watchers, 2); + EXPECT_EQ(ctx.watchers[1].ddprof_event_type, DDPROF_PWE_sALLOC); + EXPECT_EQ(ctx.watchers[1].output_mode, EventConfMode::kLiveCallgraph); + EXPECT_EQ(ctx.watchers[0].ddprof_event_type, DDPROF_PWE_sCPU); + EXPECT_EQ(ctx.watchers[0].output_mode, EventConfMode::kCallgraph); + + ddprof_input_free(&input); + ddprof_context_free(&ctx); + } { // Default preset should not be loaded if an event is given in input DDProfInput input; diff --git a/test/ddprof_pprof-ut.cc b/test/ddprof_pprof-ut.cc index 31f16b464..d8f5db010 100644 --- a/test/ddprof_pprof-ut.cc +++ b/test/ddprof_pprof-ut.cc @@ -74,7 +74,7 @@ TEST(DDProfPProf, aggregate) { ctx.num_watchers = 1; DDRes res = pprof_create_profile(&pprof, &ctx); EXPECT_TRUE(IsDDResOK(res)); - res = pprof_aggregate(&mock_output, &symbol_hdr, 1000, 1, &ctx.watchers[0], + res = pprof_aggregate(&mock_output, symbol_hdr, 1000, 1, &ctx.watchers[0], &pprof); EXPECT_TRUE(IsDDResOK(res)); diff --git a/test/ddprofcmdline-ut.cc b/test/ddprofcmdline-ut.cc index 1b7ae4b40..44e2afb13 100644 --- a/test/ddprofcmdline-ut.cc +++ b/test/ddprofcmdline-ut.cc @@ -157,43 +157,75 @@ TEST(CmdLineTst, ParserKeyPatterns) { // Mode is permissive ASSERT_TRUE(watcher_from_str("e=hCPU mode=magnanimous", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & EventConfMode::kCallgraph)); + ASSERT_TRUE(Any(watcher.output_mode & EventConfMode::kMetric)); // A or a designate all ASSERT_TRUE(watcher_from_str("e=hCPU mode=A", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & EventConfMode::kCallgraph)); + ASSERT_TRUE(Any(watcher.output_mode & EventConfMode::kMetric)); ASSERT_TRUE(watcher_from_str("e=hCPU mode=a", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_TRUE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric // both m and g together designate all ASSERT_TRUE(watcher_from_str("e=hCPU mode=MG", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_TRUE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric ASSERT_TRUE(watcher_from_str("e=hCPU mode=mg", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_TRUE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric // M or m is a metric (no callgraph unless specified) ASSERT_TRUE(watcher_from_str("e=hCPU mode=M", &watcher)); - ASSERT_FALSE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_FALSE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_TRUE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric ASSERT_TRUE(watcher_from_str("e=hCPU mode=m", &watcher)); - ASSERT_FALSE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_FALSE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_TRUE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric // G or g designate callgraph (default) ASSERT_TRUE(watcher_from_str("e=hCPU", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_FALSE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_FALSE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric ASSERT_TRUE(watcher_from_str("e=hCPU mode=G", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_FALSE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_FALSE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric ASSERT_TRUE(watcher_from_str("e=hCPU mode=g", &watcher)); - ASSERT_TRUE(watcher.output_mode <= EventConfMode::kCallgraph); - ASSERT_FALSE(watcher.output_mode <= EventConfMode::kMetric); + ASSERT_TRUE(Any(watcher.output_mode & + EventConfMode::kCallgraph)); // watcher.output_mode <= + // EventConfMode::kCallgraph + ASSERT_FALSE(Any( + watcher.output_mode & + EventConfMode::kMetric)); // watcher.output_mode <= EventConfMode::kMetric // n|arg_num|argno ASSERT_TRUE(watcher_from_str("e=hCPU n=1", &watcher)); diff --git a/test/live_allocation-ut.cc b/test/live_allocation-ut.cc new file mode 100644 index 000000000..2137fbd28 --- /dev/null +++ b/test/live_allocation-ut.cc @@ -0,0 +1,131 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +#include "live_allocation.hpp" +#include "loghandle.hpp" + +#include + +namespace ddprof { + +TEST(LiveAllocationTest, simple) { + LogHandle handle; + UnwindOutput uo; + uo.pid = 123; + uo.tid = 456; + uo.is_incomplete = false; + uo.locs.push_back({0x1234, 0x5678, 0x9abc}); + uo.locs.push_back({0x4321, 0x8765, 0xcba9}); + + LiveAllocation live_alloc; + int watcher_pos = 0; + pid_t pid = 12; + int64_t value = 10; + int64_t nb_registered_allocs = 10; + { // allocate 10 + uintptr_t addr = 0x10; + for (int i = 0; i < nb_registered_allocs; ++i) { + live_alloc.register_allocation(uo, addr, value, watcher_pos, pid); + addr += 0x10; + } + } + // Check that the hash values are equal + // EXPECT_EQ(hash_value, expected_hash_value); + auto &pid_map = live_alloc._watcher_vector[0]; + EXPECT_EQ(pid_map.size(), 1); + auto &pid_stacks = pid_map[pid]; + // all allocations are registerd + EXPECT_EQ(pid_stacks._address_map.size(), nb_registered_allocs); + // though the stack is the same + ASSERT_EQ(pid_stacks._unique_stacks.size(), 1); + const auto &el = pid_stacks._unique_stacks[uo]; + EXPECT_EQ(el._value, 100); + + { // allocate 10 + uintptr_t addr = 0x10; + for (int i = 0; i < nb_registered_allocs; ++i) { + live_alloc.register_deallocation(addr, watcher_pos, pid); + addr += 0x10; + } + } + // all allocations are de-registerd + EXPECT_EQ(pid_stacks._address_map.size(), 0); + // though the stack is the same + EXPECT_EQ(pid_stacks._unique_stacks.size(), 0); +} + +TEST(LiveAllocationTest, invalid_inputs) { + LiveAllocation live_alloc; + int watcher_pos = 0; + pid_t pid = 12; + int64_t value = 10; + UnwindOutput uo; + + // Register allocation with empty UnwindOutput + uintptr_t addr = 0x10; + EXPECT_NO_THROW( + live_alloc.register_allocation(uo, addr, value, watcher_pos, pid)); + auto &pid_map = live_alloc._watcher_vector[0]; + auto &pid_stacks = pid_map[pid]; + // for now we don't consider them + EXPECT_EQ(pid_stacks._address_map.size(), 0); + EXPECT_EQ(pid_stacks._unique_stacks.size(), 0); + + // Register allocation with negative value + uo.pid = 123; + uo.tid = 456; + uo.is_incomplete = false; + uo.locs.push_back({0x1234, 0x5678, 0x9abc}); + // We will register them (though probably cause a UI bug...) + EXPECT_NO_THROW( + live_alloc.register_allocation(uo, addr, -1, watcher_pos, pid)); + // Register deallocation with invalid address + EXPECT_NO_THROW(live_alloc.register_deallocation(0, watcher_pos, pid)); +} + +TEST(LiveAllocationTest, overlap_registrations) { + LogHandle handle; + LiveAllocation live_alloc; + int watcher_pos = 0; + pid_t pid = 12; + int64_t value = 10; + UnwindOutput uo; + + uintptr_t addr = 0x10; + uo.pid = 123; + uo.tid = 456; + uo.is_incomplete = false; + uo.locs.push_back({0x1234, 0x5678, 0x9abc}); + + // Register the first allocation + live_alloc.register_allocation(uo, addr, value, watcher_pos, pid); + auto &pid_map = live_alloc._watcher_vector[0]; + auto &pid_stacks = pid_map[pid]; + EXPECT_EQ(pid_stacks._address_map.size(), 1); + EXPECT_EQ(pid_stacks._unique_stacks.size(), 1); + + // Register a second allocation at the same address + // elements can arrive out of order, so this can be expected + live_alloc.register_allocation(uo, addr, value * 2, watcher_pos, pid); + EXPECT_EQ(pid_stacks._address_map.size(), 1); + EXPECT_EQ(pid_stacks._unique_stacks.size(), 1); + + // Check that the value and count have the latest value + auto &el = pid_stacks._unique_stacks[uo]; + EXPECT_EQ(el._value, value * 2); + EXPECT_EQ(el._count, 1); + + // Deallocate the first allocation + live_alloc.register_deallocation(addr, watcher_pos, pid); + EXPECT_EQ(pid_stacks._address_map.size(), 0); + EXPECT_EQ(pid_stacks._unique_stacks.size(), 0); + + // Deallocate the second allocation + live_alloc.register_deallocation(addr, watcher_pos, pid); + EXPECT_EQ(pid_stacks._address_map.size(), 0); + EXPECT_EQ(pid_stacks._unique_stacks.size(), 0); +} + +} // namespace ddprof diff --git a/test/savecontext-ut.cc b/test/savecontext-ut.cc index 48cc7ae21..9efbb30f7 100644 --- a/test/savecontext-ut.cc +++ b/test/savecontext-ut.cc @@ -35,12 +35,12 @@ void funcB() { auto &symbol_table = state.symbol_hdr._symbol_table; - for (size_t iloc = 0; iloc < state.output.nb_locs; ++iloc) { + for (size_t iloc = 0; iloc < state.output.locs.size(); ++iloc) { auto &symbol = symbol_table[state.output.locs[iloc]._symbol_idx]; printf("%zu: %s\n", iloc, symbol._demangle_name.c_str()); } - EXPECT_GT(state.output.nb_locs, 3); + EXPECT_GT(state.output.locs.size(), 3); auto &symbol0 = symbol_table[state.output.locs[0]._symbol_idx]; EXPECT_TRUE(symbol0._demangle_name.starts_with("save_context(")); auto &symbol1 = symbol_table[state.output.locs[1]._symbol_idx]; @@ -104,7 +104,7 @@ TEST(getcontext, unwind_from_sighandler) { auto &symbol_table = state.symbol_hdr._symbol_table; - for (size_t iloc = 0; iloc < state.output.nb_locs; ++iloc) { + for (size_t iloc = 0; iloc < state.output.locs.size(); ++iloc) { auto &symbol = symbol_table[state.output.locs[iloc]._symbol_idx]; printf("%zu: %s %lx \n", iloc, symbol._demangle_name.c_str(), state.output.locs[iloc].ip); @@ -113,12 +113,12 @@ TEST(getcontext, unwind_from_sighandler) { return symbol_table[state.output.locs[idx]._symbol_idx]; }; - EXPECT_GT(state.output.nb_locs, 5); - EXPECT_LT(state.output.nb_locs, 20); + EXPECT_GT(state.output.locs.size(), 5); + EXPECT_LT(state.output.locs.size(), 25); EXPECT_TRUE(get_symbol(0)._demangle_name.starts_with("save_context(")); EXPECT_EQ(get_symbol(1)._demangle_name, "handler(int)"); size_t next_idx = 3; - while (next_idx < state.output.nb_locs - 1 && + while (next_idx < state.output.locs.size() - 1 && get_symbol(next_idx)._demangle_name != "funcD()") { ++next_idx; } diff --git a/test/self_unwind/self_unwind.cc b/test/self_unwind/self_unwind.cc index 575cfd62b..9c990915d 100644 --- a/test/self_unwind/self_unwind.cc +++ b/test/self_unwind/self_unwind.cc @@ -118,7 +118,7 @@ bool stack_addtomap(const UnwindOutput *unwind_output, const DDProfContext *ctx, assert(callback_ctx); suw::SymbolMap *symbol_map = reinterpret_cast(callback_ctx); assert(perf_option_pos == 0); - for (unsigned i = 0; i < unwind_output->nb_locs; ++i) { + for (unsigned i = 0; i < unwind_output->locs.size(); ++i) { const ddprof::Symbol &symbol = ddprof::get_symbol(ctx, unwind_output, i); if (symbol._demangle_name.find("0x") != std::string::npos) { // skip non symbolized frames diff --git a/test/simple_malloc-ut.sh b/test/simple_malloc-ut.sh index 2919f1640..1cf68e4b6 100755 --- a/test/simple_malloc-ut.sh +++ b/test/simple_malloc-ut.sh @@ -93,6 +93,9 @@ check "./test/simple_malloc-shared --profile ${opts}" 1 # Test wrapper mode check "./ddprof ./test/simple_malloc ${opts}" 1 +# Test live heap mode +check "./ddprof --preset cpu_live_heap ./test/simple_malloc ${opts} --skip-free 100" 1 + # Test wrapper mode with forks + threads check "./ddprof ./test/simple_malloc ${opts} --fork 2 --threads 2" 2 4 diff --git a/test/simple_malloc.cc b/test/simple_malloc.cc index 4239180e6..65723bc9a 100644 --- a/test/simple_malloc.cc +++ b/test/simple_malloc.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,28 @@ # include "dd_profiling.h" #endif +#ifdef __GLIBC__ +# include +#endif + +/***************************** SIGSEGV Handler *******************************/ +static void sigsegv_handler(int sig, siginfo_t *si, void *uc) { + // TODO this really shouldn't call printf-family functions... + (void)uc; +#ifdef __GLIBC__ + static void *buf[4096] = {0}; + size_t sz = backtrace(buf, 4096); +#endif + fprintf(stderr, "simplemalloc[%d]:has encountered an error and will exit\n", + getpid()); + if (sig == SIGSEGV) + printf("Fault address: %p\n", si->si_addr); +#ifdef __GLIBC__ + backtrace_symbols_fd(buf, sz, STDERR_FILENO); +#endif + exit(-1); +} + struct thread_cpu_clock { using duration = std::chrono::nanoseconds; using rep = duration::rep; @@ -57,6 +80,7 @@ struct Options { std::chrono::milliseconds timeout_duration; uint32_t callstack_depth; uint32_t frame_size; + uint32_t skip_free; }; extern "C" DDPROF_NOINLINE void do_lot_of_allocations(const Options &options, @@ -67,6 +91,7 @@ extern "C" DDPROF_NOINLINE void do_lot_of_allocations(const Options &options, auto start_time = std::chrono::steady_clock::now(); auto deadline_time = start_time + options.timeout_duration; auto start_cpu = thread_cpu_clock::now(); + unsigned skip_free = 0; for (uint64_t i = 0; i < options.loop_count; ++i) { void *p = nullptr; if (options.malloc_size) { @@ -84,7 +109,12 @@ extern "C" DDPROF_NOINLINE void do_lot_of_allocations(const Options &options, p2 = p; } ddprof::DoNotOptimize(p2); - free(p2); + + if (skip_free++ >= options.skip_free) { + free(p2); + skip_free = 0; + } + if (options.sleep_duration_per_loop.count()) { std::this_thread::sleep_for(options.sleep_duration_per_loop); } @@ -142,6 +172,11 @@ void print_stats(pid_t pid, const Stats &stats) { } int main(int argc, char *argv[]) { + struct sigaction sigaction_handlers = {}; + sigaction_handlers.sa_sigaction = sigsegv_handler; + sigaction_handlers.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &(sigaction_handlers), NULL); + try { CLI::App app{"Simple allocation test"}; @@ -170,6 +205,9 @@ int main(int argc, char *argv[]) { app.add_option("--frame-size", opts.frame_size, "Size to allocate on the stack for each frame") ->default_val(0); + app.add_option("--skip-free", opts.skip_free, + "Only free every N allocations (default is 0)") + ->default_val(0); app.add_option( "--timeout", opts.timeout_duration, "Timeout after N milliseconds") diff --git a/test/tracepoint_config-ut.cc b/test/tracepoint_config-ut.cc new file mode 100644 index 000000000..074fb391c --- /dev/null +++ b/test/tracepoint_config-ut.cc @@ -0,0 +1,20 @@ +#include + +#include "loghandle.hpp" +#include "tracepoint_config.hpp" + +namespace ddprof { + +TEST(tracepoint_config, getid) { + LogHandle handle; + int64_t id = tracepoint_get_id("raw_syscalls", "sys_exit"); + // This can fail without the appropriate permissions + LG_DBG("Tracepoint: raw_syscall/sys_exit id=%ld ", id); +#if defined(__x86_64__) + if (id != -1) { + EXPECT_EQ(id, 348); + } +#endif +} + +} // namespace ddprof diff --git a/test/unwind_output_mock.hpp b/test/unwind_output_mock.hpp index af0c072c1..69742f504 100644 --- a/test/unwind_output_mock.hpp +++ b/test/unwind_output_mock.hpp @@ -43,11 +43,11 @@ static inline void fill_mapinfo_table_1(MapInfoTable &mapinfo_table) { } static inline void fill_unwind_output_1(UnwindOutput &uw_output) { - uw_output_clear(&uw_output); - uw_output.nb_locs = K_MOCK_LOC_SIZE; + uw_output.clear(); + uw_output.locs.resize(K_MOCK_LOC_SIZE); - FunLoc *locs = uw_output.locs; - for (unsigned i = 0; i < uw_output.nb_locs; ++i) { + std::vector &locs = uw_output.locs; + for (unsigned i = 0; i < uw_output.locs.size(); ++i) { locs[i].ip = 42 + i; locs[i]._symbol_idx = i; locs[i]._map_info_idx = i;