f3ac1f58a2
We now have a branch upstream for tracking el10 patches which makes this a lot easier to maintain. This updates to using that branch as a diff upon 47.1. Resolves: RHEL-65743
2788 lines
95 KiB
Diff
2788 lines
95 KiB
Diff
From 41248249b257bcb017465e3f7c70dcb19dbd121d Mon Sep 17 00:00:00 2001
|
|
From: Christian Hergert <chergert@redhat.com>
|
|
Date: Sun, 3 Nov 2024 10:41:44 -0800
|
|
Subject: [PATCH 16/31] sysprofd: add support for unwinding without frame
|
|
pointers
|
|
|
|
This provides a new sysprof-live-unwinder subprocess that runs as root to
|
|
allow accessing all processes on the system via /proc/$pid/. It is spawned
|
|
by sysprofd with various perf event FDs and a FD to write captures to.
|
|
|
|
Ideally the capture_fd is something that will naturally error if the client
|
|
application crashes (such as a socketpair() having the peer close). This
|
|
is not enforced but encouraged. Additionally, an event_fd is used to allow
|
|
the client application to signal the live-unwinder to exit.
|
|
|
|
Unwinding is performed by looking at the modules loaded into the target
|
|
pid and using libdwfl to access DWARF/CFI/etc state machinery. Stack data
|
|
does not touch the disk as it exists in a mmap buffer from perf and is
|
|
then translated into a callchain and sent to the Sysprof client.
|
|
|
|
Unwinding occurs as normal post-mortem though is improved through the use
|
|
of debuginfod to locate the appropriate symbols.
|
|
---
|
|
meson.build | 3 +-
|
|
src/meson.build | 3 +
|
|
src/sysprof-live-unwinder/main.c | 752 ++++++++++++++++++
|
|
src/sysprof-live-unwinder/meson.build | 19 +
|
|
.../sysprof-live-process.c | 505 ++++++++++++
|
|
.../sysprof-live-process.h | 50 ++
|
|
.../sysprof-live-unwinder.c | 427 ++++++++++
|
|
.../sysprof-live-unwinder.h | 79 ++
|
|
src/sysprof-live-unwinder/tests/meson.build | 35 +
|
|
.../tests/test-live-unwinder.c | 391 +++++++++
|
|
src/sysprofd/ipc-unwinder-impl.c | 247 ++++++
|
|
src/sysprofd/ipc-unwinder-impl.h | 34 +
|
|
src/sysprofd/meson.build | 10 +-
|
|
src/sysprofd/org.gnome.Sysprof3.Unwinder.xml | 23 +
|
|
src/sysprofd/sysprofd.c | 6 +-
|
|
15 files changed, 2580 insertions(+), 4 deletions(-)
|
|
create mode 100644 src/sysprof-live-unwinder/main.c
|
|
create mode 100644 src/sysprof-live-unwinder/meson.build
|
|
create mode 100644 src/sysprof-live-unwinder/sysprof-live-process.c
|
|
create mode 100644 src/sysprof-live-unwinder/sysprof-live-process.h
|
|
create mode 100644 src/sysprof-live-unwinder/sysprof-live-unwinder.c
|
|
create mode 100644 src/sysprof-live-unwinder/sysprof-live-unwinder.h
|
|
create mode 100644 src/sysprof-live-unwinder/tests/meson.build
|
|
create mode 100644 src/sysprof-live-unwinder/tests/test-live-unwinder.c
|
|
create mode 100644 src/sysprofd/ipc-unwinder-impl.c
|
|
create mode 100644 src/sysprofd/ipc-unwinder-impl.h
|
|
create mode 100644 src/sysprofd/org.gnome.Sysprof3.Unwinder.xml
|
|
|
|
diff --git a/meson.build b/meson.build
|
|
index bac8eae6..16d64e8a 100644
|
|
--- a/meson.build
|
|
+++ b/meson.build
|
|
@@ -39,12 +39,13 @@ need_glib = (need_gtk or
|
|
get_option('tools') or
|
|
get_option('tests'))
|
|
need_libsysprof = (need_gtk or
|
|
+ get_option('sysprofd') == 'bundled' or
|
|
get_option('libsysprof') or
|
|
get_option('examples') or
|
|
get_option('tools') or
|
|
get_option('tests'))
|
|
|
|
-dex_req = '0.6'
|
|
+dex_req = '0.9'
|
|
glib_req = '2.76.0'
|
|
gtk_req = '4.15'
|
|
polkit_req = '0.105'
|
|
diff --git a/src/meson.build b/src/meson.build
|
|
index f3e0c61d..9432755e 100644
|
|
--- a/src/meson.build
|
|
+++ b/src/meson.build
|
|
@@ -11,6 +11,8 @@ sysprof_version_conf.set('MINOR_VERSION', sysprof_version[1])
|
|
sysprof_version_conf.set('MICRO_VERSION', 0)
|
|
sysprof_version_conf.set('VERSION', meson.project_version())
|
|
|
|
+pkglibexecdir = join_paths(get_option('prefix'), get_option('libexecdir'))
|
|
+
|
|
subdir('libsysprof-capture')
|
|
|
|
if need_libsysprof
|
|
@@ -20,6 +22,7 @@ endif
|
|
|
|
if get_option('sysprofd') == 'bundled'
|
|
subdir('sysprofd')
|
|
+ subdir('sysprof-live-unwinder')
|
|
endif
|
|
|
|
if get_option('gtk')
|
|
diff --git a/src/sysprof-live-unwinder/main.c b/src/sysprof-live-unwinder/main.c
|
|
new file mode 100644
|
|
index 00000000..9e2733ae
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/main.c
|
|
@@ -0,0 +1,752 @@
|
|
+/*
|
|
+ * main.c
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#include "config.h"
|
|
+
|
|
+#include <stdatomic.h>
|
|
+
|
|
+#include <sys/mman.h>
|
|
+#include <sys/resource.h>
|
|
+
|
|
+#include <glib/gstdio.h>
|
|
+#include <glib-unix.h>
|
|
+
|
|
+#include <sysprof.h>
|
|
+
|
|
+#include "sysprof-live-unwinder.h"
|
|
+#include "sysprof-perf-event-stream-private.h"
|
|
+
|
|
+#define CAPTURE_BUFFER_SIZE (4096*16)
|
|
+#define N_PAGES 32
|
|
+
|
|
+#define DUMP_BYTES(_n, _b, _l) \
|
|
+ G_STMT_START { \
|
|
+ GString *str, *astr; \
|
|
+ gsize _i; \
|
|
+ g_log(G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, \
|
|
+ " %s = %p [%d]", #_n, _b, (gint)_l); \
|
|
+ str = g_string_sized_new (80); \
|
|
+ astr = g_string_sized_new (16); \
|
|
+ for (_i = 0; _i < _l; _i++) \
|
|
+ { \
|
|
+ if ((_i % 16) == 0) \
|
|
+ g_string_append_printf (str, "%06x: ", (guint)_i); \
|
|
+ g_string_append_printf (str, " %02x", _b[_i]); \
|
|
+ \
|
|
+ if (g_ascii_isprint(_b[_i])) \
|
|
+ g_string_append_printf (astr, " %c", _b[_i]); \
|
|
+ else \
|
|
+ g_string_append (astr, " ."); \
|
|
+ \
|
|
+ if ((_i % 16) == 15) \
|
|
+ { \
|
|
+ g_log (G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, \
|
|
+ "%s %s", str->str, astr->str); \
|
|
+ str->str[0] = str->len = 0; \
|
|
+ astr->str[0] = astr->len = 0; \
|
|
+ } \
|
|
+ else if ((_i % 16) == 7) \
|
|
+ { \
|
|
+ g_string_append (str, " "); \
|
|
+ g_string_append (astr, " "); \
|
|
+ } \
|
|
+ } \
|
|
+ \
|
|
+ if (_i != 16) \
|
|
+ g_log (G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, \
|
|
+ "%-56s %s", str->str, astr->str); \
|
|
+ \
|
|
+ g_string_free (str, TRUE); \
|
|
+ g_string_free (astr, TRUE); \
|
|
+ } G_STMT_END
|
|
+
|
|
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (SysprofCaptureWriter, sysprof_capture_writer_unref)
|
|
+
|
|
+typedef struct _PerfSource
|
|
+{
|
|
+ GSource gsource;
|
|
+
|
|
+ SysprofLiveUnwinder *unwinder;
|
|
+ SysprofCaptureWriter *writer;
|
|
+
|
|
+ guint64 map_size;
|
|
+ struct perf_event_mmap_page *map;
|
|
+ guint8 *map_data;
|
|
+ const guint8 *map_data_end;
|
|
+ guint64 map_data_size;
|
|
+ guint64 tail;
|
|
+
|
|
+ guint8 *buffer;
|
|
+
|
|
+ int cpu;
|
|
+ GPid self_pid;
|
|
+
|
|
+ guint stack_size;
|
|
+} PerfSource;
|
|
+
|
|
+typedef struct _PerfFDArg
|
|
+{
|
|
+ int fd;
|
|
+ int cpu;
|
|
+} PerfFDArg;
|
|
+
|
|
+SYSPROF_ALIGNED_BEGIN(8);
|
|
+typedef struct _StackRegs
|
|
+{
|
|
+ guint64 abi;
|
|
+ guint64 registers[0];
|
|
+} StackRegs
|
|
+SYSPROF_ALIGNED_END(8);
|
|
+
|
|
+SYSPROF_ALIGNED_BEGIN(8);
|
|
+typedef struct _StackUser
|
|
+{
|
|
+ guint64 size;
|
|
+ guint8 data[0];
|
|
+} StackUser
|
|
+SYSPROF_ALIGNED_END(8);
|
|
+
|
|
+static GArray *all_perf_fds;
|
|
+
|
|
+static inline void
|
|
+realign (gsize *pos,
|
|
+ gsize align)
|
|
+{
|
|
+ *pos = (*pos + align - 1) & ~(align - 1);
|
|
+}
|
|
+
|
|
+static void
|
|
+handle_event (PerfSource *source,
|
|
+ const SysprofPerfEvent *event)
|
|
+{
|
|
+ gsize offset;
|
|
+ gint64 time;
|
|
+
|
|
+ g_assert (source != NULL);
|
|
+ g_assert (event != NULL);
|
|
+
|
|
+ switch (event->header.type)
|
|
+ {
|
|
+ case PERF_RECORD_COMM:
|
|
+ offset = strlen (event->comm.comm) + 1;
|
|
+ realign (&offset, sizeof (guint64));
|
|
+ offset += sizeof (GPid) + sizeof (GPid);
|
|
+ memcpy (&time, event->comm.comm + offset, sizeof time);
|
|
+
|
|
+ if (event->comm.pid == event->comm.tid)
|
|
+ {
|
|
+ sysprof_live_unwinder_seen_process (source->unwinder,
|
|
+ time,
|
|
+ source->cpu,
|
|
+ event->comm.pid,
|
|
+ event->comm.comm);
|
|
+ }
|
|
+
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_EXIT:
|
|
+ /* Ignore fork exits for now */
|
|
+ if (event->exit.tid != event->exit.pid)
|
|
+ break;
|
|
+
|
|
+ sysprof_live_unwinder_process_exited (source->unwinder,
|
|
+ event->exit.time,
|
|
+ source->cpu,
|
|
+ event->exit.pid);
|
|
+
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_FORK:
|
|
+ sysprof_live_unwinder_process_forked (source->unwinder,
|
|
+ event->fork.time,
|
|
+ source->cpu,
|
|
+ event->fork.ptid,
|
|
+ event->fork.tid);
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_LOST:
|
|
+ {
|
|
+ char message[64];
|
|
+ g_snprintf (message, sizeof message,
|
|
+ "Lost %"G_GUINT64_FORMAT" samples",
|
|
+ event->lost.lost);
|
|
+ sysprof_capture_writer_add_log (source->writer,
|
|
+ SYSPROF_CAPTURE_CURRENT_TIME,
|
|
+ source->cpu,
|
|
+ -1,
|
|
+ G_LOG_LEVEL_CRITICAL, "Sampler", message);
|
|
+ break;
|
|
+ }
|
|
+
|
|
+ case PERF_RECORD_MMAP:
|
|
+ offset = strlen (event->mmap.filename) + 1;
|
|
+ realign (&offset, sizeof (guint64));
|
|
+ offset += sizeof (GPid) + sizeof (GPid);
|
|
+ memcpy (&time, event->mmap.filename + offset, sizeof time);
|
|
+
|
|
+ sysprof_live_unwinder_track_mmap (source->unwinder,
|
|
+ time,
|
|
+ source->cpu,
|
|
+ event->mmap.pid,
|
|
+ event->mmap.addr,
|
|
+ event->mmap.addr + event->mmap.len,
|
|
+ event->mmap.pgoff,
|
|
+ 0,
|
|
+ event->mmap.filename,
|
|
+ NULL);
|
|
+
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_MMAP2:
|
|
+ offset = strlen (event->mmap2.filename) + 1;
|
|
+ realign (&offset, sizeof (guint64));
|
|
+ offset += sizeof (GPid) + sizeof (GPid);
|
|
+ memcpy (&time, event->mmap2.filename + offset, sizeof time);
|
|
+
|
|
+ if ((event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) != 0)
|
|
+ {
|
|
+ char build_id[G_N_ELEMENTS (event->mmap2.build_id) * 2 + 1];
|
|
+ guint len = MIN (G_N_ELEMENTS (event->mmap2.build_id), event->mmap2.build_id_size);
|
|
+
|
|
+ for (guint i = 0; i < len; i++)
|
|
+ g_snprintf (&build_id[len*2], 3, "%02x", event->mmap2.build_id[i]);
|
|
+ build_id[len*2] = 0;
|
|
+
|
|
+ sysprof_live_unwinder_track_mmap (source->unwinder,
|
|
+ time,
|
|
+ source->cpu,
|
|
+ event->mmap2.pid,
|
|
+ event->mmap2.addr,
|
|
+ event->mmap2.addr + event->mmap2.len,
|
|
+ event->mmap2.pgoff,
|
|
+ event->mmap2.ino,
|
|
+ event->mmap2.filename,
|
|
+ build_id);
|
|
+ }
|
|
+ else
|
|
+ {
|
|
+ sysprof_live_unwinder_track_mmap (source->unwinder,
|
|
+ time,
|
|
+ source->cpu,
|
|
+ event->mmap2.pid,
|
|
+ event->mmap2.addr,
|
|
+ event->mmap2.addr + event->mmap2.len,
|
|
+ event->mmap2.pgoff,
|
|
+ event->mmap2.ino,
|
|
+ event->mmap2.filename,
|
|
+ NULL);
|
|
+ }
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_READ:
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_SAMPLE:
|
|
+ {
|
|
+ const guint8 *endptr = (const guint8 *)event + event->header.size;
|
|
+ const guint64 *ips = event->callchain.ips;
|
|
+ int n_ips = event->callchain.n_ips;
|
|
+ guint64 trace[3];
|
|
+
|
|
+ /* We always expect PERF_RECORD_SAMPLE to contain a callchain because
|
|
+ * we need that even if we sample the stack for user-space unwinding.
|
|
+ * Otherwise we lose the blended stack trace.
|
|
+ */
|
|
+ if (n_ips == 0)
|
|
+ {
|
|
+ if (event->callchain.header.misc & PERF_RECORD_MISC_KERNEL)
|
|
+ {
|
|
+ trace[0] = PERF_CONTEXT_KERNEL;
|
|
+ trace[1] = event->callchain.ip;
|
|
+ trace[2] = PERF_CONTEXT_USER;
|
|
+
|
|
+ ips = trace;
|
|
+ n_ips = 3;
|
|
+ }
|
|
+ else
|
|
+ {
|
|
+ trace[0] = PERF_CONTEXT_USER;
|
|
+ trace[1] = event->callchain.ip;
|
|
+
|
|
+ ips = trace;
|
|
+ n_ips = 2;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (source->stack_size && source->stack_size < event->header.size)
|
|
+ {
|
|
+ guint64 dyn_size = *((const guint64 *)endptr - 1);
|
|
+ const StackUser *stack_user = (const StackUser *)(endptr - sizeof (guint64) - source->stack_size - sizeof (StackUser));
|
|
+ const StackRegs *stack_regs = (const StackRegs *)&event->callchain.ips[event->callchain.n_ips];
|
|
+ guint n_regs = ((const guint8 *)stack_user - (const guint8 *)stack_regs->registers) / sizeof (guint64);
|
|
+
|
|
+#if 0
|
|
+ g_print ("n_ips=%u stack_size=%ld dyn_size=%ld abi=%ld n_regs=%d\n",
|
|
+ n_ips, stack_user->size, dyn_size, stack_regs->abi, n_regs);
|
|
+#endif
|
|
+
|
|
+ sysprof_live_unwinder_process_sampled_with_stack (source->unwinder,
|
|
+ event->callchain.time,
|
|
+ source->cpu,
|
|
+ event->callchain.pid,
|
|
+ event->callchain.tid,
|
|
+ ips,
|
|
+ n_ips,
|
|
+ stack_user->data,
|
|
+ stack_user->size,
|
|
+ dyn_size,
|
|
+ stack_regs->abi,
|
|
+ stack_regs->registers,
|
|
+ n_regs);
|
|
+
|
|
+ break;
|
|
+ }
|
|
+
|
|
+ sysprof_live_unwinder_process_sampled (source->unwinder,
|
|
+ event->callchain.time,
|
|
+ source->cpu,
|
|
+ event->callchain.pid,
|
|
+ event->callchain.tid,
|
|
+ ips,
|
|
+ n_ips);
|
|
+
|
|
+ break;
|
|
+ }
|
|
+
|
|
+ case PERF_RECORD_THROTTLE:
|
|
+ case PERF_RECORD_UNTHROTTLE:
|
|
+ default:
|
|
+ break;
|
|
+ }
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+perf_source_prepare (GSource *gsource,
|
|
+ int *timeout)
|
|
+{
|
|
+ *timeout = 50;
|
|
+ return FALSE;
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+perf_source_check (GSource *gsource)
|
|
+{
|
|
+ PerfSource *source = (PerfSource *)gsource;
|
|
+ guint64 head;
|
|
+ guint64 tail;
|
|
+
|
|
+ atomic_thread_fence (memory_order_acquire);
|
|
+
|
|
+ tail = source->tail;
|
|
+ head = source->map->data_head;
|
|
+
|
|
+ if (head < tail)
|
|
+ tail = head;
|
|
+
|
|
+ return (head - tail) >= sizeof (struct perf_event_header);
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+perf_source_dispatch (GSource *gsource,
|
|
+ GSourceFunc callback,
|
|
+ gpointer user_data)
|
|
+{
|
|
+ PerfSource *source = (PerfSource *)gsource;
|
|
+ guint64 n_bytes = source->map_data_size;
|
|
+ guint64 mask = n_bytes - 1;
|
|
+ guint64 head;
|
|
+ guint64 tail;
|
|
+ guint us = 0;
|
|
+ guint them = 0;
|
|
+
|
|
+ g_assert (source != NULL);
|
|
+
|
|
+ tail = source->tail;
|
|
+ head = source->map->data_head;
|
|
+
|
|
+ atomic_thread_fence (memory_order_acquire);
|
|
+
|
|
+ if (head < tail)
|
|
+ tail = head;
|
|
+
|
|
+ while ((head - tail) >= sizeof (struct perf_event_header))
|
|
+ {
|
|
+ const SysprofPerfEvent *event;
|
|
+ struct perf_event_header *header;
|
|
+ gboolean is_self = FALSE;
|
|
+
|
|
+ /* Note that:
|
|
+ *
|
|
+ * - perf events are a multiple of 64 bits
|
|
+ * - the perf event header is 64 bits
|
|
+ * - the data area is a multiple of 64 bits
|
|
+ *
|
|
+ * which means there will always be space for one header, which means we
|
|
+ * can safely dereference the size field.
|
|
+ */
|
|
+ header = (struct perf_event_header *)(gpointer)(source->map_data + (tail & mask));
|
|
+
|
|
+ if (header->size > head - tail)
|
|
+ {
|
|
+ /* The kernel did not generate a complete event.
|
|
+ * I don't think that can happen, but we may as well
|
|
+ * be paranoid.
|
|
+ */
|
|
+ g_warn_if_reached ();
|
|
+ break;
|
|
+ }
|
|
+
|
|
+ if (source->map_data + (tail & mask) + header->size > source->map_data_end)
|
|
+ {
|
|
+ guint8 *b = source->buffer;
|
|
+ gint n_before;
|
|
+ gint n_after;
|
|
+
|
|
+ n_after = (tail & mask) + header->size - n_bytes;
|
|
+ n_before = header->size - n_after;
|
|
+
|
|
+ memcpy (b, source->map_data + (tail & mask), n_before);
|
|
+ memcpy (b + n_before, source->map_data, n_after);
|
|
+
|
|
+ header = (struct perf_event_header *)(gpointer)b;
|
|
+ }
|
|
+
|
|
+ event = (SysprofPerfEvent *)header;
|
|
+
|
|
+ switch (event->header.type)
|
|
+ {
|
|
+ default:
|
|
+ case PERF_RECORD_COMM:
|
|
+ case PERF_RECORD_EXIT:
|
|
+ case PERF_RECORD_FORK:
|
|
+ case PERF_RECORD_MMAP:
|
|
+ case PERF_RECORD_MMAP2:
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_SAMPLE:
|
|
+ is_self = event->callchain.pid == source->self_pid;
|
|
+ break;
|
|
+
|
|
+ case PERF_RECORD_READ:
|
|
+ case PERF_RECORD_THROTTLE:
|
|
+ case PERF_RECORD_UNTHROTTLE:
|
|
+ goto skip_callback;
|
|
+
|
|
+ case PERF_RECORD_LOST:
|
|
+ break;
|
|
+ }
|
|
+
|
|
+ handle_event (source, event);
|
|
+
|
|
+ us += is_self;
|
|
+ them += !is_self;
|
|
+
|
|
+ skip_callback:
|
|
+ tail += header->size;
|
|
+ }
|
|
+
|
|
+ source->tail = tail;
|
|
+
|
|
+ atomic_thread_fence (memory_order_seq_cst);
|
|
+
|
|
+ source->map->data_tail = tail;
|
|
+
|
|
+ sysprof_capture_writer_flush (source->writer);
|
|
+
|
|
+ return G_SOURCE_CONTINUE;
|
|
+}
|
|
+
|
|
+static void
|
|
+perf_source_finalize (GSource *gsource)
|
|
+{
|
|
+ PerfSource *source = (PerfSource *)gsource;
|
|
+
|
|
+ if (source->map != NULL &&
|
|
+ (gpointer)source->map != MAP_FAILED)
|
|
+ munmap ((gpointer)source->map, source->map_size);
|
|
+
|
|
+ g_clear_pointer (&source->buffer, g_free);
|
|
+ g_clear_pointer (&source->writer, sysprof_capture_writer_unref);
|
|
+ g_clear_object (&source->unwinder);
|
|
+
|
|
+ source->map = NULL;
|
|
+ source->map_data = NULL;
|
|
+ source->map_data_end = NULL;
|
|
+ source->map_data_size = 0;
|
|
+ source->map_size = 0;
|
|
+ source->tail = 0;
|
|
+}
|
|
+
|
|
+static const GSourceFuncs source_funcs = {
|
|
+ .prepare = perf_source_prepare,
|
|
+ .check = perf_source_check,
|
|
+ .dispatch = perf_source_dispatch,
|
|
+ .finalize = perf_source_finalize,
|
|
+};
|
|
+
|
|
+static gboolean
|
|
+perf_source_init (PerfSource *source,
|
|
+ int fd,
|
|
+ SysprofLiveUnwinder *unwinder,
|
|
+ SysprofCaptureWriter *writer,
|
|
+ int cpu,
|
|
+ int stack_size,
|
|
+ GError **error)
|
|
+{
|
|
+ gsize map_size;
|
|
+ guint8 *map;
|
|
+
|
|
+ g_assert (source != NULL);
|
|
+ g_assert (writer != NULL);
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (unwinder));
|
|
+ g_assert (fd > STDERR_FILENO);
|
|
+
|
|
+ map_size = N_PAGES * sysprof_getpagesize () + sysprof_getpagesize ();
|
|
+ map = mmap (NULL, map_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
|
|
+
|
|
+ if ((gpointer)map == MAP_FAILED)
|
|
+ {
|
|
+ int errsv = errno;
|
|
+ g_set_error_literal (error,
|
|
+ G_FILE_ERROR,
|
|
+ g_file_error_from_errno (errsv),
|
|
+ g_strerror (errsv));
|
|
+ return FALSE;
|
|
+ }
|
|
+
|
|
+ source->writer = sysprof_capture_writer_ref (writer);
|
|
+ source->unwinder = g_object_ref (unwinder);
|
|
+ source->buffer = g_malloc (map_size);
|
|
+ source->map_size = map_size;
|
|
+ source->map = (gpointer)map;
|
|
+ source->map_data = map + sysprof_getpagesize ();
|
|
+ source->map_data_size = N_PAGES * sysprof_getpagesize ();
|
|
+ source->map_data_end = source->map_data + source->map_data_size;
|
|
+ source->tail = 0;
|
|
+ source->self_pid = getpid ();
|
|
+ source->cpu = cpu;
|
|
+ source->stack_size = stack_size;
|
|
+
|
|
+ g_source_add_unix_fd ((GSource *)source, fd, G_IO_IN);
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static GSource *
|
|
+perf_source_new (int perf_fd,
|
|
+ SysprofLiveUnwinder *unwinder,
|
|
+ SysprofCaptureWriter *writer,
|
|
+ int cpu,
|
|
+ int stack_size,
|
|
+ GError **error)
|
|
+{
|
|
+ GSource *source;
|
|
+
|
|
+ if (perf_fd <= STDERR_FILENO)
|
|
+ {
|
|
+ g_set_error (error,
|
|
+ G_FILE_ERROR,
|
|
+ G_FILE_ERROR_BADF,
|
|
+ "Invalid file-descriptor for perf event stream");
|
|
+ return NULL;
|
|
+ }
|
|
+
|
|
+ source = g_source_new ((GSourceFuncs *)&source_funcs, sizeof (PerfSource));
|
|
+ g_source_set_static_name (source, "[perf-event-stream]");
|
|
+ g_source_set_priority (source, G_PRIORITY_HIGH);
|
|
+
|
|
+ if (!perf_source_init ((PerfSource *)source, perf_fd, unwinder, writer, cpu, stack_size, error))
|
|
+ g_clear_pointer (&source, g_source_unref);
|
|
+
|
|
+ return source;
|
|
+}
|
|
+
|
|
+static void
|
|
+clear_perf_fd (gpointer data)
|
|
+{
|
|
+ PerfFDArg *arg = data;
|
|
+
|
|
+ if (arg->fd > STDERR_FILENO)
|
|
+ {
|
|
+ close (arg->fd);
|
|
+ arg->fd = -1;
|
|
+ }
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+perf_fd_callback (const char *option_name,
|
|
+ const char *option_value,
|
|
+ gpointer data,
|
|
+ GError **error)
|
|
+{
|
|
+ PerfFDArg arg = {-1, -1};
|
|
+
|
|
+ if (sscanf (option_value, "%d:%d", &arg.fd, &arg.cpu) < 1)
|
|
+ {
|
|
+ g_set_error (error,
|
|
+ G_FILE_ERROR,
|
|
+ G_FILE_ERROR_BADF,
|
|
+ "--perf-fd must be in the format FD or FD:CPU_NUMBER");
|
|
+ return FALSE;
|
|
+ }
|
|
+
|
|
+ if (arg.fd <= STDERR_FILENO)
|
|
+ {
|
|
+ g_set_error (error,
|
|
+ G_FILE_ERROR,
|
|
+ G_FILE_ERROR_BADF,
|
|
+ "--perf-fd must be >= %d",
|
|
+ STDERR_FILENO);
|
|
+ return FALSE;
|
|
+ }
|
|
+
|
|
+ g_array_append_val (all_perf_fds, arg);
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static void
|
|
+bump_to_max_fd_limit (void)
|
|
+{
|
|
+ struct rlimit limit;
|
|
+
|
|
+ if (getrlimit (RLIMIT_NOFILE, &limit) == 0)
|
|
+ {
|
|
+ limit.rlim_cur = limit.rlim_max;
|
|
+
|
|
+ if (setrlimit (RLIMIT_NOFILE, &limit) != 0)
|
|
+ g_warning ("Failed to set FD limit to %"G_GSSIZE_FORMAT"",
|
|
+ (gssize)limit.rlim_max);
|
|
+ else
|
|
+ g_debug ("Set RLIMIT_NOFILE to %"G_GSSIZE_FORMAT"",
|
|
+ (gssize)limit.rlim_max);
|
|
+ }
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+exit_callback (gpointer user_data)
|
|
+{
|
|
+ g_main_loop_quit (user_data);
|
|
+ return G_SOURCE_REMOVE;
|
|
+}
|
|
+
|
|
+int
|
|
+main (int argc,
|
|
+ char *argv[])
|
|
+{
|
|
+ g_autoptr(SysprofCaptureWriter) writer = NULL;
|
|
+ g_autoptr(SysprofLiveUnwinder) unwinder = NULL;
|
|
+ g_autoptr(GOptionContext) context = NULL;
|
|
+ g_autoptr(GMainLoop) main_loop = NULL;
|
|
+ g_autoptr(GError) error = NULL;
|
|
+ g_autofd int capture_fd = -1;
|
|
+ g_autofd int kallsyms_fd = -1;
|
|
+ g_autofd int event_fd = -1;
|
|
+ int stack_size = 0;
|
|
+ const GOptionEntry entries[] = {
|
|
+ { "perf-fd", 0, 0, G_OPTION_ARG_CALLBACK, perf_fd_callback, "A file-descriptor to the perf event stream", "FD[:CPU]" },
|
|
+ { "capture-fd", 0, 0, G_OPTION_ARG_INT, &capture_fd, "A file-descriptor to the sysprof capture", "FD" },
|
|
+ { "event-fd", 0, 0, G_OPTION_ARG_INT, &event_fd, "A file-descriptor to an event-fd used to notify unwinder should exit", "FD" },
|
|
+ { "kallsyms", 'k', 0, G_OPTION_ARG_INT, &kallsyms_fd, "Bundle kallsyms provided from passed FD", "FD" },
|
|
+ { "stack-size", 's', 0, G_OPTION_ARG_INT, &stack_size, "Size of stacks being recorded", "STACK_SIZE" },
|
|
+ { 0 }
|
|
+ };
|
|
+
|
|
+ main_loop = g_main_loop_new (NULL, FALSE);
|
|
+
|
|
+ all_perf_fds = g_array_new (FALSE, FALSE, sizeof (PerfFDArg));
|
|
+ g_array_set_clear_func (all_perf_fds, clear_perf_fd);
|
|
+
|
|
+ context = g_option_context_new ("- translate perf event stream to sysprof");
|
|
+ g_option_context_add_main_entries (context, entries, NULL);
|
|
+ g_option_context_set_summary (context, "\
|
|
+ This tool is used by sysprofd to process incoming perf events that\n\
|
|
+ include a copy of the stack and register state to the Sysprof capture\n\
|
|
+ format.\n\
|
|
+\n\
|
|
+ It should be provided two file-descriptors. One is for the perf-event\n\
|
|
+ stream and one is for the Sysprof capture writer.\n\
|
|
+\n\
|
|
+ Events that are not related to stack traces will also be passed along to\n\
|
|
+ to the capture in the standard Sysprof capture format. That includes mmap\n\
|
|
+ events, process information, and more.\n\
|
|
+\n\
|
|
+Examples:\n\
|
|
+\n\
|
|
+ # FD 3 contains perf_event stream for CPU 1\n\
|
|
+ sysprof-translate --perf-fd=3:1 --capture-fd=4");
|
|
+
|
|
+ if (!g_option_context_parse (context, &argc, &argv, &error))
|
|
+ {
|
|
+ g_printerr ("%s\n", error->message);
|
|
+ return EXIT_FAILURE;
|
|
+ }
|
|
+
|
|
+ if (capture_fd <= STDERR_FILENO)
|
|
+ {
|
|
+ g_printerr ("--capture-fd must be > %d\n", STDERR_FILENO);
|
|
+ return EXIT_FAILURE;
|
|
+ }
|
|
+
|
|
+ writer = sysprof_capture_writer_new_from_fd (g_steal_fd (&capture_fd), CAPTURE_BUFFER_SIZE);
|
|
+
|
|
+ if (all_perf_fds->len == 0)
|
|
+ {
|
|
+ g_printerr ("You must secify at least one --perf-fd\n");
|
|
+ return EXIT_FAILURE;
|
|
+ }
|
|
+
|
|
+ bump_to_max_fd_limit ();
|
|
+
|
|
+ unwinder = sysprof_live_unwinder_new (writer, g_steal_fd (&kallsyms_fd));
|
|
+
|
|
+ for (guint i = 0; i < all_perf_fds->len; i++)
|
|
+ {
|
|
+ const PerfFDArg *arg = &g_array_index (all_perf_fds, PerfFDArg, i);
|
|
+ g_autoptr(GSource) perf_source = NULL;
|
|
+
|
|
+ if (!(perf_source = perf_source_new (arg->fd, unwinder, writer, arg->cpu, stack_size, &error)))
|
|
+ {
|
|
+ g_printerr ("Failed to initialize perf event stream: %s\n",
|
|
+ error->message);
|
|
+ return EXIT_FAILURE;
|
|
+ }
|
|
+
|
|
+ g_source_attach (perf_source, NULL);
|
|
+ }
|
|
+
|
|
+ if (event_fd != -1)
|
|
+ {
|
|
+ g_autoptr(GSource) exit_source = g_unix_fd_source_new (event_fd, G_IO_IN);
|
|
+
|
|
+ g_source_set_callback (exit_source,
|
|
+ exit_callback,
|
|
+ g_main_loop_ref (main_loop),
|
|
+ (GDestroyNotify) g_main_loop_unref);
|
|
+ g_source_attach (exit_source, NULL);
|
|
+ }
|
|
+
|
|
+ g_main_loop_run (main_loop);
|
|
+
|
|
+ g_clear_pointer (&all_perf_fds, g_array_unref);
|
|
+
|
|
+ return EXIT_SUCCESS;
|
|
+}
|
|
diff --git a/src/sysprof-live-unwinder/meson.build b/src/sysprof-live-unwinder/meson.build
|
|
new file mode 100644
|
|
index 00000000..8cef7106
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/meson.build
|
|
@@ -0,0 +1,19 @@
|
|
+sysprof_live_unwinder_deps = [
|
|
+ libsysprof_static_dep,
|
|
+ dependency('libdw'),
|
|
+]
|
|
+
|
|
+sysprof_live_unwinder_sources = [
|
|
+ 'sysprof-live-process.c',
|
|
+ 'sysprof-live-unwinder.c',
|
|
+ 'main.c',
|
|
+]
|
|
+
|
|
+sysprof_live_unwinder = executable('sysprof-live-unwinder', sysprof_live_unwinder_sources,
|
|
+ dependencies: sysprof_live_unwinder_deps,
|
|
+ c_args: release_flags,
|
|
+ install: true,
|
|
+ install_dir: pkglibexecdir,
|
|
+)
|
|
+
|
|
+subdir('tests')
|
|
diff --git a/src/sysprof-live-unwinder/sysprof-live-process.c b/src/sysprof-live-unwinder/sysprof-live-process.c
|
|
new file mode 100644
|
|
index 00000000..7932048b
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/sysprof-live-process.c
|
|
@@ -0,0 +1,505 @@
|
|
+/*
|
|
+ * sysprof-live-pid.c
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#include "config.h"
|
|
+
|
|
+#include <fcntl.h>
|
|
+#include <unistd.h>
|
|
+
|
|
+#include <sys/wait.h>
|
|
+#include <sys/syscall.h>
|
|
+
|
|
+/* Workaround for linux/fcntl.h also including the
|
|
+ * flock definitions in addition to libc.
|
|
+ */
|
|
+#ifndef _LINUX_FCNTL_H
|
|
+# define _LINUX_FCNTL_H
|
|
+# include <linux/pidfd.h>
|
|
+# undef _LINUX_FCNTL_H
|
|
+#else
|
|
+# include <linux/pidfd.h>
|
|
+#endif
|
|
+
|
|
+#include <linux/perf_event.h>
|
|
+
|
|
+#include <libelf.h>
|
|
+#include <elfutils/libdwelf.h>
|
|
+#include <elfutils/libdwfl.h>
|
|
+
|
|
+#include <glib/gstdio.h>
|
|
+
|
|
+#include <sysprof-capture.h>
|
|
+
|
|
+#include "sysprof-live-process.h"
|
|
+
|
|
+typedef struct _SysprofLiveProcess
|
|
+{
|
|
+ Dwfl_Callbacks callbacks;
|
|
+ Dwfl *dwfl;
|
|
+ Elf *elf;
|
|
+ char *root;
|
|
+ GPid pid;
|
|
+ int fd;
|
|
+} SysprofLiveProcess;
|
|
+
|
|
+typedef struct _SysprofUnwinder
|
|
+{
|
|
+ SysprofLiveProcess *process;
|
|
+ const guint8 *stack;
|
|
+ gsize stack_len;
|
|
+ const guint64 *registers;
|
|
+ guint n_registers;
|
|
+ guint64 *addresses;
|
|
+ guint addresses_capacity;
|
|
+ guint addresses_len;
|
|
+ guint64 abi;
|
|
+ guint64 sp;
|
|
+ guint64 pc;
|
|
+ guint64 base_address;
|
|
+ GPid pid;
|
|
+ GPid tid;
|
|
+} SysprofUnwinder;
|
|
+
|
|
+static SysprofUnwinder *current_unwinder;
|
|
+
|
|
+static inline GPid
|
|
+sysprof_unwinder_next_thread (Dwfl *dwfl,
|
|
+ void *user_data,
|
|
+ void **thread_argp)
|
|
+{
|
|
+ SysprofUnwinder *unwinder = current_unwinder;
|
|
+
|
|
+ if (*thread_argp == NULL)
|
|
+ {
|
|
+ *thread_argp = unwinder;
|
|
+ return unwinder->tid;
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
+static inline bool
|
|
+sysprof_unwinder_get_thread (Dwfl *dwfl,
|
|
+ pid_t tid,
|
|
+ void *user_data,
|
|
+ void **thread_argp)
|
|
+{
|
|
+ SysprofUnwinder *unwinder = current_unwinder;
|
|
+
|
|
+ if (unwinder->tid == tid)
|
|
+ {
|
|
+ *thread_argp = unwinder;
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ return FALSE;
|
|
+}
|
|
+
|
|
+static inline bool
|
|
+copy_word (const guint8 *data,
|
|
+ Dwarf_Word *result,
|
|
+ guint64 abi)
|
|
+{
|
|
+ if (abi == PERF_SAMPLE_REGS_ABI_64)
|
|
+ memcpy (result, data, sizeof (guint64));
|
|
+ else if (abi == PERF_SAMPLE_REGS_ABI_32)
|
|
+ memcpy (result, data, sizeof (guint32));
|
|
+ else
|
|
+ g_assert_not_reached ();
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static inline bool
|
|
+sysprof_unwinder_memory_read (Dwfl *dwfl,
|
|
+ Dwarf_Addr addr,
|
|
+ Dwarf_Word *result,
|
|
+ void *user_data)
|
|
+{
|
|
+ SysprofUnwinder *unwinder = current_unwinder;
|
|
+
|
|
+ if (addr < unwinder->base_address || addr - unwinder->base_address >= unwinder->stack_len)
|
|
+ {
|
|
+ Dwfl_Module *module = NULL;
|
|
+ Elf_Data *data = NULL;
|
|
+ Elf_Scn *section = NULL;
|
|
+ Dwarf_Addr bias;
|
|
+
|
|
+ if (!(module = dwfl_addrmodule (dwfl, addr)) ||
|
|
+ !(section = dwfl_module_address_section (module, &addr, &bias)) ||
|
|
+ !(data = elf_getdata (section, NULL)))
|
|
+ return FALSE;
|
|
+
|
|
+ if (data->d_buf && data->d_size > addr)
|
|
+ return copy_word ((guint8 *)data->d_buf + addr, result, unwinder->abi);
|
|
+
|
|
+ return FALSE;
|
|
+ }
|
|
+
|
|
+ return copy_word (&unwinder->stack[addr - unwinder->base_address], result, unwinder->abi);
|
|
+}
|
|
+
|
|
+static inline bool
|
|
+sysprof_unwinder_set_initial_registers (Dwfl_Thread *thread,
|
|
+ void *user_data)
|
|
+{
|
|
+ SysprofUnwinder *unwinder = current_unwinder;
|
|
+
|
|
+ dwfl_thread_state_register_pc (thread, unwinder->pc);
|
|
+
|
|
+ if (unwinder->abi == PERF_SAMPLE_REGS_ABI_64)
|
|
+ {
|
|
+ static const int regs_x86_64[] = {0, 3, 2, 1, 4, 5, 6, 7/*sp*/, 9, 10, 11, 12, 13, 14, 15, 16, 8/*ip*/};
|
|
+
|
|
+ for (int i = 0; i < G_N_ELEMENTS (regs_x86_64); i++)
|
|
+ {
|
|
+ int j = regs_x86_64[i];
|
|
+
|
|
+ dwfl_thread_state_registers (thread, i, 1, &unwinder->registers[j]);
|
|
+ }
|
|
+ }
|
|
+ else if (unwinder->abi == PERF_SAMPLE_REGS_ABI_32)
|
|
+ {
|
|
+ static const int regs_i386[] = {0, 2, 3, 1, 7/*sp*/, 6, 4, 5, 8/*ip*/};
|
|
+
|
|
+ for (int i = 0; i < G_N_ELEMENTS (regs_i386); i++)
|
|
+ {
|
|
+ int j = regs_i386[i];
|
|
+
|
|
+ dwfl_thread_state_registers (thread, i, 1, &unwinder->registers[j]);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static const Dwfl_Thread_Callbacks thread_callbacks = {
|
|
+ sysprof_unwinder_next_thread,
|
|
+ sysprof_unwinder_get_thread,
|
|
+ sysprof_unwinder_memory_read,
|
|
+ sysprof_unwinder_set_initial_registers,
|
|
+ NULL, /* detach */
|
|
+ NULL, /* thread_detach */
|
|
+};
|
|
+
|
|
+static inline int
|
|
+sysprof_unwinder_frame_cb (Dwfl_Frame *frame,
|
|
+ void *user_data)
|
|
+{
|
|
+ SysprofUnwinder *unwinder = current_unwinder;
|
|
+ Dwarf_Addr pc;
|
|
+ Dwarf_Addr sp;
|
|
+ bool is_activation;
|
|
+ guint8 sp_register_id;
|
|
+
|
|
+ if (unwinder->addresses_len == unwinder->addresses_capacity)
|
|
+ return DWARF_CB_ABORT;
|
|
+
|
|
+ if (!dwfl_frame_pc (frame, &pc, &is_activation))
|
|
+ return DWARF_CB_ABORT;
|
|
+
|
|
+ if (unwinder->abi == PERF_SAMPLE_REGS_ABI_64)
|
|
+ sp_register_id = 7;
|
|
+ else if (unwinder->abi == PERF_SAMPLE_REGS_ABI_32)
|
|
+ sp_register_id = 4;
|
|
+ else
|
|
+ return DWARF_CB_ABORT;
|
|
+
|
|
+ if (dwfl_frame_reg (frame, sp_register_id, &sp) < 0)
|
|
+ return DWARF_CB_ABORT;
|
|
+
|
|
+ unwinder->addresses[unwinder->addresses_len++] = pc;
|
|
+
|
|
+ return DWARF_CB_OK;
|
|
+}
|
|
+
|
|
+static inline guint
|
|
+sysprof_unwind (SysprofLiveProcess *self,
|
|
+ Dwfl *dwfl,
|
|
+ Elf *elf,
|
|
+ GPid pid,
|
|
+ GPid tid,
|
|
+ guint64 abi,
|
|
+ const guint64 *registers,
|
|
+ guint n_registers,
|
|
+ const guint8 *stack,
|
|
+ gsize stack_len,
|
|
+ guint64 *addresses,
|
|
+ guint n_addresses)
|
|
+{
|
|
+#if defined(__x86_64__) || defined(__i386__)
|
|
+ SysprofUnwinder unwinder;
|
|
+
|
|
+ g_assert (dwfl != NULL);
|
|
+ g_assert (elf != NULL);
|
|
+
|
|
+ /* Ignore anything byt 32/64 defined abi */
|
|
+ if (!(abi == PERF_SAMPLE_REGS_ABI_32 || abi == PERF_SAMPLE_REGS_ABI_64))
|
|
+ return 0;
|
|
+
|
|
+ /* Make sure we have registers/stack to work with */
|
|
+ if (registers == NULL || stack == NULL || stack_len == 0)
|
|
+ return 0;
|
|
+
|
|
+ /* 9 registers on 32-bit x86, 17 on 64-bit x86_64 */
|
|
+ if (!((abi == PERF_SAMPLE_REGS_ABI_32 && n_registers == 9) ||
|
|
+ (abi == PERF_SAMPLE_REGS_ABI_64 && n_registers == 17)))
|
|
+ return 0;
|
|
+
|
|
+ unwinder.process = self;
|
|
+ unwinder.sp = registers[7];
|
|
+ unwinder.pc = registers[8];
|
|
+ unwinder.base_address = unwinder.sp;
|
|
+ unwinder.addresses = addresses;
|
|
+ unwinder.addresses_capacity = n_addresses;
|
|
+ unwinder.addresses_len = 0;
|
|
+ unwinder.pid = pid;
|
|
+ unwinder.tid = tid;
|
|
+ unwinder.stack = stack;
|
|
+ unwinder.stack_len = stack_len;
|
|
+ unwinder.abi = abi;
|
|
+ unwinder.registers = registers;
|
|
+ unwinder.n_registers = n_registers;
|
|
+
|
|
+ current_unwinder = &unwinder;
|
|
+ dwfl_getthread_frames (dwfl, tid, sysprof_unwinder_frame_cb, NULL);
|
|
+ current_unwinder = NULL;
|
|
+
|
|
+ return unwinder.addresses_len;
|
|
+#else
|
|
+ return 0;
|
|
+#endif
|
|
+}
|
|
+
|
|
+G_GNUC_NO_INLINE static int
|
|
+_pidfd_open (int pid,
|
|
+ unsigned flags)
|
|
+{
|
|
+ int pidfd = syscall (SYS_pidfd_open, pid, flags);
|
|
+
|
|
+ if (pidfd != -1)
|
|
+ {
|
|
+ int old_flags = fcntl (pidfd, F_GETFD);
|
|
+
|
|
+ if (old_flags != -1)
|
|
+ fcntl (pidfd, F_SETFD, old_flags | FD_CLOEXEC);
|
|
+ }
|
|
+
|
|
+ return pidfd;
|
|
+}
|
|
+
|
|
+static int
|
|
+sysprof_live_process_find_elf (Dwfl_Module *module,
|
|
+ void **user_data,
|
|
+ const char *module_name,
|
|
+ Dwarf_Addr base_addr,
|
|
+ char **filename,
|
|
+ Elf **elf)
|
|
+{
|
|
+ g_assert (current_unwinder != NULL);
|
|
+ g_assert (current_unwinder->process != NULL);
|
|
+
|
|
+ *filename = NULL;
|
|
+ *elf = NULL;
|
|
+
|
|
+ if (module_name[0] == '/')
|
|
+ {
|
|
+ g_autofree char *path = g_strdup_printf ("/proc/%u/root/%s", current_unwinder->pid, module_name);
|
|
+ g_autofd int fd = open (path, O_RDONLY | O_CLOEXEC);
|
|
+
|
|
+ if (fd != -1)
|
|
+ {
|
|
+ *elf = dwelf_elf_begin (fd);
|
|
+ return g_steal_fd (&fd);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return dwfl_linux_proc_find_elf (module, user_data, module_name, base_addr, filename, elf);
|
|
+}
|
|
+
|
|
+static int
|
|
+sysprof_live_process_find_debuginfo (Dwfl_Module *module,
|
|
+ void **user_data,
|
|
+ const char *module_name,
|
|
+ Dwarf_Addr base_addr,
|
|
+ const char *file_name,
|
|
+ const char *debuglink_file,
|
|
+ GElf_Word debuglink_crc,
|
|
+ char **debuginfo_file_name)
|
|
+{
|
|
+ return -1;
|
|
+}
|
|
+
|
|
+SysprofLiveProcess *
|
|
+sysprof_live_process_new (GPid pid)
|
|
+{
|
|
+ SysprofLiveProcess *live_process;
|
|
+
|
|
+ live_process = g_atomic_rc_box_new0 (SysprofLiveProcess);
|
|
+ live_process->pid = pid;
|
|
+ live_process->fd = _pidfd_open (pid, 0);
|
|
+ live_process->root = g_strdup_printf ("/proc/%u/root/", pid);
|
|
+ live_process->callbacks.find_elf = sysprof_live_process_find_elf;
|
|
+ live_process->callbacks.find_debuginfo = sysprof_live_process_find_debuginfo;
|
|
+ live_process->callbacks.debuginfo_path = g_new0 (char *, 2);
|
|
+ live_process->callbacks.debuginfo_path[0] = g_build_filename (live_process->root, "usr/lib/debug", NULL);
|
|
+
|
|
+ return live_process;
|
|
+}
|
|
+
|
|
+SysprofLiveProcess *
|
|
+sysprof_live_process_ref (SysprofLiveProcess *live_process)
|
|
+{
|
|
+ g_return_val_if_fail (live_process != NULL, NULL);
|
|
+
|
|
+ return g_atomic_rc_box_acquire (live_process);
|
|
+}
|
|
+
|
|
+static void
|
|
+sysprof_live_process_finalize (gpointer data)
|
|
+{
|
|
+ SysprofLiveProcess *live_process = data;
|
|
+
|
|
+ if (live_process->fd != -1)
|
|
+ {
|
|
+ close (live_process->fd);
|
|
+ live_process->fd = -1;
|
|
+ }
|
|
+
|
|
+ g_clear_pointer (&live_process->elf, elf_end);
|
|
+ g_clear_pointer (&live_process->dwfl, dwfl_end);
|
|
+ g_clear_pointer (&live_process->root, g_free);
|
|
+ g_clear_pointer (&live_process->callbacks.debuginfo_path, g_free);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_process_unref (SysprofLiveProcess *live_process)
|
|
+{
|
|
+ g_return_if_fail (live_process != NULL);
|
|
+
|
|
+ g_atomic_rc_box_release_full (live_process, sysprof_live_process_finalize);
|
|
+}
|
|
+
|
|
+gboolean
|
|
+sysprof_live_process_is_active (SysprofLiveProcess *self)
|
|
+{
|
|
+ g_return_val_if_fail (self != NULL, FALSE);
|
|
+
|
|
+ return self->fd > -1;
|
|
+}
|
|
+
|
|
+static Dwfl *
|
|
+sysprof_live_process_get_dwfl (SysprofLiveProcess *self)
|
|
+{
|
|
+ g_assert (self != NULL);
|
|
+
|
|
+ if G_UNLIKELY (self->dwfl == NULL)
|
|
+ {
|
|
+ self->dwfl = dwfl_begin (&self->callbacks);
|
|
+
|
|
+ dwfl_linux_proc_report (self->dwfl, self->pid);
|
|
+ dwfl_report_end (self->dwfl, NULL, NULL);
|
|
+
|
|
+ if (self->fd > -1)
|
|
+ {
|
|
+ char path[64];
|
|
+ g_autofd int exe_fd = -1;
|
|
+
|
|
+ g_snprintf (path, sizeof path, "/proc/%u/exe", self->pid);
|
|
+ exe_fd = open (path, O_RDONLY);
|
|
+
|
|
+ if (exe_fd > -1)
|
|
+ {
|
|
+ self->elf = elf_begin (exe_fd, ELF_C_READ_MMAP, NULL);
|
|
+
|
|
+ if (self->elf != NULL)
|
|
+ dwfl_attach_state (self->dwfl, self->elf, self->pid, &thread_callbacks, self);
|
|
+ }
|
|
+ }
|
|
+ else
|
|
+ g_warning ("Attmpting to load exited process\n");
|
|
+ }
|
|
+
|
|
+ return self->dwfl;
|
|
+}
|
|
+
|
|
+guint
|
|
+sysprof_live_process_unwind (SysprofLiveProcess *self,
|
|
+ GPid tid,
|
|
+ guint64 abi,
|
|
+ const guint8 *stack,
|
|
+ gsize stack_len,
|
|
+ const guint64 *registers,
|
|
+ guint n_registers,
|
|
+ guint64 *addresses,
|
|
+ guint n_addresses)
|
|
+{
|
|
+#if defined(__x86_64__) || defined(__i386__)
|
|
+ Dwfl *dwfl;
|
|
+
|
|
+ g_assert (self != NULL);
|
|
+ g_assert (stack != NULL);
|
|
+ g_assert (registers != NULL);
|
|
+ g_assert (addresses != NULL);
|
|
+
|
|
+ if (!sysprof_live_process_is_active (self))
|
|
+ return 0;
|
|
+
|
|
+ if (!(dwfl = sysprof_live_process_get_dwfl (self)))
|
|
+ return 0;
|
|
+
|
|
+ if (self->elf == NULL)
|
|
+ return 0;
|
|
+
|
|
+ g_assert (self->dwfl != NULL);
|
|
+ g_assert (self->elf != NULL);
|
|
+
|
|
+ return sysprof_unwind (self,
|
|
+ self->dwfl,
|
|
+ self->elf,
|
|
+ self->pid,
|
|
+ tid,
|
|
+ abi,
|
|
+ registers,
|
|
+ n_registers,
|
|
+ stack,
|
|
+ stack_len,
|
|
+ addresses,
|
|
+ n_addresses);
|
|
+#else
|
|
+ return 0;
|
|
+#endif
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_process_add_map (SysprofLiveProcess *self,
|
|
+ guint64 begin,
|
|
+ guint64 end,
|
|
+ guint64 offset,
|
|
+ guint64 inode,
|
|
+ const char *filename)
|
|
+{
|
|
+ g_assert (self != NULL);
|
|
+
|
|
+ /* We'll reparse VMAs on next use */
|
|
+ g_clear_pointer (&self->dwfl, dwfl_end);
|
|
+ g_clear_pointer (&self->elf, elf_end);
|
|
+}
|
|
diff --git a/src/sysprof-live-unwinder/sysprof-live-process.h b/src/sysprof-live-unwinder/sysprof-live-process.h
|
|
new file mode 100644
|
|
index 00000000..e2601a5b
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/sysprof-live-process.h
|
|
@@ -0,0 +1,50 @@
|
|
+/*
|
|
+ * sysprof-live-pid.h
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#pragma once
|
|
+
|
|
+#include <glib.h>
|
|
+
|
|
+G_BEGIN_DECLS
|
|
+
|
|
+typedef struct _SysprofLiveProcess SysprofLiveProcess;
|
|
+
|
|
+SysprofLiveProcess *sysprof_live_process_new (GPid pid);
|
|
+SysprofLiveProcess *sysprof_live_process_ref (SysprofLiveProcess *self);
|
|
+void sysprof_live_process_unref (SysprofLiveProcess *self);
|
|
+gboolean sysprof_live_process_is_active (SysprofLiveProcess *self);
|
|
+void sysprof_live_process_add_map (SysprofLiveProcess *self,
|
|
+ guint64 begin,
|
|
+ guint64 end,
|
|
+ guint64 offset,
|
|
+ guint64 inode,
|
|
+ const char *filename);
|
|
+guint sysprof_live_process_unwind (SysprofLiveProcess *self,
|
|
+ GPid tid,
|
|
+ guint64 abi,
|
|
+ const guint8 *stack,
|
|
+ gsize stack_len,
|
|
+ const guint64 *registers,
|
|
+ guint n_registers,
|
|
+ guint64 *addresses,
|
|
+ guint n_addresses);
|
|
+
|
|
+G_END_DECLS
|
|
diff --git a/src/sysprof-live-unwinder/sysprof-live-unwinder.c b/src/sysprof-live-unwinder/sysprof-live-unwinder.c
|
|
new file mode 100644
|
|
index 00000000..c3b954b2
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/sysprof-live-unwinder.c
|
|
@@ -0,0 +1,427 @@
|
|
+/*
|
|
+ * sysprof-live-unwinder.c
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#include "config.h"
|
|
+
|
|
+#include <fcntl.h>
|
|
+#include <unistd.h>
|
|
+
|
|
+#include <linux/perf_event.h>
|
|
+
|
|
+#include <glib/gstdio.h>
|
|
+
|
|
+#include "sysprof-live-process.h"
|
|
+#include "sysprof-live-unwinder.h"
|
|
+#include "sysprof-maps-parser-private.h"
|
|
+
|
|
+struct _SysprofLiveUnwinder
|
|
+{
|
|
+ GObject parent_instance;
|
|
+ SysprofCaptureWriter *writer;
|
|
+ GHashTable *live_pids_by_pid;
|
|
+};
|
|
+
|
|
+G_DEFINE_FINAL_TYPE (SysprofLiveUnwinder, sysprof_live_unwinder, G_TYPE_OBJECT)
|
|
+
|
|
+enum {
|
|
+ CLOSED,
|
|
+ N_SIGNALS
|
|
+};
|
|
+
|
|
+static guint signals[N_SIGNALS];
|
|
+
|
|
+static char *
|
|
+sysprof_live_unwinder_read_file (SysprofLiveUnwinder *self,
|
|
+ const char *path,
|
|
+ gboolean insert_into_capture)
|
|
+{
|
|
+ gint64 when = SYSPROF_CAPTURE_CURRENT_TIME;
|
|
+ char *contents = NULL;
|
|
+ gsize len = 0;
|
|
+ gsize offset = 0;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+ g_assert (self->writer != NULL);
|
|
+
|
|
+ if (!g_file_get_contents (path, &contents, &len, NULL))
|
|
+ return NULL;
|
|
+
|
|
+ if (insert_into_capture)
|
|
+ {
|
|
+ while (len > 0)
|
|
+ {
|
|
+ gsize this_write = MIN (len, 4096*4);
|
|
+
|
|
+ if (!sysprof_capture_writer_add_file (self->writer,
|
|
+ when,
|
|
+ -1,
|
|
+ -1,
|
|
+ path,
|
|
+ this_write == len,
|
|
+ (const guint8 *)&contents[offset], this_write))
|
|
+ break;
|
|
+
|
|
+ len -= this_write;
|
|
+ offset += this_write;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return contents;
|
|
+}
|
|
+
|
|
+static char *
|
|
+sysprof_live_unwinder_read_pid_file (SysprofLiveUnwinder *self,
|
|
+ GPid pid,
|
|
+ const char *path_part,
|
|
+ gboolean insert_into_capture)
|
|
+{
|
|
+ g_autofree char *path = NULL;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+ g_assert (self->writer != NULL);
|
|
+
|
|
+ path = g_strdup_printf ("/proc/%d/%s", pid, path_part);
|
|
+
|
|
+ return sysprof_live_unwinder_read_file (self, path, insert_into_capture);
|
|
+}
|
|
+
|
|
+static SysprofLiveProcess *
|
|
+sysprof_live_unwinder_find_pid (SysprofLiveUnwinder *self,
|
|
+ GPid pid,
|
|
+ gboolean send_comm)
|
|
+{
|
|
+ SysprofLiveProcess *live_pid;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ if (pid < 0)
|
|
+ return NULL;
|
|
+
|
|
+ if (!(live_pid = g_hash_table_lookup (self->live_pids_by_pid, GINT_TO_POINTER (pid))))
|
|
+ {
|
|
+ gint64 now = SYSPROF_CAPTURE_CURRENT_TIME;
|
|
+
|
|
+ live_pid = sysprof_live_process_new (pid);
|
|
+
|
|
+ g_hash_table_replace (self->live_pids_by_pid, GINT_TO_POINTER (pid), live_pid);
|
|
+
|
|
+ if (send_comm)
|
|
+ {
|
|
+ g_autofree char *path = g_strdup_printf ("/proc/%d/comm", pid);
|
|
+ g_autofree char *comm = NULL;
|
|
+ gsize len;
|
|
+
|
|
+ if (g_file_get_contents (path, &comm, &len, NULL))
|
|
+ {
|
|
+ g_autofree char *tmp = comm;
|
|
+ comm = g_strstrip (g_utf8_make_valid (tmp, len));
|
|
+ sysprof_capture_writer_add_process (self->writer, now, -1, pid, comm);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (sysprof_live_process_is_active (live_pid))
|
|
+ {
|
|
+ g_autofree char *mountinfo = sysprof_live_unwinder_read_pid_file (self, pid, "mountinfo", TRUE);
|
|
+ g_autofree char *maps = sysprof_live_unwinder_read_pid_file (self, pid, "maps", FALSE);
|
|
+
|
|
+ if (maps != NULL)
|
|
+ {
|
|
+ SysprofMapsParser maps_parser;
|
|
+ guint64 begin, end, offset, inode;
|
|
+ char *filename;
|
|
+
|
|
+ sysprof_maps_parser_init (&maps_parser, maps, -1);
|
|
+
|
|
+ while (sysprof_maps_parser_next (&maps_parser, &begin, &end, &offset, &inode, &filename))
|
|
+ {
|
|
+ sysprof_live_process_add_map (live_pid, begin, end, offset, inode, filename);
|
|
+ sysprof_capture_writer_add_map (self->writer, now, -1, pid,
|
|
+ begin, end, offset,
|
|
+ inode, filename);
|
|
+ g_free (filename);
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return live_pid;
|
|
+}
|
|
+
|
|
+static void
|
|
+sysprof_live_unwinder_mine_pids (SysprofLiveUnwinder *self)
|
|
+{
|
|
+ g_autoptr(GDir) dir = NULL;
|
|
+ const char *name;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+ g_assert (self->writer != NULL);
|
|
+
|
|
+ if (!(dir = g_dir_open ("/proc", 0, NULL)))
|
|
+ return;
|
|
+
|
|
+ while ((name = g_dir_read_name (dir)))
|
|
+ {
|
|
+ GPid pid;
|
|
+
|
|
+ if (!g_ascii_isdigit (*name))
|
|
+ continue;
|
|
+
|
|
+ if (!(pid = atoi (name)))
|
|
+ continue;
|
|
+
|
|
+ sysprof_live_unwinder_find_pid (self, pid, TRUE);
|
|
+ }
|
|
+}
|
|
+
|
|
+static void
|
|
+sysprof_live_unwinder_finalize (GObject *object)
|
|
+{
|
|
+ SysprofLiveUnwinder *self = (SysprofLiveUnwinder *)object;
|
|
+
|
|
+ g_clear_pointer (&self->writer, sysprof_capture_writer_unref);
|
|
+ g_clear_pointer (&self->live_pids_by_pid, g_hash_table_unref);
|
|
+
|
|
+ G_OBJECT_CLASS (sysprof_live_unwinder_parent_class)->finalize (object);
|
|
+}
|
|
+
|
|
+static void
|
|
+sysprof_live_unwinder_class_init (SysprofLiveUnwinderClass *klass)
|
|
+{
|
|
+ GObjectClass *object_class = G_OBJECT_CLASS (klass);
|
|
+
|
|
+ object_class->finalize = sysprof_live_unwinder_finalize;
|
|
+
|
|
+ signals[CLOSED] =
|
|
+ g_signal_new ("closed",
|
|
+ G_TYPE_FROM_CLASS (klass),
|
|
+ G_SIGNAL_RUN_LAST,
|
|
+ 0,
|
|
+ NULL, NULL,
|
|
+ NULL,
|
|
+ G_TYPE_NONE, 0);
|
|
+}
|
|
+
|
|
+static void
|
|
+sysprof_live_unwinder_init (SysprofLiveUnwinder *self)
|
|
+{
|
|
+ self->live_pids_by_pid = g_hash_table_new_full (NULL,
|
|
+ NULL,
|
|
+ NULL,
|
|
+ (GDestroyNotify)sysprof_live_process_unref);
|
|
+}
|
|
+
|
|
+SysprofLiveUnwinder *
|
|
+sysprof_live_unwinder_new (SysprofCaptureWriter *writer,
|
|
+ int kallsyms_fd)
|
|
+{
|
|
+ SysprofLiveUnwinder *self;
|
|
+ g_autofree char *mounts = NULL;
|
|
+
|
|
+ g_return_val_if_fail (writer != NULL, NULL);
|
|
+
|
|
+ self = g_object_new (SYSPROF_TYPE_LIVE_UNWINDER, NULL);
|
|
+ self->writer = sysprof_capture_writer_ref (writer);
|
|
+
|
|
+ if (kallsyms_fd != -1)
|
|
+ {
|
|
+ sysprof_capture_writer_add_file_fd (writer,
|
|
+ SYSPROF_CAPTURE_CURRENT_TIME,
|
|
+ -1,
|
|
+ -1,
|
|
+ "/proc/kallsyms",
|
|
+ kallsyms_fd);
|
|
+ close (kallsyms_fd);
|
|
+ }
|
|
+
|
|
+ mounts = sysprof_live_unwinder_read_file (self, "/proc/mounts", TRUE);
|
|
+
|
|
+ sysprof_live_unwinder_mine_pids (self);
|
|
+
|
|
+ return self;
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_seen_process (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ const char *comm)
|
|
+{
|
|
+ G_GNUC_UNUSED SysprofLiveProcess *live_pid;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ live_pid = sysprof_live_unwinder_find_pid (self, pid, FALSE);
|
|
+
|
|
+ sysprof_capture_writer_add_process (self->writer, time, cpu, pid, comm);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_process_exited (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid)
|
|
+{
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ sysprof_capture_writer_add_exit (self->writer, time, cpu, pid);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_process_forked (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid parent_tid,
|
|
+ GPid tid)
|
|
+{
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ sysprof_capture_writer_add_fork (self->writer, time, cpu, parent_tid, tid);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_track_mmap (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ SysprofCaptureAddress begin,
|
|
+ SysprofCaptureAddress end,
|
|
+ SysprofCaptureAddress offset,
|
|
+ guint64 inode,
|
|
+ const char *filename,
|
|
+ const char *build_id)
|
|
+{
|
|
+ G_GNUC_UNUSED SysprofLiveProcess *live_pid;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ live_pid = sysprof_live_unwinder_find_pid (self, pid, TRUE);
|
|
+
|
|
+ if (build_id != NULL)
|
|
+ sysprof_capture_writer_add_map_with_build_id (self->writer, time, cpu, pid,
|
|
+ begin, end, offset,
|
|
+ inode, filename,
|
|
+ build_id);
|
|
+ else
|
|
+ sysprof_capture_writer_add_map (self->writer, time, cpu, pid,
|
|
+ begin, end, offset,
|
|
+ inode, filename);
|
|
+
|
|
+ sysprof_live_process_add_map (live_pid, begin, end, offset, inode, filename);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_process_sampled (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ GPid tid,
|
|
+ const SysprofCaptureAddress *addresses,
|
|
+ guint n_addresses)
|
|
+{
|
|
+ G_GNUC_UNUSED SysprofLiveProcess *live_pid;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+
|
|
+ live_pid = sysprof_live_unwinder_find_pid (self, pid, TRUE);
|
|
+
|
|
+ sysprof_capture_writer_add_sample (self->writer, time, cpu, pid, tid,
|
|
+ addresses, n_addresses);
|
|
+}
|
|
+
|
|
+void
|
|
+sysprof_live_unwinder_process_sampled_with_stack (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ GPid tid,
|
|
+ const SysprofCaptureAddress *addresses,
|
|
+ guint n_addresses,
|
|
+ const guint8 *stack,
|
|
+ guint64 stack_size,
|
|
+ guint64 stack_dyn_size,
|
|
+ guint64 abi,
|
|
+ const guint64 *registers,
|
|
+ guint n_registers)
|
|
+{
|
|
+ SysprofLiveProcess *live_pid;
|
|
+ SysprofCaptureAddress unwound[256];
|
|
+ gboolean found_user = FALSE;
|
|
+ guint pos;
|
|
+
|
|
+ g_assert (SYSPROF_IS_LIVE_UNWINDER (self));
|
|
+ g_assert (stack != NULL);
|
|
+ g_assert (stack_dyn_size <= stack_size);
|
|
+
|
|
+ if (stack_dyn_size == 0 || n_addresses >= G_N_ELEMENTS (unwound))
|
|
+ {
|
|
+ sysprof_live_unwinder_process_sampled (self, time, cpu, pid, tid, addresses, n_addresses);
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ live_pid = sysprof_live_unwinder_find_pid (self, pid, TRUE);
|
|
+
|
|
+ /* Copy addresses over (which might be kernel, context-switch, etc until
|
|
+ * we get to the PERF_CONTEXT_USER. We'll decode the stack right into the
|
|
+ * location after that.
|
|
+ */
|
|
+ for (pos = 0; pos < n_addresses; pos++)
|
|
+ {
|
|
+ unwound[pos] = addresses[pos];
|
|
+
|
|
+ if (addresses[pos] == PERF_CONTEXT_USER)
|
|
+ {
|
|
+ found_user = TRUE;
|
|
+ break;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ /* If we didn't find a user context (but we have a stack size) synthesize
|
|
+ * the PERF_CONTEXT_USER now.
|
|
+ */
|
|
+ if (!found_user && pos < G_N_ELEMENTS (unwound))
|
|
+ unwound[pos++] = PERF_CONTEXT_USER;
|
|
+
|
|
+ /* Now request the live process unwind the user-space stack */
|
|
+ if (pos < G_N_ELEMENTS (unwound))
|
|
+ {
|
|
+ guint n_unwound;
|
|
+
|
|
+ n_unwound = sysprof_live_process_unwind (live_pid,
|
|
+ tid,
|
|
+ abi,
|
|
+ stack,
|
|
+ stack_dyn_size,
|
|
+ registers,
|
|
+ n_registers,
|
|
+ &unwound[pos],
|
|
+ G_N_ELEMENTS (unwound) - pos);
|
|
+
|
|
+ /* Only take DWARF unwind if it was better */
|
|
+ if (pos + n_unwound > n_addresses)
|
|
+ {
|
|
+ addresses = unwound;
|
|
+ n_addresses = pos + n_unwound;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ sysprof_capture_writer_add_sample (self->writer, time, cpu, pid, tid, addresses, n_addresses);
|
|
+}
|
|
diff --git a/src/sysprof-live-unwinder/sysprof-live-unwinder.h b/src/sysprof-live-unwinder/sysprof-live-unwinder.h
|
|
new file mode 100644
|
|
index 00000000..e27ed9a4
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/sysprof-live-unwinder.h
|
|
@@ -0,0 +1,79 @@
|
|
+/*
|
|
+ * sysprof-live-unwinder.h
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#pragma once
|
|
+
|
|
+#include <sysprof.h>
|
|
+
|
|
+G_BEGIN_DECLS
|
|
+
|
|
+#define SYSPROF_TYPE_LIVE_UNWINDER (sysprof_live_unwinder_get_type())
|
|
+
|
|
+G_DECLARE_FINAL_TYPE (SysprofLiveUnwinder, sysprof_live_unwinder, SYSPROF, LIVE_UNWINDER, GObject)
|
|
+
|
|
+SysprofLiveUnwinder *sysprof_live_unwinder_new (SysprofCaptureWriter *writer,
|
|
+ int kallsyms_fd);
|
|
+void sysprof_live_unwinder_seen_process (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ const char *comm);
|
|
+void sysprof_live_unwinder_process_exited (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid);
|
|
+void sysprof_live_unwinder_process_forked (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid parent_tid,
|
|
+ GPid tid);
|
|
+void sysprof_live_unwinder_track_mmap (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ SysprofCaptureAddress begin,
|
|
+ SysprofCaptureAddress end,
|
|
+ SysprofCaptureAddress offset,
|
|
+ guint64 inode,
|
|
+ const char *filename,
|
|
+ const char *build_id);
|
|
+void sysprof_live_unwinder_process_sampled (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ GPid tid,
|
|
+ const SysprofCaptureAddress *addresses,
|
|
+ guint n_addresses);
|
|
+void sysprof_live_unwinder_process_sampled_with_stack (SysprofLiveUnwinder *self,
|
|
+ gint64 time,
|
|
+ int cpu,
|
|
+ GPid pid,
|
|
+ GPid tid,
|
|
+ const SysprofCaptureAddress *addresses,
|
|
+ guint n_addresses,
|
|
+ const guint8 *stack,
|
|
+ guint64 stack_size,
|
|
+ guint64 stack_dyn_size,
|
|
+ guint64 abi,
|
|
+ const guint64 *registers,
|
|
+ guint n_registers);
|
|
+
|
|
+G_END_DECLS
|
|
diff --git a/src/sysprof-live-unwinder/tests/meson.build b/src/sysprof-live-unwinder/tests/meson.build
|
|
new file mode 100644
|
|
index 00000000..9e1a945a
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/tests/meson.build
|
|
@@ -0,0 +1,35 @@
|
|
+sysprof_live_unwinder_test_env = [
|
|
+ 'G_DEBUG=gc-friendly',
|
|
+ 'GSETTINGS_BACKEND=memory',
|
|
+ 'MALLOC_CHECK_=2',
|
|
+]
|
|
+
|
|
+sysprof_live_unwinder_testsuite_c_args = [
|
|
+ '-DG_ENABLE_DEBUG',
|
|
+ '-UG_DISABLE_ASSERT',
|
|
+ '-UG_DISABLE_CAST_CHECKS',
|
|
+ '-DBUILDDIR="@0@"'.format(meson.current_build_dir()),
|
|
+]
|
|
+
|
|
+sysprof_live_unwinder_testsuite = {
|
|
+ 'test-live-unwinder' : {'skip': true},
|
|
+}
|
|
+
|
|
+sysprof_live_unwinder_testsuite_deps = [
|
|
+ libsysprof_static_dep,
|
|
+]
|
|
+
|
|
+if polkit_agent_dep.found()
|
|
+ sysprof_live_unwinder_testsuite_deps += polkit_agent_dep
|
|
+endif
|
|
+
|
|
+foreach test, params: sysprof_live_unwinder_testsuite
|
|
+ test_exe = executable(test, '@0@.c'.format(test),
|
|
+ c_args: sysprof_live_unwinder_testsuite_c_args,
|
|
+ dependencies: sysprof_live_unwinder_testsuite_deps,
|
|
+ )
|
|
+
|
|
+ if not params.get('skip', false)
|
|
+ test(test, test_exe, env: sysprof_live_unwinder_test_env)
|
|
+ endif
|
|
+endforeach
|
|
diff --git a/src/sysprof-live-unwinder/tests/test-live-unwinder.c b/src/sysprof-live-unwinder/tests/test-live-unwinder.c
|
|
new file mode 100644
|
|
index 00000000..114cc568
|
|
--- /dev/null
|
|
+++ b/src/sysprof-live-unwinder/tests/test-live-unwinder.c
|
|
@@ -0,0 +1,391 @@
|
|
+/*
|
|
+ * test-live-unwinder.c
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#include "config.h"
|
|
+
|
|
+#include <fcntl.h>
|
|
+#include <unistd.h>
|
|
+
|
|
+#include <sys/ioctl.h>
|
|
+
|
|
+#include <glib/gstdio.h>
|
|
+#include <gio/gio.h>
|
|
+
|
|
+#include <libdex.h>
|
|
+
|
|
+#include <sysprof.h>
|
|
+
|
|
+#include "sysprof-perf-event-stream-private.h"
|
|
+
|
|
+#if HAVE_POLKIT_AGENT
|
|
+# define POLKIT_AGENT_I_KNOW_API_IS_SUBJECT_TO_CHANGE
|
|
+# include <polkit/polkit.h>
|
|
+# include <polkitagent/polkitagent.h>
|
|
+#endif
|
|
+
|
|
+#include <asm/perf_regs.h>
|
|
+
|
|
+#define N_WAKEUP_EVENTS 149
|
|
+
|
|
+/* The following was provided to Sysprof by Serhei Makarov as part
|
|
+ * of the eu-stacktrace prototype work.
|
|
+ */
|
|
+#ifdef _ASM_X86_PERF_REGS_H
|
|
+/* #define SYSPROF_ARCH_PREFERRED_REGS PERF_REG_EXTENDED_MASK -- error on x86_64 due to including segment regs*/
|
|
+#define REG(R) (1ULL << PERF_REG_X86_ ## R)
|
|
+#define DWARF_NEEDED_REGS (/* no FLAGS */ REG(IP) | REG(SP) | REG(AX) | REG(CX) | REG(DX) | REG(BX) | REG(SI) | REG(DI) | REG(SP) | REG(BP) | /* no segment regs */ REG(R8) | REG(R9) | REG(R10) | REG(R11) | REG(R12) | REG(R13) | REG(R14) | REG(R15))
|
|
+/* XXX register ordering is defined in linux arch/x86/include/uapi/asm/perf_regs.h;
|
|
+ see code in tools/perf/util/intel-pt.c intel_pt_add_gp_regs()
|
|
+ and note how registers are added in the same order as the perf_regs.h enum */
|
|
+#define SYSPROF_ARCH_PREFERRED_REGS DWARF_NEEDED_REGS
|
|
+/* TODO: add other architectures, imitating the linux tools/perf tree */
|
|
+#else
|
|
+# define SYSPROF_ARCH_PREFERRED_REGS PERF_REG_EXTENDED_MASK
|
|
+#endif /* _ASM_{arch}_PERF_REGS_H */
|
|
+
|
|
+static gboolean sample_stack;
|
|
+static char *kallsyms = NULL;
|
|
+static int sample_stack_size = 8192;
|
|
+
|
|
+static void
|
|
+open_perf_stream_cb (GObject *object,
|
|
+ GAsyncResult *result,
|
|
+ gpointer user_data)
|
|
+{
|
|
+ GDBusConnection *connection = (GDBusConnection *)object;
|
|
+ g_autoptr(DexPromise) promise = user_data;
|
|
+ g_autoptr(GUnixFDList) fd_list = NULL;
|
|
+ g_autoptr(GVariant) ret = NULL;
|
|
+ g_autoptr(GError) error = NULL;
|
|
+
|
|
+ g_assert (G_IS_DBUS_CONNECTION (connection));
|
|
+ g_assert (G_IS_ASYNC_RESULT (result));
|
|
+ g_assert (DEX_IS_PROMISE (promise));
|
|
+
|
|
+ if ((ret = g_dbus_connection_call_with_unix_fd_list_finish (connection, &fd_list, result, &error)))
|
|
+ {
|
|
+ int handle;
|
|
+ int fd;
|
|
+
|
|
+ g_variant_get (ret, "(h)", &handle);
|
|
+
|
|
+ if (-1 != (fd = g_unix_fd_list_get (fd_list, handle, &error)))
|
|
+ {
|
|
+ dex_promise_resolve_fd (promise, g_steal_fd (&fd));
|
|
+ return;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ g_assert (error != NULL);
|
|
+
|
|
+ dex_promise_reject (promise, g_steal_pointer (&error));
|
|
+}
|
|
+
|
|
+static int
|
|
+open_perf_stream (GDBusConnection *bus,
|
|
+ int cpu,
|
|
+ GError **error)
|
|
+{
|
|
+ struct perf_event_attr attr = {0};
|
|
+ gboolean with_mmap2 = TRUE;
|
|
+ gboolean use_software = FALSE;
|
|
+
|
|
+ g_assert (G_IS_DBUS_CONNECTION (bus));
|
|
+ g_assert (cpu >= 0);
|
|
+ g_assert (error != NULL);
|
|
+
|
|
+try_again:
|
|
+ attr.sample_type = PERF_SAMPLE_IP
|
|
+ | PERF_SAMPLE_TID
|
|
+ | PERF_SAMPLE_IDENTIFIER
|
|
+ | PERF_SAMPLE_CALLCHAIN
|
|
+ | PERF_SAMPLE_TIME;
|
|
+
|
|
+ if (sample_stack)
|
|
+ {
|
|
+ attr.sample_type |= (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER);
|
|
+ attr.sample_stack_user = sample_stack_size;
|
|
+ attr.sample_regs_user = SYSPROF_ARCH_PREFERRED_REGS;
|
|
+ }
|
|
+
|
|
+ attr.wakeup_events = N_WAKEUP_EVENTS;
|
|
+ attr.disabled = TRUE;
|
|
+ attr.mmap = TRUE;
|
|
+ attr.mmap2 = with_mmap2;
|
|
+ attr.comm = 1;
|
|
+ attr.task = 1;
|
|
+ attr.exclude_idle = 1;
|
|
+ attr.sample_id_all = 1;
|
|
+
|
|
+#ifdef HAVE_PERF_CLOCKID
|
|
+ attr.clockid = sysprof_clock;
|
|
+ attr.use_clockid = 1;
|
|
+#endif
|
|
+
|
|
+ attr.size = sizeof attr;
|
|
+
|
|
+ if (use_software)
|
|
+ {
|
|
+ attr.type = PERF_TYPE_SOFTWARE;
|
|
+ attr.config = PERF_COUNT_SW_CPU_CLOCK;
|
|
+ attr.sample_period = 1000000;
|
|
+ }
|
|
+ else
|
|
+ {
|
|
+ attr.type = PERF_TYPE_HARDWARE;
|
|
+ attr.config = PERF_COUNT_HW_CPU_CYCLES;
|
|
+ attr.sample_period = 1200000;
|
|
+ }
|
|
+
|
|
+ {
|
|
+ g_autoptr(GVariant) options = _sysprof_perf_event_attr_to_variant (&attr);
|
|
+ g_autoptr(DexPromise) promise = dex_promise_new ();
|
|
+ g_autofd int fd = -1;
|
|
+
|
|
+ g_dbus_connection_call_with_unix_fd_list (bus,
|
|
+ "org.gnome.Sysprof3",
|
|
+ "/org/gnome/Sysprof3",
|
|
+ "org.gnome.Sysprof3.Service",
|
|
+ "PerfEventOpen",
|
|
+ g_variant_new ("(@a{sv}iiht)",
|
|
+ options,
|
|
+ -1,
|
|
+ cpu,
|
|
+ -1,
|
|
+ 0),
|
|
+ G_VARIANT_TYPE ("(h)"),
|
|
+ G_DBUS_CALL_FLAGS_ALLOW_INTERACTIVE_AUTHORIZATION,
|
|
+ G_MAXUINT,
|
|
+ NULL,
|
|
+ dex_promise_get_cancellable (promise),
|
|
+ open_perf_stream_cb,
|
|
+ dex_ref (promise));
|
|
+
|
|
+ fd = dex_await_fd (dex_ref (promise), error);
|
|
+
|
|
+ if (*error == NULL)
|
|
+ {
|
|
+ g_printerr ("CPU[%d]: opened perf_event stream as FD %d\n", cpu, fd);
|
|
+ return g_steal_fd (&fd);
|
|
+ }
|
|
+
|
|
+ fd = -1;
|
|
+ }
|
|
+
|
|
+ if (with_mmap2)
|
|
+ {
|
|
+ g_clear_error (error);
|
|
+ with_mmap2 = FALSE;
|
|
+ goto try_again;
|
|
+ }
|
|
+
|
|
+ if (use_software == FALSE)
|
|
+ {
|
|
+ g_clear_error (error);
|
|
+ with_mmap2 = TRUE;
|
|
+ use_software = TRUE;
|
|
+ goto try_again;
|
|
+ }
|
|
+
|
|
+ g_assert (*error != NULL);
|
|
+
|
|
+ return -1;
|
|
+}
|
|
+
|
|
+static DexFuture *
|
|
+main_fiber (gpointer user_data)
|
|
+{
|
|
+ g_autoptr(GSubprocessLauncher) launcher = NULL;
|
|
+ g_autoptr(GDBusConnection) bus = NULL;
|
|
+ g_autoptr(GSubprocess) subprocess = NULL;
|
|
+ g_autoptr(GPtrArray) argv = NULL;
|
|
+ g_autoptr(GError) error = NULL;
|
|
+ g_autofd int writer_fd = -1;
|
|
+ int n_cpu = g_get_num_processors ();
|
|
+ int next_target_fd = 3;
|
|
+
|
|
+ /* Get our bus we will use for authorization */
|
|
+ if (!(bus = dex_await_object (dex_bus_get (G_BUS_TYPE_SYSTEM), &error)))
|
|
+ return dex_future_new_for_error (g_steal_pointer (&error));
|
|
+
|
|
+ /* Setup our launcher we'll use to map FDs into the translator */
|
|
+ launcher = g_subprocess_launcher_new (0);
|
|
+
|
|
+ /* Setup our argv which will notify the child about where to
|
|
+ * find the FDs containing perf event streams.
|
|
+ */
|
|
+ argv = g_ptr_array_new_with_free_func (g_free);
|
|
+ g_ptr_array_add (argv, g_build_filename (BUILDDIR, "..", "sysprof-live-unwinder", NULL));
|
|
+
|
|
+ /* Provide kallsyms from the file provided */
|
|
+ if (kallsyms != NULL)
|
|
+ {
|
|
+ int fd = open (kallsyms, O_RDONLY | O_CLOEXEC);
|
|
+
|
|
+ if (fd != -1)
|
|
+ {
|
|
+ g_subprocess_launcher_take_fd (launcher, fd, next_target_fd);
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--kallsyms=%d", next_target_fd++));
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (sample_stack)
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--stack-size=%u", sample_stack_size));
|
|
+
|
|
+ g_printerr ("sysprof-live-unwinder at %s\n", (const char *)argv->pdata[0]);
|
|
+
|
|
+ /* First try to open a perf_event stream for as many CPUs as we
|
|
+ * can before we get complaints from the kernel.
|
|
+ */
|
|
+ for (int cpu = 0; cpu < n_cpu; cpu++)
|
|
+ {
|
|
+ g_autoptr(GError) cpu_error = NULL;
|
|
+ g_autofd int perf_fd = open_perf_stream (bus, cpu, &cpu_error);
|
|
+
|
|
+ if (perf_fd == -1)
|
|
+ {
|
|
+ g_printerr ("CPU[%d]: %s\n", cpu, cpu_error->message);
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ if (0 != ioctl (perf_fd, PERF_EVENT_IOC_ENABLE))
|
|
+ {
|
|
+ int errsv = errno;
|
|
+ g_warning ("Failed to enable perf_fd: %s", g_strerror (errsv));
|
|
+ }
|
|
+
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--perf-fd=%d:%d", next_target_fd, cpu));
|
|
+ g_subprocess_launcher_take_fd (launcher, g_steal_fd (&perf_fd), next_target_fd);
|
|
+
|
|
+ next_target_fd++;
|
|
+ }
|
|
+
|
|
+ /* Now create a FD for our destination capture. */
|
|
+ if (-1 == (writer_fd = open ("translated.syscap", O_CREAT|O_RDWR|O_CLOEXEC, 0664)) ||
|
|
+ ftruncate (writer_fd, 0) != 0)
|
|
+ return dex_future_new_for_errno (errno);
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--capture-fd=%d", next_target_fd));
|
|
+ g_subprocess_launcher_take_fd (launcher, g_steal_fd (&writer_fd), next_target_fd);
|
|
+ next_target_fd++;
|
|
+
|
|
+ /* Null-terminate our argv */
|
|
+ g_ptr_array_add (argv, NULL);
|
|
+
|
|
+ /* Spawn our worker process with the perf FDs and writer provided */
|
|
+ if (!(subprocess = g_subprocess_launcher_spawnv (launcher,
|
|
+ (const char * const *)argv->pdata,
|
|
+ &error)))
|
|
+ return dex_future_new_for_error (g_steal_pointer (&error));
|
|
+
|
|
+ /* Now wait for the translation process to complete */
|
|
+ if (!dex_await_boolean (dex_subprocess_wait_check (subprocess), &error))
|
|
+ return dex_future_new_for_error (g_steal_pointer (&error));
|
|
+
|
|
+ return dex_future_new_true ();
|
|
+}
|
|
+
|
|
+static DexFuture *
|
|
+finally_cb (DexFuture *future,
|
|
+ gpointer user_data)
|
|
+{
|
|
+ g_autoptr(GError) error = NULL;
|
|
+ GMainLoop *main_loop = user_data;
|
|
+
|
|
+ if (!dex_await (dex_ref (future), &error))
|
|
+ {
|
|
+ g_printerr ("Error: %s\n", error->message);
|
|
+ exit (EXIT_FAILURE);
|
|
+ }
|
|
+
|
|
+ g_main_loop_quit (main_loop);
|
|
+
|
|
+ return NULL;
|
|
+}
|
|
+
|
|
+int
|
|
+main (int argc,
|
|
+ char *argv[])
|
|
+{
|
|
+#if HAVE_POLKIT_AGENT
|
|
+ PolkitAgentListener *polkit = NULL;
|
|
+ PolkitSubject *subject = NULL;
|
|
+#endif
|
|
+ g_autoptr(GOptionContext) context = NULL;
|
|
+ g_autoptr(GMainLoop) main_loop = NULL;
|
|
+ g_autoptr(GError) error = NULL;
|
|
+ DexFuture *future;
|
|
+ GOptionEntry entries[] = {
|
|
+ { "sample-stack", 's', 0, G_OPTION_ARG_NONE, &sample_stack, "If the stack should be sampled for user-space unwinding" },
|
|
+ { "sample-stack-size", 'S', 0, G_OPTION_ARG_INT, &sample_stack_size, "If size of the stack to sample in bytes" },
|
|
+ { "kallsyms", 'k', 0, G_OPTION_ARG_FILENAME, &kallsyms, "Specify kallsyms for use" },
|
|
+ { NULL }
|
|
+ };
|
|
+
|
|
+ sysprof_clock_init ();
|
|
+ dex_init ();
|
|
+
|
|
+ main_loop = g_main_loop_new (NULL, FALSE);
|
|
+ context = g_option_context_new ("- test sysprof-live-unwinder");
|
|
+ g_option_context_add_main_entries (context, entries, NULL);
|
|
+
|
|
+ if (!g_option_context_parse (context, &argc, &argv, &error))
|
|
+ {
|
|
+ g_printerr ("%s\n", error->message);
|
|
+ return EXIT_FAILURE;
|
|
+ }
|
|
+
|
|
+#if HAVE_POLKIT_AGENT
|
|
+ /* Start polkit agent so that we can elevate privileges from a TTY */
|
|
+ if (g_getenv ("DESKTOP_SESSION") == NULL &&
|
|
+ (subject = polkit_unix_process_new_for_owner (getpid (), 0, -1)))
|
|
+ {
|
|
+ g_autoptr(GError) pkerror = NULL;
|
|
+
|
|
+ polkit = polkit_agent_text_listener_new (NULL, NULL);
|
|
+ polkit_agent_listener_register (polkit,
|
|
+ POLKIT_AGENT_REGISTER_FLAGS_NONE,
|
|
+ subject,
|
|
+ NULL,
|
|
+ NULL,
|
|
+ &pkerror);
|
|
+
|
|
+ if (pkerror != NULL)
|
|
+ {
|
|
+ g_dbus_error_strip_remote_error (pkerror);
|
|
+ g_printerr ("Failed to register polkit agent: %s\n",
|
|
+ pkerror->message);
|
|
+ }
|
|
+ }
|
|
+#endif
|
|
+
|
|
+ future = dex_scheduler_spawn (NULL, 0, main_fiber, NULL, NULL);
|
|
+ future = dex_future_finally (future,
|
|
+ finally_cb,
|
|
+ g_main_loop_ref (main_loop),
|
|
+ (GDestroyNotify) g_main_loop_unref);
|
|
+ dex_future_disown (future);
|
|
+
|
|
+ g_main_loop_run (main_loop);
|
|
+
|
|
+ g_clear_pointer (&kallsyms, g_free);
|
|
+
|
|
+ return EXIT_SUCCESS;
|
|
+}
|
|
diff --git a/src/sysprofd/ipc-unwinder-impl.c b/src/sysprofd/ipc-unwinder-impl.c
|
|
new file mode 100644
|
|
index 00000000..7f218de6
|
|
--- /dev/null
|
|
+++ b/src/sysprofd/ipc-unwinder-impl.c
|
|
@@ -0,0 +1,247 @@
|
|
+/*
|
|
+ * ipc-unwinder-impl.c
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#define G_LOG_DOMAIN "ipc-unwinder-impl"
|
|
+
|
|
+#include "config.h"
|
|
+
|
|
+#include <errno.h>
|
|
+
|
|
+#include <signal.h>
|
|
+#include <sys/prctl.h>
|
|
+#include <sys/socket.h>
|
|
+
|
|
+#include <glib/gstdio.h>
|
|
+
|
|
+#include <polkit/polkit.h>
|
|
+
|
|
+#include "ipc-unwinder-impl.h"
|
|
+
|
|
+struct _IpcUnwinderImpl
|
|
+{
|
|
+ IpcUnwinderSkeleton parent_instance;
|
|
+};
|
|
+
|
|
+static void
|
|
+child_setup (gpointer data)
|
|
+{
|
|
+ prctl (PR_SET_PDEATHSIG, SIGKILL);
|
|
+}
|
|
+
|
|
+static gboolean
|
|
+ipc_unwinder_impl_handle_unwind (IpcUnwinder *unwinder,
|
|
+ GDBusMethodInvocation *invocation,
|
|
+ GUnixFDList *fd_list,
|
|
+ guint stack_size,
|
|
+ GVariant *arg_perf_fds,
|
|
+ GVariant *arg_event_fd)
|
|
+{
|
|
+ g_autoptr(GSubprocessLauncher) launcher = NULL;
|
|
+ g_autoptr(GSubprocess) subprocess = NULL;
|
|
+ g_autoptr(GUnixFDList) out_fd_list = NULL;
|
|
+ g_autoptr(GPtrArray) argv = NULL;
|
|
+ g_autoptr(GError) error = NULL;
|
|
+ g_autofd int our_fd = -1;
|
|
+ g_autofd int their_fd = -1;
|
|
+ g_autofd int event_fd = -1;
|
|
+ GVariantIter iter;
|
|
+ int capture_fd_handle;
|
|
+ int pair[2];
|
|
+ int next_target_fd = 3;
|
|
+ int perf_fd_handle;
|
|
+ int cpu;
|
|
+
|
|
+ g_assert (IPC_IS_UNWINDER_IMPL (unwinder));
|
|
+ g_assert (G_IS_DBUS_METHOD_INVOCATION (invocation));
|
|
+ g_assert (!fd_list || G_IS_UNIX_FD_LIST (fd_list));
|
|
+
|
|
+ if (stack_size == 0 || stack_size % sysconf (_SC_PAGESIZE) != 0)
|
|
+ {
|
|
+ g_dbus_method_invocation_return_error_literal (g_steal_pointer (&invocation),
|
|
+ G_DBUS_ERROR,
|
|
+ G_DBUS_ERROR_INVALID_ARGS,
|
|
+ "Stack size must be a multiple of the page size");
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ if (fd_list == NULL)
|
|
+ {
|
|
+ g_dbus_method_invocation_return_error_literal (g_steal_pointer (&invocation),
|
|
+ G_DBUS_ERROR,
|
|
+ G_DBUS_ERROR_FILE_NOT_FOUND,
|
|
+ "Missing perf FDs");
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ launcher = g_subprocess_launcher_new (0);
|
|
+ argv = g_ptr_array_new_with_free_func (g_free);
|
|
+
|
|
+ g_ptr_array_add (argv, g_strdup (PACKAGE_LIBEXECDIR "/sysprof-live-unwinder"));
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--stack-size=%u", stack_size));
|
|
+
|
|
+ if (-1 == (event_fd = g_unix_fd_list_get (fd_list, g_variant_get_handle (arg_event_fd), &error)))
|
|
+ {
|
|
+ g_dbus_method_invocation_return_gerror (g_steal_pointer (&invocation), error);
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--event-fd=%u", next_target_fd));
|
|
+ g_subprocess_launcher_take_fd (launcher, g_steal_fd (&event_fd), next_target_fd++);
|
|
+
|
|
+ g_variant_iter_init (&iter, arg_perf_fds);
|
|
+
|
|
+ while (g_variant_iter_loop (&iter, "(hi)", &perf_fd_handle, &cpu))
|
|
+ {
|
|
+ g_autofd int perf_fd = g_unix_fd_list_get (fd_list, perf_fd_handle, &error);
|
|
+
|
|
+ if (perf_fd < 0)
|
|
+ {
|
|
+ g_dbus_method_invocation_return_gerror (g_steal_pointer (&invocation), error);
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--perf-fd=%d:%d", next_target_fd, cpu));
|
|
+ g_subprocess_launcher_take_fd (launcher,
|
|
+ g_steal_fd (&perf_fd),
|
|
+ next_target_fd++);
|
|
+ }
|
|
+
|
|
+ g_subprocess_launcher_set_child_setup (launcher, child_setup, NULL, NULL);
|
|
+
|
|
+ if (socketpair (AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, pair) < 0)
|
|
+ {
|
|
+ int errsv = errno;
|
|
+ g_dbus_method_invocation_return_error_literal (g_steal_pointer (&invocation),
|
|
+ G_IO_ERROR,
|
|
+ g_io_error_from_errno (errsv),
|
|
+ g_strerror (errsv));
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ our_fd = g_steal_fd (&pair[0]);
|
|
+ their_fd = g_steal_fd (&pair[1]);
|
|
+
|
|
+ out_fd_list = g_unix_fd_list_new ();
|
|
+ capture_fd_handle = g_unix_fd_list_append (out_fd_list, their_fd, &error);
|
|
+
|
|
+ if (capture_fd_handle < 0)
|
|
+ {
|
|
+ g_dbus_method_invocation_return_gerror (g_steal_pointer (&invocation), error);
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ g_ptr_array_add (argv, g_strdup_printf ("--capture-fd=%d", next_target_fd));
|
|
+ g_subprocess_launcher_take_fd (launcher, g_steal_fd (&our_fd), next_target_fd++);
|
|
+
|
|
+ g_ptr_array_add (argv, NULL);
|
|
+
|
|
+ if (!(subprocess = g_subprocess_launcher_spawnv (launcher, (const char * const *)argv->pdata, &error)))
|
|
+ {
|
|
+ g_dbus_method_invocation_return_gerror (g_steal_pointer (&invocation), error);
|
|
+ return TRUE;
|
|
+ }
|
|
+
|
|
+ ipc_unwinder_complete_unwind (unwinder,
|
|
+ g_steal_pointer (&invocation),
|
|
+ out_fd_list,
|
|
+ g_variant_new_handle (capture_fd_handle));
|
|
+
|
|
+ g_subprocess_wait_async (subprocess, NULL, NULL, NULL);
|
|
+
|
|
+ return TRUE;
|
|
+}
|
|
+
|
|
+static void
|
|
+unwinder_iface_init (IpcUnwinderIface *iface)
|
|
+{
|
|
+ iface->handle_unwind = ipc_unwinder_impl_handle_unwind;
|
|
+}
|
|
+
|
|
+G_DEFINE_FINAL_TYPE_WITH_CODE (IpcUnwinderImpl, ipc_unwinder_impl, IPC_TYPE_UNWINDER_SKELETON,
|
|
+ G_IMPLEMENT_INTERFACE (IPC_TYPE_UNWINDER, unwinder_iface_init))
|
|
+
|
|
+static gboolean
|
|
+ipc_unwinder_impl_g_authorize_method (GDBusInterfaceSkeleton *skeleton,
|
|
+ GDBusMethodInvocation *invocation)
|
|
+{
|
|
+ PolkitAuthorizationResult *res = NULL;
|
|
+ PolkitAuthority *authority = NULL;
|
|
+ PolkitSubject *subject = NULL;
|
|
+ const gchar *peer_name;
|
|
+ gboolean ret = TRUE;
|
|
+
|
|
+ g_assert (IPC_IS_UNWINDER_IMPL (skeleton));
|
|
+ g_assert (G_IS_DBUS_METHOD_INVOCATION (invocation));
|
|
+
|
|
+ peer_name = g_dbus_method_invocation_get_sender (invocation);
|
|
+
|
|
+ if (!(authority = polkit_authority_get_sync (NULL, NULL)) ||
|
|
+ !(subject = polkit_system_bus_name_new (peer_name)) ||
|
|
+ !(res = polkit_authority_check_authorization_sync (authority,
|
|
+ POLKIT_SUBJECT (subject),
|
|
+ "org.gnome.sysprof3.profile",
|
|
+ NULL,
|
|
+ POLKIT_CHECK_AUTHORIZATION_FLAGS_ALLOW_USER_INTERACTION,
|
|
+ NULL,
|
|
+ NULL)) ||
|
|
+ !polkit_authorization_result_get_is_authorized (res))
|
|
+ {
|
|
+ g_dbus_method_invocation_return_error (g_steal_pointer (&invocation),
|
|
+ G_DBUS_ERROR,
|
|
+ G_DBUS_ERROR_ACCESS_DENIED,
|
|
+ "Not authorized to make request");
|
|
+ ret = FALSE;
|
|
+ }
|
|
+
|
|
+ g_clear_object (&authority);
|
|
+ g_clear_object (&subject);
|
|
+ g_clear_object (&res);
|
|
+
|
|
+ return ret;
|
|
+}
|
|
+
|
|
+static void
|
|
+ipc_unwinder_impl_finalize (GObject *object)
|
|
+{
|
|
+ G_OBJECT_CLASS (ipc_unwinder_impl_parent_class)->finalize (object);
|
|
+}
|
|
+
|
|
+static void
|
|
+ipc_unwinder_impl_class_init (IpcUnwinderImplClass *klass)
|
|
+{
|
|
+ GObjectClass *object_class = G_OBJECT_CLASS (klass);
|
|
+ GDBusInterfaceSkeletonClass *skeleton_class = G_DBUS_INTERFACE_SKELETON_CLASS (klass);
|
|
+
|
|
+ object_class->finalize = ipc_unwinder_impl_finalize;
|
|
+
|
|
+ skeleton_class->g_authorize_method = ipc_unwinder_impl_g_authorize_method;
|
|
+}
|
|
+
|
|
+static void
|
|
+ipc_unwinder_impl_init (IpcUnwinderImpl *self)
|
|
+{
|
|
+}
|
|
+
|
|
+IpcUnwinder *
|
|
+ipc_unwinder_impl_new (void)
|
|
+{
|
|
+ return g_object_new (IPC_TYPE_UNWINDER_IMPL, NULL);
|
|
+}
|
|
diff --git a/src/sysprofd/ipc-unwinder-impl.h b/src/sysprofd/ipc-unwinder-impl.h
|
|
new file mode 100644
|
|
index 00000000..ebe7a2f0
|
|
--- /dev/null
|
|
+++ b/src/sysprofd/ipc-unwinder-impl.h
|
|
@@ -0,0 +1,34 @@
|
|
+/*
|
|
+ * ipc-unwinder-impl.h
|
|
+ *
|
|
+ * Copyright 2024 Christian Hergert <chergert@redhat.com>
|
|
+ *
|
|
+ * This program is free software: you can redistribute it and/or modify
|
|
+ * it under the terms of the GNU General Public License as published by
|
|
+ * the Free Software Foundation, either version 3 of the License, or
|
|
+ * (at your option) any later version.
|
|
+ *
|
|
+ * This program is distributed in the hope that it will be useful,
|
|
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
+ * GNU General Public License for more details.
|
|
+ *
|
|
+ * You should have received a copy of the GNU General Public License
|
|
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
+ *
|
|
+ * SPDX-License-Identifier: GPL-3.0-or-later
|
|
+ */
|
|
+
|
|
+#pragma once
|
|
+
|
|
+#include "ipc-unwinder.h"
|
|
+
|
|
+G_BEGIN_DECLS
|
|
+
|
|
+#define IPC_TYPE_UNWINDER_IMPL (ipc_unwinder_impl_get_type())
|
|
+
|
|
+G_DECLARE_FINAL_TYPE (IpcUnwinderImpl, ipc_unwinder_impl, IPC, UNWINDER_IMPL, IpcUnwinderSkeleton)
|
|
+
|
|
+IpcUnwinder *ipc_unwinder_impl_new (void);
|
|
+
|
|
+G_END_DECLS
|
|
diff --git a/src/sysprofd/meson.build b/src/sysprofd/meson.build
|
|
index 34901f44..b7e87ebe 100644
|
|
--- a/src/sysprofd/meson.build
|
|
+++ b/src/sysprofd/meson.build
|
|
@@ -10,18 +10,24 @@ ipc_service_src = gnome.gdbus_codegen('ipc-service',
|
|
namespace: 'Ipc',
|
|
)
|
|
|
|
+ipc_unwinder_src = gnome.gdbus_codegen('ipc-unwinder',
|
|
+ sources: 'org.gnome.Sysprof3.Unwinder.xml',
|
|
+ interface_prefix: 'org.gnome.Sysprof3.',
|
|
+ namespace: 'Ipc',
|
|
+)
|
|
+
|
|
sysprofd_sources = [
|
|
'sysprofd.c',
|
|
'ipc-rapl-profiler.c',
|
|
'ipc-service-impl.c',
|
|
+ 'ipc-unwinder-impl.c',
|
|
'sysprof-turbostat.c',
|
|
'helpers.c',
|
|
ipc_profiler_src,
|
|
ipc_service_src,
|
|
+ ipc_unwinder_src,
|
|
]
|
|
|
|
-pkglibexecdir = join_paths(get_option('prefix'), get_option('libexecdir'))
|
|
-
|
|
sysprofd_deps = [
|
|
glib_dep,
|
|
gio_dep,
|
|
diff --git a/src/sysprofd/org.gnome.Sysprof3.Unwinder.xml b/src/sysprofd/org.gnome.Sysprof3.Unwinder.xml
|
|
new file mode 100644
|
|
index 00000000..fb2c7848
|
|
--- /dev/null
|
|
+++ b/src/sysprofd/org.gnome.Sysprof3.Unwinder.xml
|
|
@@ -0,0 +1,23 @@
|
|
+<!DOCTYPE node PUBLIC "-//freedesktop//DTD D-BUS Object Introspection 1.0//EN"
|
|
+"http://www.freedesktop.org/standards/dbus/1.0/introspect.dtd">
|
|
+<node>
|
|
+ <interface name="org.gnome.Sysprof3.Unwinder">
|
|
+ <!--
|
|
+ Unwind:
|
|
+ @stack_size: the size of stacks that are sampled
|
|
+ @perf_fds: an array of (perf_fd, CPU number)
|
|
+ @event_fd: an event fd to write to for notifying unwinder should exit
|
|
+ @capture_fd: (out): a FD that will be written to containing samples
|
|
+
|
|
+ Unwinding will stop when capture_fd can no longer be written to
|
|
+ such as being closed by the consumer of this API.
|
|
+ -->
|
|
+ <method name="Unwind">
|
|
+ <annotation name="org.gtk.GDBus.C.UnixFD" value="true"/>
|
|
+ <arg type="u" name="stack_size" direction="in"/>
|
|
+ <arg type="a(hi)" name="perf_fds" direction="in"/>
|
|
+ <arg type="h" name="event_fd" direction="in"/>
|
|
+ <arg type="h" name="capture_fd" direction="out"/>
|
|
+ </method>
|
|
+ </interface>
|
|
+</node>
|
|
diff --git a/src/sysprofd/sysprofd.c b/src/sysprofd/sysprofd.c
|
|
index e39d59c4..7a7101e8 100644
|
|
--- a/src/sysprofd/sysprofd.c
|
|
+++ b/src/sysprofd/sysprofd.c
|
|
@@ -30,9 +30,11 @@
|
|
|
|
#include "ipc-rapl-profiler.h"
|
|
#include "ipc-service-impl.h"
|
|
+#include "ipc-unwinder-impl.h"
|
|
|
|
#define V3_PATH "/org/gnome/Sysprof3"
|
|
#define RAPL_PATH "/org/gnome/Sysprof3/RAPL"
|
|
+#define UNWINDER_PATH "/org/gnome/Sysprof3/Unwinder"
|
|
#define NAME_ACQUIRE_DELAY_SECS 3
|
|
#define INACTIVITY_TIMEOUT_SECS 120
|
|
|
|
@@ -126,6 +128,7 @@ main (gint argc,
|
|
{
|
|
g_autoptr(IpcProfiler) rapl = ipc_rapl_profiler_new ();
|
|
g_autoptr(IpcService) v3_service = ipc_service_impl_new ();
|
|
+ g_autoptr(IpcUnwinder) unwinder = ipc_unwinder_impl_new ();
|
|
|
|
g_signal_connect (v3_service, "activity", G_CALLBACK (activity_cb), NULL);
|
|
g_signal_connect (rapl, "activity", G_CALLBACK (activity_cb), NULL);
|
|
@@ -133,7 +136,8 @@ main (gint argc,
|
|
activity_cb (NULL, NULL);
|
|
|
|
if (g_dbus_interface_skeleton_export (G_DBUS_INTERFACE_SKELETON (v3_service), bus, V3_PATH, &error) &&
|
|
- g_dbus_interface_skeleton_export (G_DBUS_INTERFACE_SKELETON (rapl), bus, RAPL_PATH, &error))
|
|
+ g_dbus_interface_skeleton_export (G_DBUS_INTERFACE_SKELETON (rapl), bus, RAPL_PATH, &error) &&
|
|
+ g_dbus_interface_skeleton_export (G_DBUS_INTERFACE_SKELETON (unwinder), bus, UNWINDER_PATH, &error))
|
|
{
|
|
for (guint i = 0; i < G_N_ELEMENTS (bus_names); i++)
|
|
{
|
|
--
|
|
2.45.2
|
|
|