Skip to content

Commit 3fd18c0

Browse files
author
lexmach
committed
add basic ARM64 support
commit_hash:ac6f12aba192034e41c01a7c18c030f277bb7e6c
1 parent a4e4c79 commit 3fd18c0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+918
-320
lines changed

‎perforator/agent/collector/pkg/dso/bpf/unwindtable/manager.go‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -579,7 +579,7 @@ func (b *pageTableBuilder) flushNodes() error {
579579
}
580580

581581
// FIXME(sskvor): Generate this from the BTF
582-
const dwarfUnwindRBPRuleUndefined = 0x7f
582+
const dwarfUnwindRuleUndefined = 0x7f
583583

584584
func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int) {
585585
rule := unwinder.UnwindRule{}
@@ -588,8 +588,10 @@ func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int) {
588588
if rbp := row.RBP(); rbp != nil && rbp.GetCfaPlusOffset() != nil {
589589
offset := rbp.GetCfaPlusOffset().GetOffset()
590590
rule.Rbp = unwinder.RbpUnwindRule{Offset: int8(offset)}
591+
} else if rbp != nil && rbp.GetCfaMinus8() != nil {
592+
rule.Rbp = unwinder.RbpUnwindRule{Offset: int8(-8)}
591593
} else {
592-
rule.Rbp = unwinder.RbpUnwindRule{Offset: dwarfUnwindRBPRuleUndefined}
594+
rule.Rbp = unwinder.RbpUnwindRule{Offset: dwarfUnwindRuleUndefined}
593595
}
594596

595597
// Fill CFA rule
@@ -601,6 +603,16 @@ func fillRule(row row, page *unwinder.UnwindTablePageLeaf, idx int) {
601603
rule.Cfa.Kind = unwinder.UnwindRuleUnsupported
602604
}
603605

606+
// Should we really use separate rule for CfaMinus8?
607+
if ra := row.RA(); ra != nil && ra.GetCfaPlusOffset() != nil {
608+
offset := ra.GetCfaPlusOffset().GetOffset()
609+
rule.Ra = unwinder.RaUnwindRule{Offset: int8(offset)}
610+
} else if ra != nil && ra.GetCfaMinus8() != nil {
611+
rule.Ra = unwinder.RaUnwindRule{Offset: int8(-8)}
612+
} else {
613+
rule.Ra = unwinder.RaUnwindRule{Offset: dwarfUnwindRuleUndefined}
614+
}
615+
604616
page.Pc[idx] = uint32(row.StartPC())
605617
page.Ranges[idx] = uint32(row.PCRange())
606618
page.Rules[idx] = rule

‎perforator/agent/collector/progs/unwinder/arch/arm/dwarf.h‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
#pragma once
2+
3+
#include <bpf/bpf.h>
4+
5+
#include "../../dwarf-fwd.h"
6+
7+
////////////////////////////////////////////////////////////////////////////////
8+
9+
enum {
10+
DWARF_CFI_UNKNOWN_REGISTER = 0xfffffffffffffffd,
11+
12+
DWARF_CFI_STACK_REGISTER_NUMBER = 31,
13+
DWARF_CFI_FRAME_REGISTER_NUMBER = 29
14+
};
15+
16+
struct dwarf_cfi_context {
17+
u64 cfa;
18+
u64 fp;
19+
u64 ip;
20+
21+
u64 lr;
22+
};
23+
24+
static void ALWAYS_INLINE dwarf_cfi_context_init_next(struct dwarf_cfi_context* ctx) {
25+
ctx->cfa = DWARF_CFI_UNKNOWN_REGISTER;
26+
ctx->fp = DWARF_CFI_UNKNOWN_REGISTER;
27+
ctx->pc = DWARF_CFI_UNKNOWN_REGISTER;
28+
ctx->lr = DWARF_CFI_UNKNOWN_REGISTER;
29+
}
30+
31+
static ALWAYS_INLINE void dwarf_unwind_setup_userspace_registers(
32+
struct dwarf_cfi_context* cfi,
33+
struct user_regs* regs
34+
) {
35+
cfi->cfa = regs->sp;
36+
cfi->fp = regs->fp;
37+
cfi->ip = regs->pc;
38+
39+
cfi->lr = regs->lr;
40+
}
41+
42+
////////////////////////////////////////////////////////////////////////////////
43+
44+
// FIXME: we are fully dependendent on Link register here, but it might be invalidated
45+
// See: https://github.com/ARM-software/abi-aa/blob/main/aadwarf64/aadwarf64.rst#note-8
46+
// Currently not a single observed binary has this DWARF expression
47+
ALWAYS_INLINE bool dwarf_cfi_eval_ra(
48+
struct dwarf_cfi_context* prev,
49+
struct dwarf_cfi_context* next,
50+
struct ra_unwind_rule* rule
51+
) {
52+
if (rule->offset == DWARF_UNWIND_CFA_RULE_UNDEFINED) {
53+
DWARF_TRACE("no need to update LR");
54+
next->lr = prev->lr;
55+
next->ip = prev->lr;
56+
return true;
57+
}
58+
59+
u64 address = next->cfa + rule->offset;
60+
if (!read_return_address((void*)address, &next->lr)) {
61+
return false;
62+
}
63+
next->ip = next->lr;
64+
65+
return true;
66+
}
67+
68+
////////////////////////////////////////////////////////////////////////////////
69+
70+
// There is no canonical prolouge/epilogue:
71+
// See: https://www.codalogic.com/blog/2022/10/20/Aarch64-Stack-Frames-Again
72+
//
73+
// Clang function prologue and epilogue:
74+
// foo:
75+
// sub sp, sp, #size
76+
// stp x29, x30, [sp, #(size - 16)]
77+
// add x29, sp, #(size - 16)
78+
// ...
79+
// ldp x29, x30, [sp, #(size - 16)]
80+
// add sp, sp, #size
81+
// ret
82+
//
83+
// Stack layout:
84+
// | |
85+
// +---------------------+
86+
// | lr |
87+
// +---------------------+
88+
// | original fp | <- fp
89+
// +---------------------+
90+
// | |
91+
// | |
92+
// | ...space... |
93+
// | |
94+
// | | <- sp
95+
// +---------------------+
96+
static NOINLINE enum dwarf_unwind_step_result dwarf_unwind_step_fp(struct dwarf_cfi_context* cfi, u32* framepointers) {
97+
if (cfi == NULL || framepointers == NULL) {
98+
return DWARF_UNWIND_STEP_FAILED;
99+
}
100+
101+
(*framepointers)++;
102+
103+
cfi->ip = cfi->lr;
104+
if (!read_return_address((void*)(cfi->fp + 8), &cfi->lr)) {
105+
metric_increment(METRIC_FP_ERROR_READ_RETURNADDRESS_COUNT);
106+
return DWARF_UNWIND_STEP_FAILED;
107+
}
108+
109+
cfi->cfa = cfi->fp + 16;
110+
111+
u64 prev_fp = 0;
112+
int err = bpf_probe_read_user(&prev_fp, sizeof(prev_fp), (void*)cfi->fp);
113+
if (err != 0) {
114+
DWARF_TRACE("fp: bpf_probe_read_user failed: %d\n", err);
115+
metric_increment(METRIC_FP_ERROR_READ_BASEPOINTER_COUNT);
116+
return DWARF_UNWIND_STEP_FAILED;
117+
}
118+
cfi->fp = prev_fp;
119+
120+
return DWARF_UNWIND_STEP_CONTINUE;
121+
}

‎perforator/agent/collector/progs/unwinder/arch/arm/regs.h‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#pragma once
2+
3+
#include <bpf/attrs.h>
4+
#include <bpf/core.h>
5+
#include <bpf/types.h>
6+
7+
#include "../../core.h"
8+
#include "../../linux.h"
9+
#include "../../task.h"
10+
11+
////////////////////////////////////////////////////////////////////////////////
12+
13+
struct user_regs {
14+
u64 sp;
15+
u64 fp;
16+
u64 pc;
17+
u64 lr;
18+
};
19+
20+
static ALWAYS_INLINE u64 regs_get_current_instruction(struct user_regs* regs) {
21+
return regs->pc;
22+
}
23+
24+
// FIXME: this is suboptimal, we are using only part of user-spaced regs
25+
struct pt_regs___kernel {
26+
u64 regs[31];
27+
u64 sp;
28+
u64 pc;
29+
};
30+
31+
////////////////////////////////////////////////////////////////////////////////
32+
33+
// See https://www.kernel.org/doc/Documentation/arm64/memory.txt
34+
ALWAYS_INLINE bool is_kernel_pc(u64 pc) {
35+
return pc > 0xffff000000000000;
36+
}
37+
38+
// See https://github.com/iovisor/bcc/issues/2073#issuecomment-446844179
39+
// And https://elixir.bootlin.com/linux/v5.4.254/source/arch/arm/include/asm/ptrace.h#L16
40+
// And https://elixir.bootlin.com/linux/v5.4.254/source/arch/arm/include/asm/processor.h#L98
41+
static NOINLINE bool extract_saved_userspace_registers(struct user_regs* regs) {
42+
struct pt_regs___kernel* kregs = 0;
43+
if (bpf_core_enum_value_exists(enum bpf_func_id, BPF_FUNC_task_pt_regs)) {
44+
struct task_struct* task = bpf_get_current_task_btf();
45+
if (task == NULL) {
46+
return false;
47+
}
48+
49+
kregs = (void*)bpf_task_pt_regs(task);
50+
if (kregs == 0) {
51+
return false;
52+
}
53+
} else {
54+
struct task_struct* task = get_current_task();
55+
if (task == NULL) {
56+
return false;
57+
}
58+
59+
void* ptr = BPF_CORE_READ(task, stack);
60+
if (ptr == NULL) {
61+
return false;
62+
}
63+
64+
ptr += THREAD_START_SP;
65+
kregs = (void*)((struct user_pt_regs*)(ptr) - 1);
66+
}
67+
68+
regs->pc = BPF_CORE_READ(kregs, pc);
69+
regs->fp = BPF_CORE_READ(kregs, ARM_FP);
70+
regs->sp = BPF_CORE_READ(kregs, sp);
71+
regs->lr = BPF_CORE_READ(kregs, ARM_LR);
72+
73+
return true;
74+
}
75+
76+
// FIXME: bpf_perf_event_data contains only user_pt_regs, not full regs
77+
static NOINLINE bool find_task_userspace_registers_bpf(struct user_pt_regs* kregs, struct user_regs* uregs) {
78+
if (is_kernel_pc(kregs->pc)) {
79+
return extract_saved_userspace_registers(uregs);
80+
}
81+
82+
uregs->sp = kregs->sp;
83+
uregs->fp = kregs->ARM_FP;
84+
uregs->pc = kregs->pc;
85+
uregs->lr = kregs->ARM_LR;
86+
87+
return true;
88+
}
89+
90+
static ALWAYS_INLINE bool find_task_userspace_registers(struct pt_regs* kregs, struct user_regs* uregs) {
91+
return find_task_userspace_registers_bpf(&kregs->user_regs, uregs);
92+
}

‎perforator/agent/collector/progs/unwinder/arch/arm/thread.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#pragma once
2+
3+
#include "../../core.h"
4+
5+
static ALWAYS_INLINE unsigned long get_tcb_pointer() {
6+
struct task_struct* task = (void*)bpf_get_current_task();
7+
return BPF_CORE_READ(task, thread.uw.tp_value);
8+
}

‎perforator/agent/collector/progs/unwinder/arch/x86/dwarf.h‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#pragma once
2+
3+
#include <bpf/bpf.h>
4+
5+
#include "../../dwarf-fwd.h"
6+
7+
////////////////////////////////////////////////////////////////////////////////
8+
9+
enum {
10+
DWARF_CFI_UNKNOWN_REGISTER = 0xfffffffffffffffd,
11+
12+
DWARF_CFI_STACK_REGISTER_NUMBER = 7,
13+
DWARF_CFI_FRAME_REGISTER_NUMBER = 6
14+
};
15+
16+
struct dwarf_cfi_context {
17+
u64 cfa;
18+
u64 fp;
19+
u64 ip;
20+
};
21+
22+
static void ALWAYS_INLINE dwarf_cfi_context_init_next(struct dwarf_cfi_context* ctx) {
23+
ctx->cfa = DWARF_CFI_UNKNOWN_REGISTER;
24+
ctx->fp = DWARF_CFI_UNKNOWN_REGISTER;
25+
ctx->ip = DWARF_CFI_UNKNOWN_REGISTER;
26+
}
27+
28+
static ALWAYS_INLINE void dwarf_unwind_setup_userspace_registers(
29+
struct dwarf_cfi_context* cfi,
30+
struct user_regs* regs
31+
) {
32+
cfi->cfa = regs->rsp;
33+
cfi->fp = regs->rbp;
34+
cfi->ip = regs->rip;
35+
}
36+
37+
////////////////////////////////////////////////////////////////////////////////
38+
39+
ALWAYS_INLINE bool dwarf_cfi_eval_ra(
40+
struct dwarf_cfi_context* prev,
41+
struct dwarf_cfi_context* next,
42+
struct ra_unwind_rule* rule
43+
) {
44+
u64 address = next->cfa - 8;
45+
return read_return_address((void*)address, &next->ip);
46+
}
47+
48+
// Canonical function prologue and epilogue:
49+
// foo:
50+
// push %rbp
51+
// mov %rsp, %rbp
52+
// ...
53+
// mov %rbp, %rsp
54+
// pop %rbp
55+
// ret
56+
//
57+
// Stack layout:
58+
// rsp0 -> [....]
59+
// rsp0-8 -> [ra0 ]
60+
// rsp0-16 -> [rbp0]
61+
// ...
62+
// rsp1 -> [....]
63+
// rsp1-8 -> [ra1 ]
64+
// rsp1-16 -> [rbp1]
65+
// ...
66+
// rsp2 -> [....]
67+
// rsp2-8 -> [ra2 ]
68+
// rsp2-16 -> [rbp2]
69+
static NOINLINE enum dwarf_unwind_step_result dwarf_unwind_step_fp(struct dwarf_cfi_context* cfi, u32* framepointers) {
70+
if (cfi == NULL || framepointers == NULL) {
71+
return DWARF_UNWIND_STEP_FAILED;
72+
}
73+
74+
(*framepointers)++;
75+
if (!read_return_address((void*)(cfi->fp + 8), &cfi->ip)) {
76+
metric_increment(METRIC_FP_ERROR_READ_RETURNADDRESS_COUNT);
77+
return DWARF_UNWIND_STEP_FAILED;
78+
}
79+
80+
// We need to restore rsp in order to support mixed dwarf and frame pointers unwinding.
81+
// Previous rsp is equal to current rbp + 2*sizeof(register): look at the stack layout.
82+
cfi->cfa = cfi->fp + 16;
83+
84+
u64 prev_rbp = 0;
85+
int err = bpf_probe_read_user(&prev_rbp, sizeof(prev_rbp), (void*)cfi->fp);
86+
if (err != 0) {
87+
DWARF_TRACE("fp: bpf_probe_read_user failed: %d\n", err);
88+
metric_increment(METRIC_FP_ERROR_READ_BASEPOINTER_COUNT);
89+
return DWARF_UNWIND_STEP_FAILED;
90+
}
91+
cfi->fp = prev_rbp;
92+
93+
return DWARF_UNWIND_STEP_CONTINUE;
94+
}

‎perforator/agent/collector/progs/unwinder/arch/x86/regs.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ struct user_regs {
1616
u64 rip;
1717
};
1818

19+
static ALWAYS_INLINE u64 regs_get_current_instruction(struct user_regs* regs) {
20+
return regs->rip;
21+
}
22+
1923
struct pt_regs___kernel {
2024
u64 ip;
2125
u64 sp;
@@ -78,4 +82,8 @@ static NOINLINE bool find_task_userspace_registers(struct pt_regs* kregs, struct
7882
return true;
7983
}
8084

85+
static ALWAYS_INLINE bool find_task_userspace_registers_bpf(struct pt_regs* kregs, struct user_regs* uregs) {
86+
return find_task_userspace_registers(kregs, uregs);
87+
}
88+
8189
////////////////////////////////////////////////////////////////////////////////

‎perforator/agent/collector/progs/unwinder/arch/x86/thread.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#pragma once
2+
3+
#include "../../core.h"
4+
5+
static ALWAYS_INLINE unsigned long get_tcb_pointer() {
6+
struct task_struct* task = (void*)bpf_get_current_task();
7+
return BPF_CORE_READ(task, thread.fsbase);
8+
}

‎perforator/agent/collector/progs/unwinder/binary.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ typedef u64 binary_id;
66

77
enum binary_storage_params : u32 {
88
MAX_BINARIES = 1024 * 1024
9-
};
9+
};

0 commit comments

Comments
 (0)