diff --git a/Documentation/compel.txt b/Documentation/compel.txt index 506228f592..cdca1e6229 100644 --- a/Documentation/compel.txt +++ b/Documentation/compel.txt @@ -93,7 +93,7 @@ The parasitic code is compiled and converted to a header using *compel*, and inc Following steps are performed to infect the victim process: - stop the task: *int compel_stop_task(int pid);* - - prepare infection handler: *struct parasite_ctl *compel_prepare(int pid);* + - prepare infection handler: *struct parasite_ctl *compel_prepare(int pid, bool handle_rseq);* - execute system call: *int compel_syscall(ctl, int syscall_nr, long *ret, int arg ...);* - infect victim: *int compel_infect(ctl, nr_thread, size_of_args_area);* - cure the victim: *int compel_cure(ctl);* //ctl pointer is freed by this call diff --git a/compel/include/uapi/infect.h b/compel/include/uapi/infect.h index 4bbf57c4db..f88d91b1e0 100644 --- a/compel/include/uapi/infect.h +++ b/compel/include/uapi/infect.h @@ -47,12 +47,12 @@ extern int compel_resume_task_sig(pid_t pid, int orig_state, int state, int stop struct parasite_ctl; struct parasite_thread_ctl; -extern struct parasite_ctl __must_check *compel_prepare(int pid); -extern struct parasite_ctl __must_check *compel_prepare_noctx(int pid); +extern struct parasite_ctl __must_check *compel_prepare(int pid, bool handle_rseq); +extern struct parasite_ctl __must_check *compel_prepare_noctx(int pid, bool handle_rseq); extern int __must_check compel_infect(struct parasite_ctl *ctl, unsigned long nr_threads, unsigned long args_size); extern int __must_check compel_infect_no_daemon(struct parasite_ctl *ctl, unsigned long nr_threads, unsigned long args_size); -extern struct parasite_thread_ctl __must_check *compel_prepare_thread(struct parasite_ctl *ctl, int pid); +extern struct parasite_thread_ctl __must_check *compel_prepare_thread(struct parasite_ctl *ctl, int pid, bool handle_rseq); extern void compel_release_thread(struct parasite_thread_ctl *); extern int __must_check compel_start_daemon(struct parasite_ctl *ctl); diff --git a/compel/src/lib/infect.c b/compel/src/lib/infect.c index 8f44dbfc3e..b4764ce836 100644 --- a/compel/src/lib/infect.c +++ b/compel/src/lib/infect.c @@ -14,6 +14,7 @@ #include "common/xmalloc.h" #include "common/lock.h" #include "common/page.h" +#include "linux/rseq.h" #include #include @@ -42,7 +43,7 @@ #define SECCOMP_MODE_DISABLED 0 #endif -static int prepare_thread(int pid, struct thread_ctx *ctx); +static int prepare_thread(int pid, struct thread_ctx *ctx, bool handle_rseq); static inline void close_safe(int *pfd) { @@ -1111,13 +1112,13 @@ int compel_infect(struct parasite_ctl *ctl, unsigned long nr_threads, unsigned l return 0; } -struct parasite_thread_ctl *compel_prepare_thread(struct parasite_ctl *ctl, int pid) +struct parasite_thread_ctl *compel_prepare_thread(struct parasite_ctl *ctl, int pid, bool handle_rseq) { struct parasite_thread_ctl *tctl; tctl = xmalloc(sizeof(*tctl)); if (tctl) { - if (prepare_thread(pid, &tctl->th)) { + if (prepare_thread(pid, &tctl->th, handle_rseq)) { xfree(tctl); tctl = NULL; } else { @@ -1129,7 +1130,117 @@ struct parasite_thread_ctl *compel_prepare_thread(struct parasite_ctl *ctl, int return tctl; } -static int prepare_thread(int pid, struct thread_ctx *ctx) +#define decode_pointer(x) ((void *)(unsigned long)(x)) + +static bool task_in_rseq(struct criu_rseq_cs *rseq_cs, uint64_t addr) +{ + return addr - rseq_cs->start_ip < rseq_cs->post_commit_offset; +} + +static int read_rseq_cs(pid_t tid, struct __ptrace_rseq_configuration *rseqc, struct criu_rseq_cs *rseq_cs, + struct criu_rseq *rseq) +{ + int ret; + + if (!rseqc->rseq_abi_pointer) + return 0; + + ret = ptrace_peek_area(tid, rseq, + decode_pointer(rseqc->rseq_abi_pointer), + sizeof(struct criu_rseq)); + if (ret) { + pr_err("ptrace_peek_area(%d, %lx, %lx, %lx): fail to read rseq struct\n", + tid, (unsigned long)rseq, (unsigned long)rseqc->rseq_abi_pointer, + (unsigned long)sizeof(struct criu_rseq)); + return -1; + } + + if (!rseq->rseq_cs) + return 0; + + ret = ptrace_peek_area(tid, rseq_cs, + decode_pointer(rseq->rseq_cs), + sizeof(struct criu_rseq_cs)); + if (ret) { + pr_err("ptrace_peek_area(%d, %lx, %lx, %lx): fail to read rseq_cs struct\n", + tid, (unsigned long)rseq_cs, (unsigned long)rseq->rseq_cs, + (unsigned long)sizeof(struct criu_rseq_cs)); + return -1; + } + + return 0; +} + +/* + * parasite_thread_rseq checks if the victim process is in an rseq critical + * section and, if so, aborts it. + */ +static int parasite_thread_rseq(int pid, struct thread_ctx *ctx) +{ + struct criu_rseq_cs _rseq_cs = {}, *rseq_cs = &_rseq_cs; + struct __ptrace_rseq_configuration rseqc; + int ret; + struct criu_rseq rseq = {}; + + ret = ptrace(PTRACE_GET_RSEQ_CONFIGURATION, pid, sizeof(rseqc), &rseqc); + if (ret < 0) { + pr_perror("ptrace(PTRACE_GET_RSEQ_CONFIGURATION, %d) = %d", pid, ret); + return -1; + } else if (ret != sizeof(rseqc)) { + pr_err("ptrace(PTRACE_GET_RSEQ_CONFIGURATION, %d) returned unexpected size %d\n", pid, ret); + return -1; + } + + if (rseqc.flags != 0) { + pr_err("something wrong with ptrace(PTRACE_GET_RSEQ_CONFIGURATION, %d) flags = 0x%x\n", pid, + rseqc.flags); + return -1; + } + + pr_debug("rseq of %d: ptr = 0x%lx sign = 0x%x\n", pid, (unsigned long)rseqc.rseq_abi_pointer, + rseqc.signature); + + if (read_rseq_cs(pid, &rseqc, rseq_cs, &rseq)) + return -1; + + if (rseq.rseq_cs) { + uint64_t zero_addr = 0; + + pr_debug( + "fixup_thread_rseq for %d: rseq_cs start_ip = %llx abort_ip = %llx post_commit_offset = %llx flags = %x version = %x; IP = %lx\n", + pid, rseq_cs->start_ip, rseq_cs->abort_ip, rseq_cs->post_commit_offset, rseq_cs->flags, + rseq_cs->version, (unsigned long)REG_IP(ctx->regs)); + + if (rseq_cs->version != 0) { + pr_err("unsupported RSEQ ABI version = %d\n", rseq_cs->version); + return -1; + } + + if (rseq.flags || rseq_cs->flags) + pr_warn("deprecated rseq flags are ignored for %d: rseq.flags = %#x rseq_cs.flags = %#x\n", + pid, rseq.flags, rseq_cs->flags); + + if (task_in_rseq(rseq_cs, REG_IP(ctx->regs))) { + SET_REG_IP(ctx->regs, rseq_cs->abort_ip); + if (ptrace_set_regs(pid, &ctx->regs)) { + pr_perror("Can't apply rseq abort registers (pid: %d)", pid); + return -1; + } + } + + if (ptrace_poke_area(pid, &zero_addr, + decode_pointer(rseqc.rseq_abi_pointer) + + offsetof(struct criu_rseq, rseq_cs), + sizeof(zero_addr))) { + pr_err("ptrace_poke_area(%d) failed to zero out rseq_cs\n", pid); + return -1; + } + } + + return 0; +} + +static int prepare_thread(int pid, struct thread_ctx *ctx, bool handle_rseq) { if (ptrace(PTRACE_GETSIGMASK, pid, sizeof(k_rtsigset_t), &ctx->sigmask)) { pr_perror("can't get signal blocking mask for %d", pid); @@ -1141,6 +1252,11 @@ static int prepare_thread(int pid, struct thread_ctx *ctx) return -1; } + if (handle_rseq && parasite_thread_rseq(pid, ctx)) { + pr_err("Can't handle rseq (pid: %d)\n", pid); + return -1; + } + return 0; } @@ -1153,7 +1269,7 @@ void compel_release_thread(struct parasite_thread_ctl *tctl) xfree(tctl); } -struct parasite_ctl *compel_prepare_noctx(int pid) +struct parasite_ctl *compel_prepare_noctx(int pid, bool handle_rseq) { struct parasite_ctl *ctl = NULL; @@ -1170,7 +1286,7 @@ struct parasite_ctl *compel_prepare_noctx(int pid) ctl->tsock = -1; ctl->ictx.log_fd = -1; - if (prepare_thread(pid, &ctl->orig)) + if (prepare_thread(pid, &ctl->orig, handle_rseq)) goto err; ctl->rpid = pid; @@ -1346,12 +1462,12 @@ static int make_sigframe_plain(void *from, struct rt_sigframe *f, struct rt_sigf return 0; } -struct parasite_ctl *compel_prepare(int pid) +struct parasite_ctl *compel_prepare(int pid, bool handle_rseq) { struct parasite_ctl *ctl; struct infect_ctx *ictx; - ctl = compel_prepare_noctx(pid); + ctl = compel_prepare_noctx(pid, handle_rseq); if (ctl == NULL) goto out; diff --git a/compel/test/fdspy/spy.c b/compel/test/fdspy/spy.c index 41de99e200..35a6805715 100644 --- a/compel/test/fdspy/spy.c +++ b/compel/test/fdspy/spy.c @@ -39,7 +39,7 @@ static int do_infection(int pid, int *stolen_fd) err_and_ret("Can't stop task"); printf("Preparing parasite ctl\n"); - ctl = compel_prepare(pid); + ctl = compel_prepare(pid, false); if (!ctl) err_and_ret("Can't prepare for infection"); diff --git a/compel/test/infect/spy.c b/compel/test/infect/spy.c index 143946941e..da9b2d729a 100644 --- a/compel/test/infect/spy.c +++ b/compel/test/infect/spy.c @@ -38,7 +38,7 @@ static int do_infection(int pid) err_and_ret("Can't stop task"); printf("Preparing parasite ctl\n"); - ctl = compel_prepare(pid); + ctl = compel_prepare(pid, false); if (!ctl) err_and_ret("Can't prepare for infection"); diff --git a/compel/test/rsys/spy.c b/compel/test/rsys/spy.c index 4a6fcef29c..ca5630332f 100644 --- a/compel/test/rsys/spy.c +++ b/compel/test/rsys/spy.c @@ -33,7 +33,7 @@ static int do_rsetsid(int pid) err_and_ret("Can't stop task"); printf("Preparing parasite ctl\n"); - ctl = compel_prepare(pid); + ctl = compel_prepare(pid, false); if (!ctl) err_and_ret("Can't prepare for infection"); diff --git a/compel/test/stack/spy.c b/compel/test/stack/spy.c index 184c8ab318..d0ac16e718 100644 --- a/compel/test/stack/spy.c +++ b/compel/test/stack/spy.c @@ -123,7 +123,7 @@ static int do_infection(int pid) err_and_ret("Can't stop task\n"); printf("Preparing parasite ctl\n"); - ctl = compel_prepare(pid); + ctl = compel_prepare(pid, false); if (!ctl) err_and_ret("Can't prepare for infection\n"); diff --git a/criu/cr-dump.c b/criu/cr-dump.c index 6fe7c42e4e..fe3a3edd38 100644 --- a/criu/cr-dump.c +++ b/criu/cr-dump.c @@ -30,6 +30,7 @@ #include "images/siginfo.pb-c.h" #include "common/list.h" +#include "linux/rseq.h" #include "imgset.h" #include "file-ids.h" #include "kcmp-ids.h" @@ -908,72 +909,6 @@ static int collect_file_locks(void) return parse_file_locks(); } -static bool task_in_rseq(struct criu_rseq_cs *rseq_cs, uint64_t addr) -{ - return addr >= rseq_cs->start_ip && addr < rseq_cs->start_ip + rseq_cs->post_commit_offset; -} - -static int fixup_thread_rseq(const struct pstree_item *item, int i) -{ - CoreEntry *core = item->core[i]; - struct criu_rseq_cs *rseq_cs = &dmpi(item)->thread_rseq_cs[i]; - pid_t tid = item->threads[i].real; - - if (!kdat.has_ptrace_get_rseq_conf) - return 0; - - /* equivalent to (struct rseq)->rseq_cs is NULL */ - if (!rseq_cs->start_ip) - return 0; - - pr_debug( - "fixup_thread_rseq for %d: rseq_cs start_ip = %llx abort_ip = %llx post_commit_offset = %llx flags = %x version = %x; IP = %lx\n", - tid, rseq_cs->start_ip, rseq_cs->abort_ip, rseq_cs->post_commit_offset, rseq_cs->flags, - rseq_cs->version, (unsigned long)TI_IP(core)); - - if (rseq_cs->version != 0) { - pr_err("unsupported RSEQ ABI version = %d\n", rseq_cs->version); - return -1; - } - - if (task_in_rseq(rseq_cs, TI_IP(core))) { - struct pid *tid = &item->threads[i]; - - /* - * We need to fixup task instruction pointer from - * the original one (which lays inside rseq critical section) - * to rseq abort handler address. But we need to look on rseq_cs->flags - * (please refer to struct rseq -> flags field description). - * Naive idea of flags support may be like... let's change instruction pointer (IP) - * to rseq_cs->abort_ip if !(rseq_cs->flags & RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL). - * But unfortunately, it doesn't work properly, because the kernel does - * clean up of rseq_cs field in the struct rseq (modifies userspace memory). - * So, we need to preserve original value of (struct rseq)->rseq_cs field in the - * image and restore it's value before releasing threads (see restore_rseq_cs()). - * - * It's worth to mention that we need to fixup IP in CoreEntry - * (used when full dump/restore is performed) and also in - * the parasite regs storage (used if --leave-running option is used, - * or if dump error occurred and process execution is resumed). - */ - - if (!(rseq_cs->flags & RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL)) { - pr_warn("The %d task is in rseq critical section. IP will be set to rseq abort handler addr\n", - tid->real); - - TI_IP(core) = rseq_cs->abort_ip; - - if (item->pid->real == tid->real) { - compel_set_leader_ip(dmpi(item)->parasite_ctl, rseq_cs->abort_ip); - } else { - compel_set_thread_ip(dmpi(item)->thread_ctls[i], rseq_cs->abort_ip); - } - } - } - - return 0; -} - static int dump_task_thread(struct parasite_ctl *parasite_ctl, const struct pstree_item *item, int id) { struct parasite_thread_ctl *tctl = dmpi(item)->thread_ctls[id]; @@ -997,12 +932,6 @@ static int dump_task_thread(struct parasite_ctl *parasite_ctl, const struct pstr core->thread_core->creds->lsm_profile = dmpi(item)->thread_lsms[id]->profile; core->thread_core->creds->lsm_sockcreate = dmpi(item)->thread_lsms[0]->sockcreate; - ret = fixup_thread_rseq(item, id); - if (ret) { - pr_err("Can't fixup rseq for pid %d\n", pid); - goto err; - } - img = open_image(CR_FD_CORE, O_DUMP, tid->ns[0].virt); if (!img) goto err; @@ -1148,48 +1077,6 @@ static int dump_task_signals(pid_t pid, struct pstree_item *item) return 0; } -static int read_rseq_cs(pid_t tid, struct __ptrace_rseq_configuration *rseqc, struct criu_rseq_cs *rseq_cs, - struct criu_rseq *rseq) -{ - int ret; - - /* rseq is not registered */ - if (!rseqc->rseq_abi_pointer) - return 0; - - /* - * We need to cover the case when victim process was inside rseq critical section - * at the moment when CRIU comes and seized it. We need to determine the borders - * of rseq critical section at first. To achieve that we need to access thread - * memory and read pointer to struct rseq_cs. - * - * We have two ways to access thread memory: from the parasite and using ptrace(). - * But it this case we can't use parasite, because if victim process returns to the - * execution, on the kernel side __rseq_handle_notify_resume hook will be called, - * then rseq_ip_fixup() -> clear_rseq_cs() and user space memory with struct rseq - * will be cleared. So, let's use ptrace(PTRACE_PEEKDATA). - */ - ret = ptrace_peek_area(tid, rseq, decode_pointer(rseqc->rseq_abi_pointer), sizeof(struct criu_rseq)); - if (ret) { - pr_err("ptrace_peek_area(%d, %lx, %lx, %lx): fail to read rseq struct\n", tid, (unsigned long)rseq, - (unsigned long)(rseqc->rseq_abi_pointer), (unsigned long)sizeof(uint64_t)); - return -1; - } - - if (!rseq->rseq_cs) - return 0; - - ret = ptrace_peek_area(tid, rseq_cs, decode_pointer(rseq->rseq_cs), sizeof(struct criu_rseq_cs)); - if (ret) { - pr_err("ptrace_peek_area(%d, %lx, %lx, %lx): fail to read rseq_cs struct\n", tid, - (unsigned long)rseq_cs, (unsigned long)rseq->rseq_cs, - (unsigned long)sizeof(struct criu_rseq_cs)); - return -1; - } - - return 0; -} - static int dump_thread_rseq(struct pstree_item *item, int i) { struct __ptrace_rseq_configuration rseqc; @@ -1197,8 +1084,6 @@ static int dump_thread_rseq(struct pstree_item *item, int i) int ret; CoreEntry *core = item->core[i]; RseqEntry **rseqep = &core->thread_core->rseq_entry; - struct criu_rseq rseq = {}; - struct criu_rseq_cs *rseq_cs = &dmpi(item)->thread_rseq_cs[i]; pid_t tid = item->threads[i].real; /* @@ -1237,56 +1122,26 @@ static int dump_thread_rseq(struct pstree_item *item, int i) rseqe->rseq_abi_size = rseqc.rseq_abi_size; rseqe->signature = rseqc.signature; - if (read_rseq_cs(tid, &rseqc, rseq_cs, &rseq)) - goto err; - - /* we won't save rseq_cs to the image (only pointer), - * so let's combine flags from both struct rseq and struct rseq_cs - * (kernel does the same when interpreting RSEQ_CS_FLAG_*) - */ - rseq_cs->flags |= rseq.flags; - - if (rseq_cs->flags & RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL) { - rseqe->has_rseq_cs_pointer = true; - rseqe->rseq_cs_pointer = rseq.rseq_cs; - } - /* save rseq entry to the image */ *rseqep = rseqe; return 0; - -err: - xfree(rseqe); - return -1; } static int dump_task_rseq(pid_t pid, struct pstree_item *item) { int i; - struct criu_rseq_cs *thread_rseq_cs; /* if rseq() syscall isn't supported then nothing to dump */ if (!kdat.has_rseq) return 0; - thread_rseq_cs = xzalloc(sizeof(*thread_rseq_cs) * item->nr_threads); - if (!thread_rseq_cs) - return -1; - - dmpi(item)->thread_rseq_cs = thread_rseq_cs; - for (i = 0; i < item->nr_threads; i++) { if (dump_thread_rseq(item, i)) - goto free_rseq; + return -1; } return 0; - -free_rseq: - xfree(thread_rseq_cs); - dmpi(item)->thread_rseq_cs = NULL; - return -1; } static struct proc_pid_stat pps_buf; @@ -1306,8 +1161,6 @@ static int dump_task_threads(struct parasite_ctl *parasite_ctl, const struct pst break; } - xfree(dmpi(item)->thread_rseq_cs); - dmpi(item)->thread_rseq_cs = NULL; return ret; } @@ -1632,12 +1485,6 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie) goto err; } - ret = fixup_thread_rseq(item, 0); - if (ret) { - pr_err("Fixup rseq for %d failed %d\n", pid, ret); - goto err; - } - if (fault_injected(FI_DUMP_EARLY)) { pr_info("fault: CRIU sudden detach\n"); kill(getpid(), SIGKILL); diff --git a/criu/cr-restore.c b/criu/cr-restore.c index 7579639a20..d8164366a0 100644 --- a/criu/cr-restore.c +++ b/criu/cr-restore.c @@ -25,6 +25,13 @@ #include "linux/rseq.h" +#ifdef __has_include +#if __has_include("sys/rseq.h") +#include +#include "asm/thread_pointer.h" +#endif +#endif + #include "clone-noasan.h" #include "cr_options.h" #include "servicefd.h" @@ -1922,7 +1929,7 @@ static void finalize_restore(void) continue; /* Unmap the restorer blob */ - ctl = compel_prepare_noctx(pid); + ctl = compel_prepare_noctx(pid, false); if (ctl == NULL) continue; diff --git a/criu/include/pstree.h b/criu/include/pstree.h index b750a919e6..b501ded721 100644 --- a/criu/include/pstree.h +++ b/criu/include/pstree.h @@ -63,7 +63,6 @@ struct dmp_info { struct parasite_ctl *parasite_ctl; struct parasite_thread_ctl **thread_ctls; uint64_t *thread_sp; - struct criu_rseq_cs *thread_rseq_cs; /* * Although we don't support dumping different struct creds in general, diff --git a/criu/parasite-syscall.c b/criu/parasite-syscall.c index c2d9ad976b..1d8a388c85 100644 --- a/criu/parasite-syscall.c +++ b/criu/parasite-syscall.c @@ -339,6 +339,7 @@ static int make_sigframe(void *arg, struct rt_sigframe *sf, struct rt_sigframe * static int parasite_prepare_threads(struct parasite_ctl *ctl, struct pstree_item *item) { + bool handle_rseq = kdat.has_ptrace_get_rseq_conf; struct parasite_thread_ctl **thread_ctls; uint64_t *thread_sp; int i; @@ -359,7 +360,7 @@ static int parasite_prepare_threads(struct parasite_ctl *ctl, struct pstree_item continue; } - thread_ctls[i] = compel_prepare_thread(ctl, tid->real); + thread_ctls[i] = compel_prepare_thread(ctl, tid->real, handle_rseq); if (!thread_ctls[i]) goto free_sp; @@ -380,6 +381,7 @@ static int parasite_prepare_threads(struct parasite_ctl *ctl, struct pstree_item struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item, struct vm_area_list *vma_area_list) { + bool handle_rseq = kdat.has_ptrace_get_rseq_conf; struct parasite_ctl *ctl; struct infect_ctx *ictx; unsigned long p; @@ -393,7 +395,7 @@ struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item, return NULL; } - ctl = compel_prepare_noctx(pid); + ctl = compel_prepare_noctx(pid, handle_rseq); if (!ctl) return NULL; diff --git a/criu/include/linux/rseq.h b/include/linux/rseq.h similarity index 98% rename from criu/include/linux/rseq.h rename to include/linux/rseq.h index 5ceefbf8e1..4ffc1a4f37 100644 --- a/criu/include/linux/rseq.h +++ b/include/linux/rseq.h @@ -2,13 +2,6 @@ #ifndef _UAPI_LINUX_RSEQ_H #define _UAPI_LINUX_RSEQ_H -#ifdef __has_include -#if __has_include("sys/rseq.h") -#include -#include "asm/thread_pointer.h" -#endif -#endif - #include #include @@ -43,6 +36,8 @@ enum rseq_cs_flags { RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL = (1U << RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE = (1U << RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), }; +#else +#include #endif /* CONFIG_HAS_NO_LIBC_RSEQ_DEFS */ /* diff --git a/test/zdtm/transition/Makefile b/test/zdtm/transition/Makefile index ddf2faaad1..8ed9248c7e 100644 --- a/test/zdtm/transition/Makefile +++ b/test/zdtm/transition/Makefile @@ -25,6 +25,7 @@ TST_NOFILE = \ pidfd_store_sk \ rseq01 \ rseq02 \ + rseq03 \ stack \ diff --git a/test/zdtm/transition/rseq03.c b/test/zdtm/transition/rseq03.c new file mode 100644 index 0000000000..5f3633b71b --- /dev/null +++ b/test/zdtm/transition/rseq03.c @@ -0,0 +1,287 @@ +/* + * test for rseq() syscall + * See also https://www.efficios.com/blog/2019/02/08/linux-restartable-sequences/ + * https://github.com/torvalds/linux/commit/d7822b1e24f2df5df98c76f0e94a5416349ff759 + */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "zdtmtst.h" +#include "lock.h" + +#ifdef __has_include +#if __has_include("sys/rseq.h") +#include +#endif +#endif + +#if defined(__x86_64__) + +#if defined(RSEQ_SIG) +static inline void *__criu_thread_pointer(void) +{ +#if __GNUC_PREREQ(11, 1) + return __builtin_thread_pointer(); +#else + void *__result; +#ifdef __x86_64__ + __asm__("mov %%fs:0, %0" : "=r"(__result)); +#else + __asm__("mov %%gs:0, %0" : "=r"(__result)); +#endif /* __x86_64__ */ + return __result; +#endif /* !GCC 11 */ +} + +static inline void unregister_glibc_rseq(void) +{ + struct rseq *rseq = (struct rseq *)((char *)__criu_thread_pointer() + __rseq_offset); + unsigned int size = __rseq_size; + + /* hack: mark glibc rseq structure as failed to register */ + rseq->cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED; + + /* unregister rseq */ + if (__rseq_size < 32) + size = 32; + syscall(__NR_rseq, (void *)rseq, size, 1, RSEQ_SIG); +} +#else +static inline void unregister_glibc_rseq(void) +{ +} +#endif /* defined(RSEQ_SIG) */ + +const char *test_doc = "Check rseq() critical section abort during C/R"; +const char *test_author = "Alexander Mikhalitsyn "; + +#ifndef RSEQ_SIG + +enum rseq_flags { + RSEQ_FLAG_UNREGISTER = (1 << 0), +}; + +struct rseq { + uint32_t cpu_id_start; + uint32_t cpu_id; + uint64_t rseq_cs; + uint32_t flags; +} __attribute__((aligned(4 * sizeof(uint64_t)))); + +struct rseq_cs { + /* Version of this structure. */ + uint32_t version; + /* enum rseq_cs_flags */ + uint32_t flags; + uint64_t start_ip; + /* Offset from start_ip. */ + uint64_t post_commit_offset; + uint64_t abort_ip; +} __attribute__((aligned(4 * sizeof(__u64)))); + +#define RSEQ_SIG 0x53053053 + +#endif /* RSEQ_SIG */ + +#ifndef __NR_rseq +#define __NR_rseq 334 +#endif +/* EOF */ + +extern futex_t sig_received; + +static __thread volatile struct rseq __rseq_abi __attribute__((aligned(64))); +static __thread volatile struct rseq_cs __rseq_cs __attribute__((aligned(64))); + +static __thread volatile int rseq_state = 0; + +static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len, int flags, uint32_t sig) +{ + return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); +} + +/* + * Return the rseq registration size. Starting with Linux 7.0, + * AT_RSEQ_ALIGN is 64 but the feature size is 33, so sizeof(struct rseq) + * (padded to alignment) no longer matches the registration size the kernel + * expects. Use __rseq_size when available, clamped to a minimum of 32 + * for older kernels. + */ +static uint32_t rseq_reg_size(void) +{ +#if defined(RSEQ_SIG) && defined(__rseq_size) + if (__rseq_size) + return (__rseq_size < 32) ? 32 : __rseq_size; +#endif + return sizeof(struct rseq); +} + +static unsigned long mmap_min_addr = 0x10000UL; + +/* + * Machine code for x86_64 that emulates: + * __rseq_abi.rseq_cs = &__rseq_cs; + * rseq_state = 1; + * while (futex_get(f) == 0) { + * if (__rseq_abi.rseq_cs != &__rseq_cs) + * return; + * } + * __rseq_abi.rseq_cs = 0; + * rseq_state = 0; + * + * Disassembly: + * 0x0: 48 89 37 mov %rsi,(%rdi) // set __rseq_abi.rseq_cs = &__rseq_cs + * 0x3: c7 01 01 00 00 00 movl $1,(%rcx) // set rseq_state = 1 + * 0x9: 48 39 37 cmp %rsi,(%rdi) // check if __rseq_abi.rseq_cs == &__rseq_cs + * 0xc: 75 13 jne 0x21 // if not equal, break out (jump to ret) + * 0xe: 8b 02 mov (%rdx),%eax // load *f + * 0x10: 85 c0 test %eax,%eax // check if 0 + * 0x12: 74 f5 je 0x9 // spin while *f == 0 + * 0x14: 48 c7 07 00 00 00 00 movq $0,(%rdi) // clear __rseq_abi.rseq_cs on exit + * 0x1b: c7 01 00 00 00 00 movl $0,(%rcx) // set rseq_state = 0 + * 0x21: c3 ret // return + */ +static const uint8_t test_go_rseq_code[] = { + 0x48, 0x89, 0x37, + 0xc7, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x48, 0x39, 0x37, + 0x75, 0x13, + 0x8b, 0x02, + 0x85, 0xc0, + 0x74, 0xf5, + 0x48, 0xc7, 0x07, 0x00, 0x00, 0x00, 0x00, + 0xc7, 0x01, 0x00, 0x00, 0x00, 0x00, + 0xc3, +}; + +/* + * Machine code for the rseq abort section: + * rseq_state = 0; + * return; + * + * Disassembly: + * 0: c7 01 00 00 00 00 movl $0,(%rcx) // set rseq_state = 0 + * 6: c3 ret // return + */ +static const uint8_t test_go_rseq_abort_code[] = { + 0xc7, 0x01, 0x00, 0x00, 0x00, 0x00, + 0xc3, +}; + +static void register_thread(void) +{ + int rc; + void *addr; + + addr = mmap((void *)mmap_min_addr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (addr == MAP_FAILED) + addr = mmap(NULL, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (addr == MAP_FAILED) { + fail("Failed to mmap rseq critical section area"); + exit(1); + } + + memcpy(addr, test_go_rseq_code, sizeof(test_go_rseq_code)); + + __rseq_cs.start_ip = (uint64_t)addr; + __rseq_cs.post_commit_offset = 2048; + __rseq_cs.abort_ip = __rseq_cs.start_ip + 2048; + *((uint32_t *)__rseq_cs.abort_ip - 1) = RSEQ_SIG; + memcpy((void *)__rseq_cs.abort_ip, test_go_rseq_abort_code, + sizeof(test_go_rseq_abort_code)); + + unregister_glibc_rseq(); + rc = sys_rseq(&__rseq_abi, rseq_reg_size(), 0, RSEQ_SIG); + if (rc) { + fail("Failed to register rseq"); + exit(1); + } +} + +static void check_thread(void) +{ + int rc; + rc = sys_rseq(&__rseq_abi, rseq_reg_size(), 0, RSEQ_SIG); + if (!(rc && errno == EBUSY)) { + fail("Failed to check rseq %d", rc); + exit(1); + } +} + +static void test_go_rseq(futex_t *f) +{ + void (*fn)(void *, volatile struct rseq_cs *, futex_t *, volatile int *) = + (void (*)(void *, volatile struct rseq_cs *, futex_t *, volatile int *))__rseq_cs.start_ip; + + while (futex_get(f) == 0) { + fn((void *)&__rseq_abi.rseq_cs, &__rseq_cs, f, &rseq_state); + if (rseq_state != 0) { + fail("rseq_state is %d (expected 0)", rseq_state); + exit(1); + } + } +} + +static void *tfunc(void *args) +{ + register_thread(); + test_go_rseq(&sig_received); + check_thread(); + return NULL; +} + +int main(int argc, char *argv[]) +{ + pthread_t th; + int ret; + + test_init(argc, argv); + + ret = pthread_create(&th, NULL, tfunc, NULL); + if (ret) { + pr_err("pthread_create -> %d\n", ret); + return 1; + } + + register_thread(); + + test_daemon(); + test_go_rseq(&sig_received); + + ret = pthread_join(th, NULL); + if (ret) { + pr_err("pthread_join -> %d\n", ret); + return 1; + } + check_thread(); + + pass(); + return 0; +} + +#else /* #if defined(__x86_64__) */ + +int main(int argc, char *argv[]) +{ + test_init(argc, argv); + skip("Unsupported arch"); + test_daemon(); + test_waitsig(); + pass(); + return 0; +} + +#endif /* #if defined(__x86_64__) */ diff --git a/test/zdtm/transition/rseq03.desc b/test/zdtm/transition/rseq03.desc new file mode 100644 index 0000000000..113feaee18 --- /dev/null +++ b/test/zdtm/transition/rseq03.desc @@ -0,0 +1 @@ +{'arch': 'x86_64', 'feature': 'get_rseq_conf'}