This is the 5.4.204 stable release
-----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEZH8oZUiU471FcZm+ONu9yGCSaT4FAmLG/ZkACgkQONu9yGCS aT7C7RAArxcDJ7LqrBQZfr9qc5bXDUqcj65SG4NfFHOFLeyK+XhbHnCqpTnyDreI 8mkzZ380MAdAZ3JCn7WIMLGnr31Brlyf3fjrQEkXwIeedyhvhXl6xBe/xDPfo9ny bhWNlyU0OFMLQ7k2xnF73ND1qam8K2pjqSmYIo5N9Ygv9nmjEWf7iDFABXMoDFUz K+qd+y/DW++wh44tHdf5p8KfgDJMf8sF8tOGG5uid6W8q7l4moPyAFmSct8L6QjT I4hRgf29IIxer8NbLRJtI0qYg0lX3IxsZbHjAqBT7WPdTaPfuxb25QjSEjX0UCuN PEQyqqYGiHo7ZTMrQrhx6CHY9l7ANH77MeyluWfcYqLrWowzQ6e03ezLnTIPlpn4 3Z+R/ITeWcg0xAR80Pt4dgRep/cLa2iMUoKr2x7gt0YbLTfp/A/x93SopAsDe6B3 LL5s5GqAa2fGknuvGx38dLYfCazlYPEB9ptSJbeQqIzjgUoOWrjahDBZsGf8R1/7 Tia5+Pm0bvUkFp1jOvF5KFJ6Nm0ljHez4D+ble8s6+H93ETdDf7l85xsny6c5nYk Zfwef09KXBJcDflHxHXNIygBTc94rUqcXUg0NF6sNBu6nfmdDNyQlceHrjbnSUaV XHqw265seqyr7sXLUtGJAEl0ReiL1p2uC+WUJmeJDuJL/UDQsV0= =hoqY -----END PGP SIGNATURE----- Merge 5.4.204 into android11-5.4-lts Changes in 5.4.204 ipv6: take care of disable_policy when restoring routes nvdimm: Fix badblocks clear off-by-one error powerpc/prom_init: Fix kernel config grep powerpc/bpf: Fix use of user_pt_regs in uapi dm raid: fix accesses beyond end of raid member array dm raid: fix KASAN warning in raid5_add_disks s390/archrandom: simplify back to earlier design and initialize earlier SUNRPC: Fix READ_PLUS crasher net: rose: fix UAF bugs caused by timer handler net: usb: ax88179_178a: Fix packet receiving virtio-net: fix race between ndo_open() and virtio_device_ready() selftests/net: pass ipv6_args to udpgso_bench's IPv6 TCP test net: tun: unlink NAPI from device on destruction net: tun: stop NAPI when detaching queues RDMA/qedr: Fix reporting QP timeout attribute linux/dim: Fix divide by 0 in RDMA DIM usbnet: fix memory allocation in helpers net: ipv6: unexport __init-annotated seg6_hmac_net_init() caif_virtio: fix race between virtio_device_ready() and ndo_open() PM / devfreq: exynos-ppmu: Fix refcount leak in of_get_devfreq_events s390: remove unneeded 'select BUILD_BIN2C' netfilter: nft_dynset: restore set element counter when failing to update net/sched: act_api: Notify user space if any actions were flushed before error net: bonding: fix possible NULL deref in rlb code net: bonding: fix use-after-free after 802.3ad slave unbind nfc: nfcmrvl: Fix irq_of_parse_and_map() return value NFC: nxp-nci: Don't issue a zero length i2c_master_read() net: tun: avoid disabling NAPI twice xen/gntdev: Avoid blocking in unmap_grant_pages() hwmon: (ibmaem) don't call platform_device_del() if platform_device_add() fails net: dsa: bcm_sf2: force pause link settings sit: use min ipv6/sit: fix ipip6_tunnel_get_prl return value rseq/selftests,x86_64: Add rseq_offset_deref_addv() selftests/rseq: remove ARRAY_SIZE define from individual tests selftests/rseq: introduce own copy of rseq uapi header selftests/rseq: Remove useless assignment to cpu variable selftests/rseq: Remove volatile from __rseq_abi selftests/rseq: Introduce rseq_get_abi() helper selftests/rseq: Introduce thread pointer getters selftests/rseq: Uplift rseq selftests for compatibility with glibc-2.35 selftests/rseq: Fix ppc32: wrong rseq_cs 32-bit field pointer on big endian selftests/rseq: Fix ppc32 missing instruction selection "u" and "x" for load/store selftests/rseq: Fix ppc32 offsets by using long rather than off_t selftests/rseq: Fix warnings about #if checks of undefined tokens selftests/rseq: Remove arm/mips asm goto compiler work-around selftests/rseq: Fix: work-around asm goto compiler bugs selftests/rseq: x86-64: use %fs segment selector for accessing rseq thread area selftests/rseq: x86-32: use %gs segment selector for accessing rseq thread area selftests/rseq: Change type of rseq_offset to ptrdiff_t xen/blkfront: fix leaking data in shared pages xen/netfront: fix leaking data in shared pages xen/netfront: force data bouncing when backend is untrusted xen/blkfront: force data bouncing when backend is untrusted xen/arm: Fix race in RB-tree based P2M accounting net: usb: qmi_wwan: add Telit 0x1060 composition net: usb: qmi_wwan: add Telit 0x1070 composition clocksource/drivers/ixp4xx: remove EXPORT_SYMBOL_GPL from ixp4xx_timer_setup() Linux 5.4.204 Signed-off-by: Greg Kroah-Hartman <gregkh@google.com> Change-Id: I3685b70677f176278354e4a86611fd56010d8788
This commit is contained in:
Greg Kroah-Hartman
commit
63b83aede5
2
Makefile
2
Makefile
@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
VERSION = 5
|
||||
PATCHLEVEL = 4
|
||||
SUBLEVEL = 203
|
||||
SUBLEVEL = 204
|
||||
EXTRAVERSION =
|
||||
NAME = Kleptomaniac Octopus
|
||||
|
||||
|
||||
@ -62,11 +62,12 @@ out:
|
||||
|
||||
unsigned long __pfn_to_mfn(unsigned long pfn)
|
||||
{
|
||||
struct rb_node *n = phys_to_mach.rb_node;
|
||||
struct rb_node *n;
|
||||
struct xen_p2m_entry *entry;
|
||||
unsigned long irqflags;
|
||||
|
||||
read_lock_irqsave(&p2m_lock, irqflags);
|
||||
n = phys_to_mach.rb_node;
|
||||
while (n) {
|
||||
entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys);
|
||||
if (entry->pfn <= pfn &&
|
||||
@ -153,10 +154,11 @@ bool __set_phys_to_machine_multi(unsigned long pfn,
|
||||
int rc;
|
||||
unsigned long irqflags;
|
||||
struct xen_p2m_entry *p2m_entry;
|
||||
struct rb_node *n = phys_to_mach.rb_node;
|
||||
struct rb_node *n;
|
||||
|
||||
if (mfn == INVALID_P2M_ENTRY) {
|
||||
write_lock_irqsave(&p2m_lock, irqflags);
|
||||
n = phys_to_mach.rb_node;
|
||||
while (n) {
|
||||
p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys);
|
||||
if (p2m_entry->pfn <= pfn &&
|
||||
|
||||
9
arch/powerpc/include/asm/bpf_perf_event.h
Normal file
9
arch/powerpc/include/asm/bpf_perf_event.h
Normal file
@ -0,0 +1,9 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _ASM_POWERPC_BPF_PERF_EVENT_H
|
||||
#define _ASM_POWERPC_BPF_PERF_EVENT_H
|
||||
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
typedef struct user_pt_regs bpf_user_pt_regs_t;
|
||||
|
||||
#endif /* _ASM_POWERPC_BPF_PERF_EVENT_H */
|
||||
@ -1,9 +0,0 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
|
||||
#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__
|
||||
#define _UAPI__ASM_BPF_PERF_EVENT_H__
|
||||
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
typedef struct user_pt_regs bpf_user_pt_regs_t;
|
||||
|
||||
#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */
|
||||
@ -13,7 +13,7 @@
|
||||
# If you really need to reference something from prom_init.o add
|
||||
# it to the list below:
|
||||
|
||||
grep "^CONFIG_KASAN=y$" .config >/dev/null
|
||||
grep "^CONFIG_KASAN=y$" ${KCONFIG_CONFIG} >/dev/null
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
MEM_FUNCS="__memcpy __memset"
|
||||
|
||||
@ -541,7 +541,6 @@ config KEXEC
|
||||
config KEXEC_FILE
|
||||
bool "kexec file based system call"
|
||||
select KEXEC_CORE
|
||||
select BUILD_BIN2C
|
||||
depends on CRYPTO
|
||||
depends on CRYPTO_SHA256
|
||||
depends on CRYPTO_SHA256_S390
|
||||
|
||||
@ -2,126 +2,17 @@
|
||||
/*
|
||||
* s390 arch random implementation.
|
||||
*
|
||||
* Copyright IBM Corp. 2017, 2018
|
||||
* Copyright IBM Corp. 2017, 2020
|
||||
* Author(s): Harald Freudenberger
|
||||
*
|
||||
* The s390_arch_random_generate() function may be called from random.c
|
||||
* in interrupt context. So this implementation does the best to be very
|
||||
* fast. There is a buffer of random data which is asynchronously checked
|
||||
* and filled by a workqueue thread.
|
||||
* If there are enough bytes in the buffer the s390_arch_random_generate()
|
||||
* just delivers these bytes. Otherwise false is returned until the
|
||||
* worker thread refills the buffer.
|
||||
* The worker fills the rng buffer by pulling fresh entropy from the
|
||||
* high quality (but slow) true hardware random generator. This entropy
|
||||
* is then spread over the buffer with an pseudo random generator PRNG.
|
||||
* As the arch_get_random_seed_long() fetches 8 bytes and the calling
|
||||
* function add_interrupt_randomness() counts this as 1 bit entropy the
|
||||
* distribution needs to make sure there is in fact 1 bit entropy contained
|
||||
* in 8 bytes of the buffer. The current values pull 32 byte entropy
|
||||
* and scatter this into a 2048 byte buffer. So 8 byte in the buffer
|
||||
* will contain 1 bit of entropy.
|
||||
* The worker thread is rescheduled based on the charge level of the
|
||||
* buffer but at least with 500 ms delay to avoid too much CPU consumption.
|
||||
* So the max. amount of rng data delivered via arch_get_random_seed is
|
||||
* limited to 4k bytes per second.
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/random.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/static_key.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <asm/cpacf.h>
|
||||
|
||||
DEFINE_STATIC_KEY_FALSE(s390_arch_random_available);
|
||||
|
||||
atomic64_t s390_arch_random_counter = ATOMIC64_INIT(0);
|
||||
EXPORT_SYMBOL(s390_arch_random_counter);
|
||||
|
||||
#define ARCH_REFILL_TICKS (HZ/2)
|
||||
#define ARCH_PRNG_SEED_SIZE 32
|
||||
#define ARCH_RNG_BUF_SIZE 2048
|
||||
|
||||
static DEFINE_SPINLOCK(arch_rng_lock);
|
||||
static u8 *arch_rng_buf;
|
||||
static unsigned int arch_rng_buf_idx;
|
||||
|
||||
static void arch_rng_refill_buffer(struct work_struct *);
|
||||
static DECLARE_DELAYED_WORK(arch_rng_work, arch_rng_refill_buffer);
|
||||
|
||||
bool s390_arch_random_generate(u8 *buf, unsigned int nbytes)
|
||||
{
|
||||
/* max hunk is ARCH_RNG_BUF_SIZE */
|
||||
if (nbytes > ARCH_RNG_BUF_SIZE)
|
||||
return false;
|
||||
|
||||
/* lock rng buffer */
|
||||
if (!spin_trylock(&arch_rng_lock))
|
||||
return false;
|
||||
|
||||
/* try to resolve the requested amount of bytes from the buffer */
|
||||
arch_rng_buf_idx -= nbytes;
|
||||
if (arch_rng_buf_idx < ARCH_RNG_BUF_SIZE) {
|
||||
memcpy(buf, arch_rng_buf + arch_rng_buf_idx, nbytes);
|
||||
atomic64_add(nbytes, &s390_arch_random_counter);
|
||||
spin_unlock(&arch_rng_lock);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* not enough bytes in rng buffer, refill is done asynchronously */
|
||||
spin_unlock(&arch_rng_lock);
|
||||
|
||||
return false;
|
||||
}
|
||||
EXPORT_SYMBOL(s390_arch_random_generate);
|
||||
|
||||
static void arch_rng_refill_buffer(struct work_struct *unused)
|
||||
{
|
||||
unsigned int delay = ARCH_REFILL_TICKS;
|
||||
|
||||
spin_lock(&arch_rng_lock);
|
||||
if (arch_rng_buf_idx > ARCH_RNG_BUF_SIZE) {
|
||||
/* buffer is exhausted and needs refill */
|
||||
u8 seed[ARCH_PRNG_SEED_SIZE];
|
||||
u8 prng_wa[240];
|
||||
/* fetch ARCH_PRNG_SEED_SIZE bytes of entropy */
|
||||
cpacf_trng(NULL, 0, seed, sizeof(seed));
|
||||
/* blow this entropy up to ARCH_RNG_BUF_SIZE with PRNG */
|
||||
memset(prng_wa, 0, sizeof(prng_wa));
|
||||
cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED,
|
||||
&prng_wa, NULL, 0, seed, sizeof(seed));
|
||||
cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN,
|
||||
&prng_wa, arch_rng_buf, ARCH_RNG_BUF_SIZE, NULL, 0);
|
||||
arch_rng_buf_idx = ARCH_RNG_BUF_SIZE;
|
||||
}
|
||||
delay += (ARCH_REFILL_TICKS * arch_rng_buf_idx) / ARCH_RNG_BUF_SIZE;
|
||||
spin_unlock(&arch_rng_lock);
|
||||
|
||||
/* kick next check */
|
||||
queue_delayed_work(system_long_wq, &arch_rng_work, delay);
|
||||
}
|
||||
|
||||
static int __init s390_arch_random_init(void)
|
||||
{
|
||||
/* all the needed PRNO subfunctions available ? */
|
||||
if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG) &&
|
||||
cpacf_query_func(CPACF_PRNO, CPACF_PRNO_SHA512_DRNG_GEN)) {
|
||||
|
||||
/* alloc arch random working buffer */
|
||||
arch_rng_buf = kmalloc(ARCH_RNG_BUF_SIZE, GFP_KERNEL);
|
||||
if (!arch_rng_buf)
|
||||
return -ENOMEM;
|
||||
|
||||
/* kick worker queue job to fill the random buffer */
|
||||
queue_delayed_work(system_long_wq,
|
||||
&arch_rng_work, ARCH_REFILL_TICKS);
|
||||
|
||||
/* enable arch random to the outside world */
|
||||
static_branch_enable(&s390_arch_random_available);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
arch_initcall(s390_arch_random_init);
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
/*
|
||||
* Kernel interface for the s390 arch_random_* functions
|
||||
*
|
||||
* Copyright IBM Corp. 2017
|
||||
* Copyright IBM Corp. 2017, 2020
|
||||
*
|
||||
* Author: Harald Freudenberger <freude@de.ibm.com>
|
||||
*
|
||||
@ -15,34 +15,37 @@
|
||||
|
||||
#include <linux/static_key.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <asm/cpacf.h>
|
||||
|
||||
DECLARE_STATIC_KEY_FALSE(s390_arch_random_available);
|
||||
extern atomic64_t s390_arch_random_counter;
|
||||
|
||||
bool s390_arch_random_generate(u8 *buf, unsigned int nbytes);
|
||||
|
||||
static inline bool arch_get_random_long(unsigned long *v)
|
||||
static inline bool __must_check arch_get_random_long(unsigned long *v)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool arch_get_random_int(unsigned int *v)
|
||||
static inline bool __must_check arch_get_random_int(unsigned int *v)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool arch_get_random_seed_long(unsigned long *v)
|
||||
static inline bool __must_check arch_get_random_seed_long(unsigned long *v)
|
||||
{
|
||||
if (static_branch_likely(&s390_arch_random_available)) {
|
||||
return s390_arch_random_generate((u8 *)v, sizeof(*v));
|
||||
cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v));
|
||||
atomic64_add(sizeof(*v), &s390_arch_random_counter);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool arch_get_random_seed_int(unsigned int *v)
|
||||
static inline bool __must_check arch_get_random_seed_int(unsigned int *v)
|
||||
{
|
||||
if (static_branch_likely(&s390_arch_random_available)) {
|
||||
return s390_arch_random_generate((u8 *)v, sizeof(*v));
|
||||
cpacf_trng(NULL, 0, (u8 *)v, sizeof(*v));
|
||||
atomic64_add(sizeof(*v), &s390_arch_random_counter);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1005,6 +1005,11 @@ static void __init setup_randomness(void)
|
||||
if (stsi(vmms, 3, 2, 2) == 0 && vmms->count)
|
||||
add_device_randomness(&vmms->vm, sizeof(vmms->vm[0]) * vmms->count);
|
||||
memblock_free((unsigned long) vmms, PAGE_SIZE);
|
||||
|
||||
#ifdef CONFIG_ARCH_RANDOM
|
||||
if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG))
|
||||
static_branch_enable(&s390_arch_random_available);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -151,6 +151,10 @@ static unsigned int xen_blkif_max_ring_order;
|
||||
module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
|
||||
MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
|
||||
|
||||
static bool __read_mostly xen_blkif_trusted = true;
|
||||
module_param_named(trusted, xen_blkif_trusted, bool, 0644);
|
||||
MODULE_PARM_DESC(trusted, "Is the backend trusted");
|
||||
|
||||
#define BLK_RING_SIZE(info) \
|
||||
__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages)
|
||||
|
||||
@ -211,6 +215,7 @@ struct blkfront_info
|
||||
unsigned int feature_discard:1;
|
||||
unsigned int feature_secdiscard:1;
|
||||
unsigned int feature_persistent:1;
|
||||
unsigned int bounce:1;
|
||||
unsigned int discard_granularity;
|
||||
unsigned int discard_alignment;
|
||||
/* Number of 4KB segments handled */
|
||||
@ -300,8 +305,8 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
|
||||
if (!gnt_list_entry)
|
||||
goto out_of_memory;
|
||||
|
||||
if (info->feature_persistent) {
|
||||
granted_page = alloc_page(GFP_NOIO);
|
||||
if (info->bounce) {
|
||||
granted_page = alloc_page(GFP_NOIO | __GFP_ZERO);
|
||||
if (!granted_page) {
|
||||
kfree(gnt_list_entry);
|
||||
goto out_of_memory;
|
||||
@ -320,7 +325,7 @@ out_of_memory:
|
||||
list_for_each_entry_safe(gnt_list_entry, n,
|
||||
&rinfo->grants, node) {
|
||||
list_del(&gnt_list_entry->node);
|
||||
if (info->feature_persistent)
|
||||
if (info->bounce)
|
||||
__free_page(gnt_list_entry->page);
|
||||
kfree(gnt_list_entry);
|
||||
i--;
|
||||
@ -366,7 +371,7 @@ static struct grant *get_grant(grant_ref_t *gref_head,
|
||||
/* Assign a gref to this page */
|
||||
gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
|
||||
BUG_ON(gnt_list_entry->gref == -ENOSPC);
|
||||
if (info->feature_persistent)
|
||||
if (info->bounce)
|
||||
grant_foreign_access(gnt_list_entry, info);
|
||||
else {
|
||||
/* Grant access to the GFN passed by the caller */
|
||||
@ -390,7 +395,7 @@ static struct grant *get_indirect_grant(grant_ref_t *gref_head,
|
||||
/* Assign a gref to this page */
|
||||
gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
|
||||
BUG_ON(gnt_list_entry->gref == -ENOSPC);
|
||||
if (!info->feature_persistent) {
|
||||
if (!info->bounce) {
|
||||
struct page *indirect_page;
|
||||
|
||||
/* Fetch a pre-allocated page to use for indirect grefs */
|
||||
@ -705,7 +710,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri
|
||||
.grant_idx = 0,
|
||||
.segments = NULL,
|
||||
.rinfo = rinfo,
|
||||
.need_copy = rq_data_dir(req) && info->feature_persistent,
|
||||
.need_copy = rq_data_dir(req) && info->bounce,
|
||||
};
|
||||
|
||||
/*
|
||||
@ -1026,11 +1031,12 @@ static void xlvbd_flush(struct blkfront_info *info)
|
||||
{
|
||||
blk_queue_write_cache(info->rq, info->feature_flush ? true : false,
|
||||
info->feature_fua ? true : false);
|
||||
pr_info("blkfront: %s: %s %s %s %s %s\n",
|
||||
pr_info("blkfront: %s: %s %s %s %s %s %s %s\n",
|
||||
info->gd->disk_name, flush_info(info),
|
||||
"persistent grants:", info->feature_persistent ?
|
||||
"enabled;" : "disabled;", "indirect descriptors:",
|
||||
info->max_indirect_segments ? "enabled;" : "disabled;");
|
||||
info->max_indirect_segments ? "enabled;" : "disabled;",
|
||||
"bounce buffer:", info->bounce ? "enabled" : "disabled;");
|
||||
}
|
||||
|
||||
static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
|
||||
@ -1265,7 +1271,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo)
|
||||
if (!list_empty(&rinfo->indirect_pages)) {
|
||||
struct page *indirect_page, *n;
|
||||
|
||||
BUG_ON(info->feature_persistent);
|
||||
BUG_ON(info->bounce);
|
||||
list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
|
||||
list_del(&indirect_page->lru);
|
||||
__free_page(indirect_page);
|
||||
@ -1282,7 +1288,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo)
|
||||
0, 0UL);
|
||||
rinfo->persistent_gnts_c--;
|
||||
}
|
||||
if (info->feature_persistent)
|
||||
if (info->bounce)
|
||||
__free_page(persistent_gnt->page);
|
||||
kfree(persistent_gnt);
|
||||
}
|
||||
@ -1303,7 +1309,7 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo)
|
||||
for (j = 0; j < segs; j++) {
|
||||
persistent_gnt = rinfo->shadow[i].grants_used[j];
|
||||
gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
|
||||
if (info->feature_persistent)
|
||||
if (info->bounce)
|
||||
__free_page(persistent_gnt->page);
|
||||
kfree(persistent_gnt);
|
||||
}
|
||||
@ -1493,7 +1499,7 @@ static int blkif_completion(unsigned long *id,
|
||||
data.s = s;
|
||||
num_sg = s->num_sg;
|
||||
|
||||
if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
|
||||
if (bret->operation == BLKIF_OP_READ && info->bounce) {
|
||||
for_each_sg(s->sg, sg, num_sg, i) {
|
||||
BUG_ON(sg->offset + sg->length > PAGE_SIZE);
|
||||
|
||||
@ -1552,7 +1558,7 @@ static int blkif_completion(unsigned long *id,
|
||||
* Add the used indirect page back to the list of
|
||||
* available pages for indirect grefs.
|
||||
*/
|
||||
if (!info->feature_persistent) {
|
||||
if (!info->bounce) {
|
||||
indirect_page = s->indirect_grants[i]->page;
|
||||
list_add(&indirect_page->lru, &rinfo->indirect_pages);
|
||||
}
|
||||
@ -1744,7 +1750,7 @@ static int setup_blkring(struct xenbus_device *dev,
|
||||
for (i = 0; i < info->nr_ring_pages; i++)
|
||||
rinfo->ring_ref[i] = GRANT_INVALID_REF;
|
||||
|
||||
sring = alloc_pages_exact(ring_size, GFP_NOIO);
|
||||
sring = alloc_pages_exact(ring_size, GFP_NOIO | __GFP_ZERO);
|
||||
if (!sring) {
|
||||
xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
|
||||
return -ENOMEM;
|
||||
@ -1847,6 +1853,10 @@ static int talk_to_blkback(struct xenbus_device *dev,
|
||||
if (!info)
|
||||
return -ENODEV;
|
||||
|
||||
/* Check if backend is trusted. */
|
||||
info->bounce = !xen_blkif_trusted ||
|
||||
!xenbus_read_unsigned(dev->nodename, "trusted", 1);
|
||||
|
||||
max_page_order = xenbus_read_unsigned(info->xbdev->otherend,
|
||||
"max-ring-page-order", 0);
|
||||
ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
|
||||
@ -2273,17 +2283,18 @@ static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
|
||||
if (err)
|
||||
goto out_of_memory;
|
||||
|
||||
if (!info->feature_persistent && info->max_indirect_segments) {
|
||||
if (!info->bounce && info->max_indirect_segments) {
|
||||
/*
|
||||
* We are using indirect descriptors but not persistent
|
||||
* grants, we need to allocate a set of pages that can be
|
||||
* We are using indirect descriptors but don't have a bounce
|
||||
* buffer, we need to allocate a set of pages that can be
|
||||
* used for mapping indirect grefs
|
||||
*/
|
||||
int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
|
||||
|
||||
BUG_ON(!list_empty(&rinfo->indirect_pages));
|
||||
for (i = 0; i < num; i++) {
|
||||
struct page *indirect_page = alloc_page(GFP_KERNEL);
|
||||
struct page *indirect_page = alloc_page(GFP_KERNEL |
|
||||
__GFP_ZERO);
|
||||
if (!indirect_page)
|
||||
goto out_of_memory;
|
||||
list_add(&indirect_page->lru, &rinfo->indirect_pages);
|
||||
@ -2375,6 +2386,8 @@ static void blkfront_gather_backend_features(struct blkfront_info *info)
|
||||
info->feature_persistent =
|
||||
!!xenbus_read_unsigned(info->xbdev->otherend,
|
||||
"feature-persistent", 0);
|
||||
if (info->feature_persistent)
|
||||
info->bounce = true;
|
||||
|
||||
indirect_segments = xenbus_read_unsigned(info->xbdev->otherend,
|
||||
"feature-max-indirect-segments", 0);
|
||||
@ -2750,6 +2763,13 @@ static void blkfront_delay_work(struct work_struct *work)
|
||||
struct blkfront_info *info;
|
||||
bool need_schedule_work = false;
|
||||
|
||||
/*
|
||||
* Note that when using bounce buffers but not persistent grants
|
||||
* there's no need to run blkfront_delay_work because grants are
|
||||
* revoked in blkif_completion or else an error is reported and the
|
||||
* connection is closed.
|
||||
*/
|
||||
|
||||
mutex_lock(&blkfront_mutex);
|
||||
|
||||
list_for_each_entry(info, &info_list, info_list) {
|
||||
|
||||
@ -258,7 +258,6 @@ void __init ixp4xx_timer_setup(resource_size_t timerbase,
|
||||
}
|
||||
ixp4xx_timer_register(base, timer_irq, timer_freq);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(ixp4xx_timer_setup);
|
||||
|
||||
#ifdef CONFIG_OF
|
||||
static __init int ixp4xx_of_timer_init(struct device_node *np)
|
||||
|
||||
@ -514,15 +514,19 @@ static int of_get_devfreq_events(struct device_node *np,
|
||||
|
||||
count = of_get_child_count(events_np);
|
||||
desc = devm_kcalloc(dev, count, sizeof(*desc), GFP_KERNEL);
|
||||
if (!desc)
|
||||
if (!desc) {
|
||||
of_node_put(events_np);
|
||||
return -ENOMEM;
|
||||
}
|
||||
info->num_events = count;
|
||||
|
||||
of_id = of_match_device(exynos_ppmu_id_match, dev);
|
||||
if (of_id)
|
||||
info->ppmu_type = (enum exynos_ppmu_type)of_id->data;
|
||||
else
|
||||
else {
|
||||
of_node_put(events_np);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
j = 0;
|
||||
for_each_child_of_node(events_np, node) {
|
||||
|
||||
@ -550,7 +550,7 @@ static int aem_init_aem1_inst(struct aem_ipmi_data *probe, u8 module_handle)
|
||||
|
||||
res = platform_device_add(data->pdev);
|
||||
if (res)
|
||||
goto ipmi_err;
|
||||
goto dev_add_err;
|
||||
|
||||
platform_set_drvdata(data->pdev, data);
|
||||
|
||||
@ -598,7 +598,9 @@ hwmon_reg_err:
|
||||
ipmi_destroy_user(data->ipmi.user);
|
||||
ipmi_err:
|
||||
platform_set_drvdata(data->pdev, NULL);
|
||||
platform_device_unregister(data->pdev);
|
||||
platform_device_del(data->pdev);
|
||||
dev_add_err:
|
||||
platform_device_put(data->pdev);
|
||||
dev_err:
|
||||
ida_simple_remove(&aem_ida, data->id);
|
||||
id_err:
|
||||
@ -690,7 +692,7 @@ static int aem_init_aem2_inst(struct aem_ipmi_data *probe,
|
||||
|
||||
res = platform_device_add(data->pdev);
|
||||
if (res)
|
||||
goto ipmi_err;
|
||||
goto dev_add_err;
|
||||
|
||||
platform_set_drvdata(data->pdev, data);
|
||||
|
||||
@ -738,7 +740,9 @@ hwmon_reg_err:
|
||||
ipmi_destroy_user(data->ipmi.user);
|
||||
ipmi_err:
|
||||
platform_set_drvdata(data->pdev, NULL);
|
||||
platform_device_unregister(data->pdev);
|
||||
platform_device_del(data->pdev);
|
||||
dev_add_err:
|
||||
platform_device_put(data->pdev);
|
||||
dev_err:
|
||||
ida_simple_remove(&aem_ida, data->id);
|
||||
id_err:
|
||||
|
||||
@ -416,6 +416,7 @@ struct qedr_qp {
|
||||
u32 sq_psn;
|
||||
u32 qkey;
|
||||
u32 dest_qp_num;
|
||||
u8 timeout;
|
||||
|
||||
/* Relevant to qps created from kernel space only (ULPs) */
|
||||
u8 prev_wqe_size;
|
||||
|
||||
@ -2259,6 +2259,8 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
|
||||
1 << max_t(int, attr->timeout - 8, 0);
|
||||
else
|
||||
qp_params.ack_timeout = 0;
|
||||
|
||||
qp->timeout = attr->timeout;
|
||||
}
|
||||
|
||||
if (attr_mask & IB_QP_RETRY_CNT) {
|
||||
@ -2418,7 +2420,7 @@ int qedr_query_qp(struct ib_qp *ibqp,
|
||||
rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
|
||||
rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
|
||||
rdma_ah_set_sl(&qp_attr->ah_attr, 0);
|
||||
qp_attr->timeout = params.timeout;
|
||||
qp_attr->timeout = qp->timeout;
|
||||
qp_attr->rnr_retry = params.rnr_retry;
|
||||
qp_attr->retry_cnt = params.retry_cnt;
|
||||
qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
|
||||
|
||||
@ -998,12 +998,13 @@ static int validate_region_size(struct raid_set *rs, unsigned long region_size)
|
||||
static int validate_raid_redundancy(struct raid_set *rs)
|
||||
{
|
||||
unsigned int i, rebuild_cnt = 0;
|
||||
unsigned int rebuilds_per_group = 0, copies;
|
||||
unsigned int rebuilds_per_group = 0, copies, raid_disks;
|
||||
unsigned int group_size, last_group_start;
|
||||
|
||||
for (i = 0; i < rs->md.raid_disks; i++)
|
||||
if (!test_bit(In_sync, &rs->dev[i].rdev.flags) ||
|
||||
!rs->dev[i].rdev.sb_page)
|
||||
for (i = 0; i < rs->raid_disks; i++)
|
||||
if (!test_bit(FirstUse, &rs->dev[i].rdev.flags) &&
|
||||
((!test_bit(In_sync, &rs->dev[i].rdev.flags) ||
|
||||
!rs->dev[i].rdev.sb_page)))
|
||||
rebuild_cnt++;
|
||||
|
||||
switch (rs->md.level) {
|
||||
@ -1043,8 +1044,9 @@ static int validate_raid_redundancy(struct raid_set *rs)
|
||||
* A A B B C
|
||||
* C D D E E
|
||||
*/
|
||||
raid_disks = min(rs->raid_disks, rs->md.raid_disks);
|
||||
if (__is_raid10_near(rs->md.new_layout)) {
|
||||
for (i = 0; i < rs->md.raid_disks; i++) {
|
||||
for (i = 0; i < raid_disks; i++) {
|
||||
if (!(i % copies))
|
||||
rebuilds_per_group = 0;
|
||||
if ((!rs->dev[i].rdev.sb_page ||
|
||||
@ -1067,10 +1069,10 @@ static int validate_raid_redundancy(struct raid_set *rs)
|
||||
* results in the need to treat the last (potentially larger)
|
||||
* set differently.
|
||||
*/
|
||||
group_size = (rs->md.raid_disks / copies);
|
||||
last_group_start = (rs->md.raid_disks / group_size) - 1;
|
||||
group_size = (raid_disks / copies);
|
||||
last_group_start = (raid_disks / group_size) - 1;
|
||||
last_group_start *= group_size;
|
||||
for (i = 0; i < rs->md.raid_disks; i++) {
|
||||
for (i = 0; i < raid_disks; i++) {
|
||||
if (!(i % copies) && !(i > last_group_start))
|
||||
rebuilds_per_group = 0;
|
||||
if ((!rs->dev[i].rdev.sb_page ||
|
||||
@ -1585,7 +1587,7 @@ static sector_t __rdev_sectors(struct raid_set *rs)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < rs->md.raid_disks; i++) {
|
||||
for (i = 0; i < rs->raid_disks; i++) {
|
||||
struct md_rdev *rdev = &rs->dev[i].rdev;
|
||||
|
||||
if (!test_bit(Journal, &rdev->flags) &&
|
||||
@ -3746,13 +3748,13 @@ static int raid_iterate_devices(struct dm_target *ti,
|
||||
unsigned int i;
|
||||
int r = 0;
|
||||
|
||||
for (i = 0; !r && i < rs->md.raid_disks; i++)
|
||||
if (rs->dev[i].data_dev)
|
||||
r = fn(ti,
|
||||
rs->dev[i].data_dev,
|
||||
0, /* No offset on data devs */
|
||||
rs->md.dev_sectors,
|
||||
data);
|
||||
for (i = 0; !r && i < rs->raid_disks; i++) {
|
||||
if (rs->dev[i].data_dev) {
|
||||
r = fn(ti, rs->dev[i].data_dev,
|
||||
0, /* No offset on data devs */
|
||||
rs->md.dev_sectors, data);
|
||||
}
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
@ -7722,6 +7722,7 @@ static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
|
||||
*/
|
||||
if (rdev->saved_raid_disk >= 0 &&
|
||||
rdev->saved_raid_disk >= first &&
|
||||
rdev->saved_raid_disk <= last &&
|
||||
conf->disks[rdev->saved_raid_disk].rdev == NULL)
|
||||
first = rdev->saved_raid_disk;
|
||||
|
||||
|
||||
@ -2218,7 +2218,8 @@ void bond_3ad_unbind_slave(struct slave *slave)
|
||||
temp_aggregator->num_of_ports--;
|
||||
if (__agg_active_ports(temp_aggregator) == 0) {
|
||||
select_new_active_agg = temp_aggregator->is_active;
|
||||
ad_clear_agg(temp_aggregator);
|
||||
if (temp_aggregator->num_of_ports == 0)
|
||||
ad_clear_agg(temp_aggregator);
|
||||
if (select_new_active_agg) {
|
||||
slave_info(bond->dev, slave->dev, "Removing an active aggregator\n");
|
||||
/* select new active aggregator */
|
||||
|
||||
@ -1276,12 +1276,12 @@ int bond_alb_initialize(struct bonding *bond, int rlb_enabled)
|
||||
return res;
|
||||
|
||||
if (rlb_enabled) {
|
||||
bond->alb_info.rlb_enabled = 1;
|
||||
res = rlb_initialize(bond);
|
||||
if (res) {
|
||||
tlb_deinitialize(bond);
|
||||
return res;
|
||||
}
|
||||
bond->alb_info.rlb_enabled = 1;
|
||||
} else {
|
||||
bond->alb_info.rlb_enabled = 0;
|
||||
}
|
||||
|
||||
@ -723,13 +723,21 @@ static int cfv_probe(struct virtio_device *vdev)
|
||||
/* Carrier is off until netdevice is opened */
|
||||
netif_carrier_off(netdev);
|
||||
|
||||
/* serialize netdev register + virtio_device_ready() with ndo_open() */
|
||||
rtnl_lock();
|
||||
|
||||
/* register Netdev */
|
||||
err = register_netdev(netdev);
|
||||
err = register_netdevice(netdev);
|
||||
if (err) {
|
||||
rtnl_unlock();
|
||||
dev_err(&vdev->dev, "Unable to register netdev (%d)\n", err);
|
||||
goto err;
|
||||
}
|
||||
|
||||
virtio_device_ready(vdev);
|
||||
|
||||
rtnl_unlock();
|
||||
|
||||
debugfs_init(cfv);
|
||||
|
||||
return 0;
|
||||
|
||||
@ -602,6 +602,11 @@ force_link:
|
||||
reg |= LINK_STS;
|
||||
if (state->duplex == DUPLEX_FULL)
|
||||
reg |= DUPLX_MODE;
|
||||
if (state->pause & MLO_PAUSE_TXRX_MASK) {
|
||||
if (state->pause & MLO_PAUSE_TX)
|
||||
reg |= TXFLOW_CNTL;
|
||||
reg |= RXFLOW_CNTL;
|
||||
}
|
||||
|
||||
core_writel(priv, reg, offset);
|
||||
}
|
||||
|
||||
@ -327,6 +327,12 @@ static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
|
||||
}
|
||||
}
|
||||
|
||||
static void tun_napi_enable(struct tun_file *tfile)
|
||||
{
|
||||
if (tfile->napi_enabled)
|
||||
napi_enable(&tfile->napi);
|
||||
}
|
||||
|
||||
static void tun_napi_disable(struct tun_file *tfile)
|
||||
{
|
||||
if (tfile->napi_enabled)
|
||||
@ -690,7 +696,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
|
||||
tun = rtnl_dereference(tfile->tun);
|
||||
|
||||
if (tun && clean) {
|
||||
tun_napi_disable(tfile);
|
||||
if (!tfile->detached)
|
||||
tun_napi_disable(tfile);
|
||||
tun_napi_del(tfile);
|
||||
}
|
||||
|
||||
@ -709,8 +716,10 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
|
||||
if (clean) {
|
||||
RCU_INIT_POINTER(tfile->tun, NULL);
|
||||
sock_put(&tfile->sk);
|
||||
} else
|
||||
} else {
|
||||
tun_disable_queue(tun, tfile);
|
||||
tun_napi_disable(tfile);
|
||||
}
|
||||
|
||||
synchronize_net();
|
||||
tun_flow_delete_by_queue(tun, tun->numqueues + 1);
|
||||
@ -783,6 +792,7 @@ static void tun_detach_all(struct net_device *dev)
|
||||
sock_put(&tfile->sk);
|
||||
}
|
||||
list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
|
||||
tun_napi_del(tfile);
|
||||
tun_enable_queue(tfile);
|
||||
tun_queue_purge(tfile);
|
||||
xdp_rxq_info_unreg(&tfile->xdp_rxq);
|
||||
@ -863,6 +873,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
|
||||
|
||||
if (tfile->detached) {
|
||||
tun_enable_queue(tfile);
|
||||
tun_napi_enable(tfile);
|
||||
} else {
|
||||
sock_hold(&tfile->sk);
|
||||
tun_napi_init(tun, tfile, napi, napi_frags);
|
||||
|
||||
@ -1365,6 +1365,42 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
|
||||
* are bundled into this buffer and where we can find an array of
|
||||
* per-packet metadata (which contains elements encoded into u16).
|
||||
*/
|
||||
|
||||
/* SKB contents for current firmware:
|
||||
* <packet 1> <padding>
|
||||
* ...
|
||||
* <packet N> <padding>
|
||||
* <per-packet metadata entry 1> <dummy header>
|
||||
* ...
|
||||
* <per-packet metadata entry N> <dummy header>
|
||||
* <padding2> <rx_hdr>
|
||||
*
|
||||
* where:
|
||||
* <packet N> contains pkt_len bytes:
|
||||
* 2 bytes of IP alignment pseudo header
|
||||
* packet received
|
||||
* <per-packet metadata entry N> contains 4 bytes:
|
||||
* pkt_len and fields AX_RXHDR_*
|
||||
* <padding> 0-7 bytes to terminate at
|
||||
* 8 bytes boundary (64-bit).
|
||||
* <padding2> 4 bytes to make rx_hdr terminate at
|
||||
* 8 bytes boundary (64-bit)
|
||||
* <dummy-header> contains 4 bytes:
|
||||
* pkt_len=0 and AX_RXHDR_DROP_ERR
|
||||
* <rx-hdr> contains 4 bytes:
|
||||
* pkt_cnt and hdr_off (offset of
|
||||
* <per-packet metadata entry 1>)
|
||||
*
|
||||
* pkt_cnt is number of entrys in the per-packet metadata.
|
||||
* In current firmware there is 2 entrys per packet.
|
||||
* The first points to the packet and the
|
||||
* second is a dummy header.
|
||||
* This was done probably to align fields in 64-bit and
|
||||
* maintain compatibility with old firmware.
|
||||
* This code assumes that <dummy header> and <padding2> are
|
||||
* optional.
|
||||
*/
|
||||
|
||||
if (skb->len < 4)
|
||||
return 0;
|
||||
skb_trim(skb, skb->len - 4);
|
||||
@ -1378,51 +1414,66 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
|
||||
/* Make sure that the bounds of the metadata array are inside the SKB
|
||||
* (and in front of the counter at the end).
|
||||
*/
|
||||
if (pkt_cnt * 2 + hdr_off > skb->len)
|
||||
if (pkt_cnt * 4 + hdr_off > skb->len)
|
||||
return 0;
|
||||
pkt_hdr = (u32 *)(skb->data + hdr_off);
|
||||
|
||||
/* Packets must not overlap the metadata array */
|
||||
skb_trim(skb, hdr_off);
|
||||
|
||||
for (; ; pkt_cnt--, pkt_hdr++) {
|
||||
for (; pkt_cnt > 0; pkt_cnt--, pkt_hdr++) {
|
||||
u16 pkt_len_plus_padd;
|
||||
u16 pkt_len;
|
||||
|
||||
le32_to_cpus(pkt_hdr);
|
||||
pkt_len = (*pkt_hdr >> 16) & 0x1fff;
|
||||
pkt_len_plus_padd = (pkt_len + 7) & 0xfff8;
|
||||
|
||||
if (pkt_len > skb->len)
|
||||
/* Skip dummy header used for alignment
|
||||
*/
|
||||
if (pkt_len == 0)
|
||||
continue;
|
||||
|
||||
if (pkt_len_plus_padd > skb->len)
|
||||
return 0;
|
||||
|
||||
/* Check CRC or runt packet */
|
||||
if (((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) == 0) &&
|
||||
pkt_len >= 2 + ETH_HLEN) {
|
||||
bool last = (pkt_cnt == 0);
|
||||
|
||||
if (last) {
|
||||
ax_skb = skb;
|
||||
} else {
|
||||
ax_skb = skb_clone(skb, GFP_ATOMIC);
|
||||
if (!ax_skb)
|
||||
return 0;
|
||||
}
|
||||
ax_skb->len = pkt_len;
|
||||
/* Skip IP alignment pseudo header */
|
||||
skb_pull(ax_skb, 2);
|
||||
skb_set_tail_pointer(ax_skb, ax_skb->len);
|
||||
ax_skb->truesize = pkt_len + sizeof(struct sk_buff);
|
||||
ax88179_rx_checksum(ax_skb, pkt_hdr);
|
||||
|
||||
if (last)
|
||||
return 1;
|
||||
|
||||
usbnet_skb_return(dev, ax_skb);
|
||||
if ((*pkt_hdr & (AX_RXHDR_CRC_ERR | AX_RXHDR_DROP_ERR)) ||
|
||||
pkt_len < 2 + ETH_HLEN) {
|
||||
dev->net->stats.rx_errors++;
|
||||
skb_pull(skb, pkt_len_plus_padd);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Trim this packet away from the SKB */
|
||||
if (!skb_pull(skb, (pkt_len + 7) & 0xFFF8))
|
||||
/* last packet */
|
||||
if (pkt_len_plus_padd == skb->len) {
|
||||
skb_trim(skb, pkt_len);
|
||||
|
||||
/* Skip IP alignment pseudo header */
|
||||
skb_pull(skb, 2);
|
||||
|
||||
skb->truesize = SKB_TRUESIZE(pkt_len_plus_padd);
|
||||
ax88179_rx_checksum(skb, pkt_hdr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ax_skb = skb_clone(skb, GFP_ATOMIC);
|
||||
if (!ax_skb)
|
||||
return 0;
|
||||
skb_trim(ax_skb, pkt_len);
|
||||
|
||||
/* Skip IP alignment pseudo header */
|
||||
skb_pull(ax_skb, 2);
|
||||
|
||||
skb->truesize = pkt_len_plus_padd +
|
||||
SKB_DATA_ALIGN(sizeof(struct sk_buff));
|
||||
ax88179_rx_checksum(ax_skb, pkt_hdr);
|
||||
usbnet_skb_return(dev, ax_skb);
|
||||
|
||||
skb_pull(skb, pkt_len_plus_padd);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct sk_buff *
|
||||
|
||||
@ -1315,6 +1315,8 @@ static const struct usb_device_id products[] = {
|
||||
{QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */
|
||||
{QMI_QUIRK_SET_DTR(0x1bc7, 0x1040, 2)}, /* Telit LE922A */
|
||||
{QMI_QUIRK_SET_DTR(0x1bc7, 0x1050, 2)}, /* Telit FN980 */
|
||||
{QMI_QUIRK_SET_DTR(0x1bc7, 0x1060, 2)}, /* Telit LN920 */
|
||||
{QMI_QUIRK_SET_DTR(0x1bc7, 0x1070, 2)}, /* Telit FN990 */
|
||||
{QMI_FIXED_INTF(0x1bc7, 0x1100, 3)}, /* Telit ME910 */
|
||||
{QMI_FIXED_INTF(0x1bc7, 0x1101, 3)}, /* Telit ME910 dual modem */
|
||||
{QMI_FIXED_INTF(0x1bc7, 0x1200, 5)}, /* Telit LE920 */
|
||||
|
||||
@ -1994,7 +1994,7 @@ static int __usbnet_read_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
|
||||
cmd, reqtype, value, index, size);
|
||||
|
||||
if (size) {
|
||||
buf = kmalloc(size, GFP_KERNEL);
|
||||
buf = kmalloc(size, GFP_NOIO);
|
||||
if (!buf)
|
||||
goto out;
|
||||
}
|
||||
@ -2026,7 +2026,7 @@ static int __usbnet_write_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
|
||||
cmd, reqtype, value, index, size);
|
||||
|
||||
if (data) {
|
||||
buf = kmemdup(data, size, GFP_KERNEL);
|
||||
buf = kmemdup(data, size, GFP_NOIO);
|
||||
if (!buf)
|
||||
goto out;
|
||||
} else {
|
||||
|
||||
@ -3222,14 +3222,20 @@ static int virtnet_probe(struct virtio_device *vdev)
|
||||
}
|
||||
}
|
||||
|
||||
err = register_netdev(dev);
|
||||
/* serialize netdev register + virtio_device_ready() with ndo_open() */
|
||||
rtnl_lock();
|
||||
|
||||
err = register_netdevice(dev);
|
||||
if (err) {
|
||||
pr_debug("virtio_net: registering device failed\n");
|
||||
rtnl_unlock();
|
||||
goto free_failover;
|
||||
}
|
||||
|
||||
virtio_device_ready(vdev);
|
||||
|
||||
rtnl_unlock();
|
||||
|
||||
err = virtnet_cpu_notif_add(vi);
|
||||
if (err) {
|
||||
pr_debug("virtio_net: registering cpu notifier failed\n");
|
||||
|
||||
@ -63,6 +63,10 @@ module_param_named(max_queues, xennet_max_queues, uint, 0644);
|
||||
MODULE_PARM_DESC(max_queues,
|
||||
"Maximum number of queues per virtual interface");
|
||||
|
||||
static bool __read_mostly xennet_trusted = true;
|
||||
module_param_named(trusted, xennet_trusted, bool, 0644);
|
||||
MODULE_PARM_DESC(trusted, "Is the backend trusted");
|
||||
|
||||
#define XENNET_TIMEOUT (5 * HZ)
|
||||
|
||||
static const struct ethtool_ops xennet_ethtool_ops;
|
||||
@ -163,6 +167,9 @@ struct netfront_info {
|
||||
/* Is device behaving sane? */
|
||||
bool broken;
|
||||
|
||||
/* Should skbs be bounced into a zeroed buffer? */
|
||||
bool bounce;
|
||||
|
||||
atomic_t rx_gso_checksum_fixup;
|
||||
};
|
||||
|
||||
@ -261,7 +268,7 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)
|
||||
if (unlikely(!skb))
|
||||
return NULL;
|
||||
|
||||
page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
|
||||
page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO);
|
||||
if (!page) {
|
||||
kfree_skb(skb);
|
||||
return NULL;
|
||||
@ -590,6 +597,34 @@ static void xennet_mark_tx_pending(struct netfront_queue *queue)
|
||||
queue->tx_link[i] = TX_PENDING;
|
||||
}
|
||||
|
||||
struct sk_buff *bounce_skb(const struct sk_buff *skb)
|
||||
{
|
||||
unsigned int headerlen = skb_headroom(skb);
|
||||
/* Align size to allocate full pages and avoid contiguous data leaks */
|
||||
unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len,
|
||||
XEN_PAGE_SIZE);
|
||||
struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO);
|
||||
|
||||
if (!n)
|
||||
return NULL;
|
||||
|
||||
if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) {
|
||||
WARN_ONCE(1, "misaligned skb allocated\n");
|
||||
kfree_skb(n);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Set the data pointer */
|
||||
skb_reserve(n, headerlen);
|
||||
/* Set the tail pointer and length */
|
||||
skb_put(n, skb->len);
|
||||
|
||||
BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
|
||||
|
||||
skb_copy_header(n, skb);
|
||||
return n;
|
||||
}
|
||||
|
||||
#define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
|
||||
|
||||
static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
||||
@ -642,9 +677,13 @@ static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev
|
||||
|
||||
/* The first req should be at least ETH_HLEN size or the packet will be
|
||||
* dropped by netback.
|
||||
*
|
||||
* If the backend is not trusted bounce all data to zeroed pages to
|
||||
* avoid exposing contiguous data on the granted page not belonging to
|
||||
* the skb.
|
||||
*/
|
||||
if (unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
|
||||
nskb = skb_copy(skb, GFP_ATOMIC);
|
||||
if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
|
||||
nskb = bounce_skb(skb);
|
||||
if (!nskb)
|
||||
goto drop;
|
||||
dev_consume_skb_any(skb);
|
||||
@ -1950,6 +1989,10 @@ static int talk_to_netback(struct xenbus_device *dev,
|
||||
|
||||
info->netdev->irq = 0;
|
||||
|
||||
/* Check if backend is trusted. */
|
||||
info->bounce = !xennet_trusted ||
|
||||
!xenbus_read_unsigned(dev->nodename, "trusted", 1);
|
||||
|
||||
/* Check if backend supports multiple queues */
|
||||
max_queues = xenbus_read_unsigned(info->xbdev->otherend,
|
||||
"multi-queue-max-queues", 1);
|
||||
@ -2103,6 +2146,9 @@ static int xennet_connect(struct net_device *dev)
|
||||
err = talk_to_netback(np->xbdev, np);
|
||||
if (err)
|
||||
return err;
|
||||
if (np->bounce)
|
||||
dev_info(&np->xbdev->dev,
|
||||
"bouncing transmitted data to zeroed pages\n");
|
||||
|
||||
/* talk_to_netback() sets the correct number of queues */
|
||||
num_queues = dev->real_num_tx_queues;
|
||||
|
||||
@ -186,9 +186,9 @@ static int nfcmrvl_i2c_parse_dt(struct device_node *node,
|
||||
pdata->irq_polarity = IRQF_TRIGGER_RISING;
|
||||
|
||||
ret = irq_of_parse_and_map(node, 0);
|
||||
if (ret < 0) {
|
||||
pr_err("Unable to get irq, error: %d\n", ret);
|
||||
return ret;
|
||||
if (!ret) {
|
||||
pr_err("Unable to get irq\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
pdata->irq = ret;
|
||||
|
||||
|
||||
@ -129,9 +129,9 @@ static int nfcmrvl_spi_parse_dt(struct device_node *node,
|
||||
}
|
||||
|
||||
ret = irq_of_parse_and_map(node, 0);
|
||||
if (ret < 0) {
|
||||
pr_err("Unable to get irq, error: %d\n", ret);
|
||||
return ret;
|
||||
if (!ret) {
|
||||
pr_err("Unable to get irq\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
pdata->irq = ret;
|
||||
|
||||
|
||||
@ -162,6 +162,9 @@ static int nxp_nci_i2c_nci_read(struct nxp_nci_i2c_phy *phy,
|
||||
|
||||
skb_put_data(*skb, (void *)&header, NCI_CTRL_HDR_SIZE);
|
||||
|
||||
if (!header.plen)
|
||||
return 0;
|
||||
|
||||
r = i2c_master_recv(client, skb_put(*skb, header.plen), header.plen);
|
||||
if (r != header.plen) {
|
||||
nfc_err(&client->dev,
|
||||
|
||||
@ -187,8 +187,8 @@ static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
|
||||
ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
|
||||
|
||||
/* make sure we are in the region */
|
||||
if (ctx->phys < nd_region->ndr_start
|
||||
|| (ctx->phys + ctx->cleared) > ndr_end)
|
||||
if (ctx->phys < nd_region->ndr_start ||
|
||||
(ctx->phys + ctx->cleared - 1) > ndr_end)
|
||||
return 0;
|
||||
|
||||
sector = (ctx->phys - nd_region->ndr_start) / 512;
|
||||
|
||||
@ -15,6 +15,8 @@
|
||||
#include <linux/mman.h>
|
||||
#include <linux/mmu_notifier.h>
|
||||
#include <linux/types.h>
|
||||
#include <xen/interface/event_channel.h>
|
||||
#include <xen/grant_table.h>
|
||||
|
||||
struct gntdev_dmabuf_priv;
|
||||
|
||||
@ -61,6 +63,7 @@ struct gntdev_grant_map {
|
||||
struct gnttab_unmap_grant_ref *unmap_ops;
|
||||
struct gnttab_map_grant_ref *kmap_ops;
|
||||
struct gnttab_unmap_grant_ref *kunmap_ops;
|
||||
bool *being_removed;
|
||||
struct page **pages;
|
||||
unsigned long pages_vm_start;
|
||||
|
||||
@ -78,6 +81,11 @@ struct gntdev_grant_map {
|
||||
/* Needed to avoid allocation in gnttab_dma_free_pages(). */
|
||||
xen_pfn_t *frames;
|
||||
#endif
|
||||
|
||||
/* Number of live grants */
|
||||
atomic_t live_grants;
|
||||
/* Needed to avoid allocation in __unmap_grant_pages */
|
||||
struct gntab_unmap_queue_data unmap_data;
|
||||
};
|
||||
|
||||
struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count,
|
||||
|
||||
@ -35,6 +35,7 @@
|
||||
#include <linux/slab.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/refcount.h>
|
||||
#include <linux/workqueue.h>
|
||||
|
||||
#include <xen/xen.h>
|
||||
#include <xen/grant_table.h>
|
||||
@ -62,11 +63,12 @@ MODULE_PARM_DESC(limit, "Maximum number of grants that may be mapped by "
|
||||
|
||||
static atomic_t pages_mapped = ATOMIC_INIT(0);
|
||||
|
||||
/* True in PV mode, false otherwise */
|
||||
static int use_ptemod;
|
||||
#define populate_freeable_maps use_ptemod
|
||||
|
||||
static int unmap_grant_pages(struct gntdev_grant_map *map,
|
||||
int offset, int pages);
|
||||
static void unmap_grant_pages(struct gntdev_grant_map *map,
|
||||
int offset, int pages);
|
||||
|
||||
static struct miscdevice gntdev_miscdev;
|
||||
|
||||
@ -123,6 +125,7 @@ static void gntdev_free_map(struct gntdev_grant_map *map)
|
||||
kfree(map->unmap_ops);
|
||||
kfree(map->kmap_ops);
|
||||
kfree(map->kunmap_ops);
|
||||
kfree(map->being_removed);
|
||||
kfree(map);
|
||||
}
|
||||
|
||||
@ -142,12 +145,15 @@ struct gntdev_grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count,
|
||||
add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL);
|
||||
add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL);
|
||||
add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL);
|
||||
add->being_removed =
|
||||
kcalloc(count, sizeof(add->being_removed[0]), GFP_KERNEL);
|
||||
if (NULL == add->grants ||
|
||||
NULL == add->map_ops ||
|
||||
NULL == add->unmap_ops ||
|
||||
NULL == add->kmap_ops ||
|
||||
NULL == add->kunmap_ops ||
|
||||
NULL == add->pages)
|
||||
NULL == add->pages ||
|
||||
NULL == add->being_removed)
|
||||
goto err;
|
||||
|
||||
#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
|
||||
@ -243,6 +249,35 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)
|
||||
return;
|
||||
|
||||
atomic_sub(map->count, &pages_mapped);
|
||||
if (map->pages && !use_ptemod) {
|
||||
/*
|
||||
* Increment the reference count. This ensures that the
|
||||
* subsequent call to unmap_grant_pages() will not wind up
|
||||
* re-entering itself. It *can* wind up calling
|
||||
* gntdev_put_map() recursively, but such calls will be with a
|
||||
* reference count greater than 1, so they will return before
|
||||
* this code is reached. The recursion depth is thus limited to
|
||||
* 1. Do NOT use refcount_inc() here, as it will detect that
|
||||
* the reference count is zero and WARN().
|
||||
*/
|
||||
refcount_set(&map->users, 1);
|
||||
|
||||
/*
|
||||
* Unmap the grants. This may or may not be asynchronous, so it
|
||||
* is possible that the reference count is 1 on return, but it
|
||||
* could also be greater than 1.
|
||||
*/
|
||||
unmap_grant_pages(map, 0, map->count);
|
||||
|
||||
/* Check if the memory now needs to be freed */
|
||||
if (!refcount_dec_and_test(&map->users))
|
||||
return;
|
||||
|
||||
/*
|
||||
* All pages have been returned to the hypervisor, so free the
|
||||
* map.
|
||||
*/
|
||||
}
|
||||
|
||||
if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {
|
||||
notify_remote_via_evtchn(map->notify.event);
|
||||
@ -298,6 +333,7 @@ static int set_grant_ptes_as_special(pte_t *pte, unsigned long addr, void *data)
|
||||
|
||||
int gntdev_map_grant_pages(struct gntdev_grant_map *map)
|
||||
{
|
||||
size_t alloced = 0;
|
||||
int i, err = 0;
|
||||
|
||||
if (!use_ptemod) {
|
||||
@ -346,87 +382,109 @@ int gntdev_map_grant_pages(struct gntdev_grant_map *map)
|
||||
map->pages, map->count);
|
||||
|
||||
for (i = 0; i < map->count; i++) {
|
||||
if (map->map_ops[i].status == GNTST_okay)
|
||||
if (map->map_ops[i].status == GNTST_okay) {
|
||||
map->unmap_ops[i].handle = map->map_ops[i].handle;
|
||||
else if (!err)
|
||||
if (!use_ptemod)
|
||||
alloced++;
|
||||
} else if (!err)
|
||||
err = -EINVAL;
|
||||
|
||||
if (map->flags & GNTMAP_device_map)
|
||||
map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr;
|
||||
|
||||
if (use_ptemod) {
|
||||
if (map->kmap_ops[i].status == GNTST_okay)
|
||||
if (map->kmap_ops[i].status == GNTST_okay) {
|
||||
if (map->map_ops[i].status == GNTST_okay)
|
||||
alloced++;
|
||||
map->kunmap_ops[i].handle = map->kmap_ops[i].handle;
|
||||
else if (!err)
|
||||
} else if (!err)
|
||||
err = -EINVAL;
|
||||
}
|
||||
}
|
||||
atomic_add(alloced, &map->live_grants);
|
||||
return err;
|
||||
}
|
||||
|
||||
static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset,
|
||||
int pages)
|
||||
static void __unmap_grant_pages_done(int result,
|
||||
struct gntab_unmap_queue_data *data)
|
||||
{
|
||||
int i, err = 0;
|
||||
struct gntab_unmap_queue_data unmap_data;
|
||||
unsigned int i;
|
||||
struct gntdev_grant_map *map = data->data;
|
||||
unsigned int offset = data->unmap_ops - map->unmap_ops;
|
||||
|
||||
if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
|
||||
int pgno = (map->notify.addr >> PAGE_SHIFT);
|
||||
if (pgno >= offset && pgno < offset + pages) {
|
||||
/* No need for kmap, pages are in lowmem */
|
||||
uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));
|
||||
tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;
|
||||
map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;
|
||||
}
|
||||
}
|
||||
|
||||
unmap_data.unmap_ops = map->unmap_ops + offset;
|
||||
unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
|
||||
unmap_data.pages = map->pages + offset;
|
||||
unmap_data.count = pages;
|
||||
|
||||
err = gnttab_unmap_refs_sync(&unmap_data);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
for (i = 0; i < pages; i++) {
|
||||
if (map->unmap_ops[offset+i].status)
|
||||
err = -EINVAL;
|
||||
for (i = 0; i < data->count; i++) {
|
||||
WARN_ON(map->unmap_ops[offset+i].status);
|
||||
pr_debug("unmap handle=%d st=%d\n",
|
||||
map->unmap_ops[offset+i].handle,
|
||||
map->unmap_ops[offset+i].status);
|
||||
map->unmap_ops[offset+i].handle = -1;
|
||||
}
|
||||
return err;
|
||||
/*
|
||||
* Decrease the live-grant counter. This must happen after the loop to
|
||||
* prevent premature reuse of the grants by gnttab_mmap().
|
||||
*/
|
||||
atomic_sub(data->count, &map->live_grants);
|
||||
|
||||
/* Release reference taken by __unmap_grant_pages */
|
||||
gntdev_put_map(NULL, map);
|
||||
}
|
||||
|
||||
static int unmap_grant_pages(struct gntdev_grant_map *map, int offset,
|
||||
int pages)
|
||||
static void __unmap_grant_pages(struct gntdev_grant_map *map, int offset,
|
||||
int pages)
|
||||
{
|
||||
int range, err = 0;
|
||||
if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
|
||||
int pgno = (map->notify.addr >> PAGE_SHIFT);
|
||||
|
||||
if (pgno >= offset && pgno < offset + pages) {
|
||||
/* No need for kmap, pages are in lowmem */
|
||||
uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));
|
||||
|
||||
tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;
|
||||
map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;
|
||||
}
|
||||
}
|
||||
|
||||
map->unmap_data.unmap_ops = map->unmap_ops + offset;
|
||||
map->unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
|
||||
map->unmap_data.pages = map->pages + offset;
|
||||
map->unmap_data.count = pages;
|
||||
map->unmap_data.done = __unmap_grant_pages_done;
|
||||
map->unmap_data.data = map;
|
||||
refcount_inc(&map->users); /* to keep map alive during async call below */
|
||||
|
||||
gnttab_unmap_refs_async(&map->unmap_data);
|
||||
}
|
||||
|
||||
static void unmap_grant_pages(struct gntdev_grant_map *map, int offset,
|
||||
int pages)
|
||||
{
|
||||
int range;
|
||||
|
||||
if (atomic_read(&map->live_grants) == 0)
|
||||
return; /* Nothing to do */
|
||||
|
||||
pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);
|
||||
|
||||
/* It is possible the requested range will have a "hole" where we
|
||||
* already unmapped some of the grants. Only unmap valid ranges.
|
||||
*/
|
||||
while (pages && !err) {
|
||||
while (pages && map->unmap_ops[offset].handle == -1) {
|
||||
while (pages) {
|
||||
while (pages && map->being_removed[offset]) {
|
||||
offset++;
|
||||
pages--;
|
||||
}
|
||||
range = 0;
|
||||
while (range < pages) {
|
||||
if (map->unmap_ops[offset+range].handle == -1)
|
||||
if (map->being_removed[offset + range])
|
||||
break;
|
||||
map->being_removed[offset + range] = true;
|
||||
range++;
|
||||
}
|
||||
err = __unmap_grant_pages(map, offset, range);
|
||||
if (range)
|
||||
__unmap_grant_pages(map, offset, range);
|
||||
offset += range;
|
||||
pages -= range;
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
@ -496,7 +554,6 @@ static int unmap_if_in_range(struct gntdev_grant_map *map,
|
||||
bool blockable)
|
||||
{
|
||||
unsigned long mstart, mend;
|
||||
int err;
|
||||
|
||||
if (!in_range(map, start, end))
|
||||
return 0;
|
||||
@ -510,10 +567,9 @@ static int unmap_if_in_range(struct gntdev_grant_map *map,
|
||||
map->index, map->count,
|
||||
map->vma->vm_start, map->vma->vm_end,
|
||||
start, end, mstart, mend);
|
||||
err = unmap_grant_pages(map,
|
||||
unmap_grant_pages(map,
|
||||
(mstart - map->vma->vm_start) >> PAGE_SHIFT,
|
||||
(mend - mstart) >> PAGE_SHIFT);
|
||||
WARN_ON(err);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -554,7 +610,6 @@ static void mn_release(struct mmu_notifier *mn,
|
||||
{
|
||||
struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
|
||||
struct gntdev_grant_map *map;
|
||||
int err;
|
||||
|
||||
mutex_lock(&priv->lock);
|
||||
list_for_each_entry(map, &priv->maps, next) {
|
||||
@ -563,8 +618,7 @@ static void mn_release(struct mmu_notifier *mn,
|
||||
pr_debug("map %d+%d (%lx %lx)\n",
|
||||
map->index, map->count,
|
||||
map->vma->vm_start, map->vma->vm_end);
|
||||
err = unmap_grant_pages(map, /* offset */ 0, map->count);
|
||||
WARN_ON(err);
|
||||
unmap_grant_pages(map, /* offset */ 0, map->count);
|
||||
}
|
||||
list_for_each_entry(map, &priv->freeable_maps, next) {
|
||||
if (!map->vma)
|
||||
@ -572,8 +626,7 @@ static void mn_release(struct mmu_notifier *mn,
|
||||
pr_debug("map %d+%d (%lx %lx)\n",
|
||||
map->index, map->count,
|
||||
map->vma->vm_start, map->vma->vm_end);
|
||||
err = unmap_grant_pages(map, /* offset */ 0, map->count);
|
||||
WARN_ON(err);
|
||||
unmap_grant_pages(map, /* offset */ 0, map->count);
|
||||
}
|
||||
mutex_unlock(&priv->lock);
|
||||
}
|
||||
@ -1102,6 +1155,10 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
|
||||
goto unlock_out;
|
||||
}
|
||||
|
||||
if (atomic_read(&map->live_grants)) {
|
||||
err = -EAGAIN;
|
||||
goto unlock_out;
|
||||
}
|
||||
refcount_inc(&map->users);
|
||||
|
||||
vma->vm_ops = &gntdev_vmops;
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
* We consider 10% difference as significant.
|
||||
*/
|
||||
#define IS_SIGNIFICANT_DIFF(val, ref) \
|
||||
(((100UL * abs((val) - (ref))) / (ref)) > 10)
|
||||
((ref) && (((100UL * abs((val) - (ref))) / (ref)) > 10))
|
||||
|
||||
/**
|
||||
* Calculate the gap between two values.
|
||||
|
||||
@ -1104,10 +1104,6 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg,
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (net->ipv6.devconf_all->disable_policy ||
|
||||
idev->cnf.disable_policy)
|
||||
f6i->dst_nopolicy = true;
|
||||
|
||||
neigh_parms_data_state_setall(idev->nd_parms);
|
||||
|
||||
ifa->addr = *cfg->pfx;
|
||||
|
||||
@ -4441,8 +4441,15 @@ struct fib6_info *addrconf_f6i_alloc(struct net *net,
|
||||
}
|
||||
|
||||
f6i = ip6_route_info_create(&cfg, gfp_flags, NULL);
|
||||
if (!IS_ERR(f6i))
|
||||
if (!IS_ERR(f6i)) {
|
||||
f6i->dst_nocount = true;
|
||||
|
||||
if (!anycast &&
|
||||
(net->ipv6.devconf_all->disable_policy ||
|
||||
idev->cnf.disable_policy))
|
||||
f6i->dst_nopolicy = true;
|
||||
}
|
||||
|
||||
return f6i;
|
||||
}
|
||||
|
||||
|
||||
@ -410,7 +410,6 @@ int __net_init seg6_hmac_net_init(struct net *net)
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(seg6_hmac_net_init);
|
||||
|
||||
void seg6_hmac_exit(void)
|
||||
{
|
||||
|
||||
@ -310,9 +310,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t,
|
||||
kcalloc(cmax, sizeof(*kp), GFP_KERNEL | __GFP_NOWARN) :
|
||||
NULL;
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
ca = t->prl_count < cmax ? t->prl_count : cmax;
|
||||
ca = min(t->prl_count, cmax);
|
||||
|
||||
if (!kp) {
|
||||
/* We don't try hard to allocate much memory for
|
||||
@ -327,7 +325,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t,
|
||||
}
|
||||
}
|
||||
|
||||
c = 0;
|
||||
rcu_read_lock();
|
||||
for_each_prl_rcu(t->prl) {
|
||||
if (c >= cmax)
|
||||
break;
|
||||
@ -339,7 +337,7 @@ static int ipip6_tunnel_get_prl(struct ip_tunnel *t,
|
||||
if (kprl.addr != htonl(INADDR_ANY))
|
||||
break;
|
||||
}
|
||||
out:
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
len = sizeof(*kp) * c;
|
||||
@ -348,7 +346,7 @@ out:
|
||||
ret = -EFAULT;
|
||||
|
||||
kfree(kp);
|
||||
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -142,6 +142,7 @@ static bool nft_rhash_update(struct nft_set *set, const u32 *key,
|
||||
/* Another cpu may race to insert the element with the same key */
|
||||
if (prev) {
|
||||
nft_set_elem_destroy(set, he, true);
|
||||
atomic_dec(&set->nelems);
|
||||
he = prev;
|
||||
}
|
||||
|
||||
@ -151,6 +152,7 @@ out:
|
||||
|
||||
err2:
|
||||
nft_set_elem_destroy(set, he, true);
|
||||
atomic_dec(&set->nelems);
|
||||
err1:
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -31,89 +31,89 @@ static void rose_idletimer_expiry(struct timer_list *);
|
||||
|
||||
void rose_start_heartbeat(struct sock *sk)
|
||||
{
|
||||
del_timer(&sk->sk_timer);
|
||||
sk_stop_timer(sk, &sk->sk_timer);
|
||||
|
||||
sk->sk_timer.function = rose_heartbeat_expiry;
|
||||
sk->sk_timer.expires = jiffies + 5 * HZ;
|
||||
|
||||
add_timer(&sk->sk_timer);
|
||||
sk_reset_timer(sk, &sk->sk_timer, sk->sk_timer.expires);
|
||||
}
|
||||
|
||||
void rose_start_t1timer(struct sock *sk)
|
||||
{
|
||||
struct rose_sock *rose = rose_sk(sk);
|
||||
|
||||
del_timer(&rose->timer);
|
||||
sk_stop_timer(sk, &rose->timer);
|
||||
|
||||
rose->timer.function = rose_timer_expiry;
|
||||
rose->timer.expires = jiffies + rose->t1;
|
||||
|
||||
add_timer(&rose->timer);
|
||||
sk_reset_timer(sk, &rose->timer, rose->timer.expires);
|
||||
}
|
||||
|
||||
void rose_start_t2timer(struct sock *sk)
|
||||
{
|
||||
struct rose_sock *rose = rose_sk(sk);
|
||||
|
||||
del_timer(&rose->timer);
|
||||
sk_stop_timer(sk, &rose->timer);
|
||||
|
||||
rose->timer.function = rose_timer_expiry;
|
||||
rose->timer.expires = jiffies + rose->t2;
|
||||
|
||||
add_timer(&rose->timer);
|
||||
sk_reset_timer(sk, &rose->timer, rose->timer.expires);
|
||||
}
|
||||
|
||||
void rose_start_t3timer(struct sock *sk)
|
||||
{
|
||||
struct rose_sock *rose = rose_sk(sk);
|
||||
|
||||
del_timer(&rose->timer);
|
||||
sk_stop_timer(sk, &rose->timer);
|
||||
|
||||
rose->timer.function = rose_timer_expiry;
|
||||
rose->timer.expires = jiffies + rose->t3;
|
||||
|
||||
add_timer(&rose->timer);
|
||||
sk_reset_timer(sk, &rose->timer, rose->timer.expires);
|
||||
}
|
||||
|
||||
void rose_start_hbtimer(struct sock *sk)
|
||||
{
|
||||
struct rose_sock *rose = rose_sk(sk);
|
||||
|
||||
del_timer(&rose->timer);
|
||||
sk_stop_timer(sk, &rose->timer);
|
||||
|
||||
rose->timer.function = rose_timer_expiry;
|
||||
rose->timer.expires = jiffies + rose->hb;
|
||||
|
||||
add_timer(&rose->timer);
|
||||
sk_reset_timer(sk, &rose->timer, rose->timer.expires);
|
||||
}
|
||||
|
||||
void rose_start_idletimer(struct sock *sk)
|
||||
{
|
||||
struct rose_sock *rose = rose_sk(sk);
|
||||
|
||||
del_timer(&rose->idletimer);
|
||||
sk_stop_timer(sk, &rose->idletimer);
|
||||
|
||||
if (rose->idle > 0) {
|
||||
rose->idletimer.function = rose_idletimer_expiry;
|
||||
rose->idletimer.expires = jiffies + rose->idle;
|
||||
|
||||
add_timer(&rose->idletimer);
|
||||
sk_reset_timer(sk, &rose->idletimer, rose->idletimer.expires);
|
||||
}
|
||||
}
|
||||
|
||||
void rose_stop_heartbeat(struct sock *sk)
|
||||
{
|
||||
del_timer(&sk->sk_timer);
|
||||
sk_stop_timer(sk, &sk->sk_timer);
|
||||
}
|
||||
|
||||
void rose_stop_timer(struct sock *sk)
|
||||
{
|
||||
del_timer(&rose_sk(sk)->timer);
|
||||
sk_stop_timer(sk, &rose_sk(sk)->timer);
|
||||
}
|
||||
|
||||
void rose_stop_idletimer(struct sock *sk)
|
||||
{
|
||||
del_timer(&rose_sk(sk)->idletimer);
|
||||
sk_stop_timer(sk, &rose_sk(sk)->idletimer);
|
||||
}
|
||||
|
||||
static void rose_heartbeat_expiry(struct timer_list *t)
|
||||
@ -130,6 +130,7 @@ static void rose_heartbeat_expiry(struct timer_list *t)
|
||||
(sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_DEAD))) {
|
||||
bh_unlock_sock(sk);
|
||||
rose_destroy_socket(sk);
|
||||
sock_put(sk);
|
||||
return;
|
||||
}
|
||||
break;
|
||||
@ -152,6 +153,7 @@ static void rose_heartbeat_expiry(struct timer_list *t)
|
||||
|
||||
rose_start_heartbeat(sk);
|
||||
bh_unlock_sock(sk);
|
||||
sock_put(sk);
|
||||
}
|
||||
|
||||
static void rose_timer_expiry(struct timer_list *t)
|
||||
@ -181,6 +183,7 @@ static void rose_timer_expiry(struct timer_list *t)
|
||||
break;
|
||||
}
|
||||
bh_unlock_sock(sk);
|
||||
sock_put(sk);
|
||||
}
|
||||
|
||||
static void rose_idletimer_expiry(struct timer_list *t)
|
||||
@ -205,4 +208,5 @@ static void rose_idletimer_expiry(struct timer_list *t)
|
||||
sock_set_flag(sk, SOCK_DEAD);
|
||||
}
|
||||
bh_unlock_sock(sk);
|
||||
sock_put(sk);
|
||||
}
|
||||
|
||||
@ -287,7 +287,8 @@ static int tcf_idr_release_unsafe(struct tc_action *p)
|
||||
}
|
||||
|
||||
static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb,
|
||||
const struct tc_action_ops *ops)
|
||||
const struct tc_action_ops *ops,
|
||||
struct netlink_ext_ack *extack)
|
||||
{
|
||||
struct nlattr *nest;
|
||||
int n_i = 0;
|
||||
@ -303,20 +304,25 @@ static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb,
|
||||
if (nla_put_string(skb, TCA_KIND, ops->kind))
|
||||
goto nla_put_failure;
|
||||
|
||||
ret = 0;
|
||||
mutex_lock(&idrinfo->lock);
|
||||
idr_for_each_entry_ul(idr, p, tmp, id) {
|
||||
if (IS_ERR(p))
|
||||
continue;
|
||||
ret = tcf_idr_release_unsafe(p);
|
||||
if (ret == ACT_P_DELETED) {
|
||||
if (ret == ACT_P_DELETED)
|
||||
module_put(ops->owner);
|
||||
n_i++;
|
||||
} else if (ret < 0) {
|
||||
mutex_unlock(&idrinfo->lock);
|
||||
goto nla_put_failure;
|
||||
}
|
||||
else if (ret < 0)
|
||||
break;
|
||||
n_i++;
|
||||
}
|
||||
mutex_unlock(&idrinfo->lock);
|
||||
if (ret < 0) {
|
||||
if (n_i)
|
||||
NL_SET_ERR_MSG(extack, "Unable to flush all TC actions");
|
||||
else
|
||||
goto nla_put_failure;
|
||||
}
|
||||
|
||||
ret = nla_put_u32(skb, TCA_FCNT, n_i);
|
||||
if (ret)
|
||||
@ -337,7 +343,7 @@ int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb,
|
||||
struct tcf_idrinfo *idrinfo = tn->idrinfo;
|
||||
|
||||
if (type == RTM_DELACTION) {
|
||||
return tcf_del_walker(idrinfo, skb, ops);
|
||||
return tcf_del_walker(idrinfo, skb, ops, extack);
|
||||
} else if (type == RTM_GETACTION) {
|
||||
return tcf_dump_walker(idrinfo, skb, cb);
|
||||
} else {
|
||||
|
||||
@ -608,7 +608,7 @@ static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
|
||||
*/
|
||||
xdr->p = (void *)p + frag2bytes;
|
||||
space_left = xdr->buf->buflen - xdr->buf->len;
|
||||
if (space_left - nbytes >= PAGE_SIZE)
|
||||
if (space_left - frag1bytes >= PAGE_SIZE)
|
||||
xdr->end = (void *)p + PAGE_SIZE;
|
||||
else
|
||||
xdr->end = (void *)p + space_left - frag1bytes;
|
||||
|
||||
@ -120,7 +120,7 @@ run_all() {
|
||||
run_udp "${ipv4_args}"
|
||||
|
||||
echo "ipv6"
|
||||
run_tcp "${ipv4_args}"
|
||||
run_tcp "${ipv6_args}"
|
||||
run_udp "${ipv6_args}"
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ endif
|
||||
|
||||
CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L$(OUTPUT) -Wl,-rpath=./ \
|
||||
$(CLANG_FLAGS)
|
||||
LDLIBS += -lpthread
|
||||
LDLIBS += -lpthread -ldl
|
||||
|
||||
# Own dependencies because we only want to build against 1st prerequisite, but
|
||||
# still track changes to header files and depend on shared object.
|
||||
|
||||
@ -9,10 +9,9 @@
|
||||
#include <string.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "../kselftest.h"
|
||||
#include "rseq.h"
|
||||
|
||||
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
|
||||
|
||||
struct percpu_lock_entry {
|
||||
intptr_t v;
|
||||
} __attribute__((aligned(128)));
|
||||
@ -168,7 +167,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
|
||||
for (;;) {
|
||||
struct percpu_list_node *head;
|
||||
intptr_t *targetptr, expectnot, *load;
|
||||
off_t offset;
|
||||
long offset;
|
||||
int ret, cpu;
|
||||
|
||||
cpu = rseq_cpu_start();
|
||||
|
||||
30
tools/testing/selftests/rseq/compiler.h
Normal file
30
tools/testing/selftests/rseq/compiler.h
Normal file
@ -0,0 +1,30 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
|
||||
/*
|
||||
* rseq/compiler.h
|
||||
*
|
||||
* Work-around asm goto compiler bugs.
|
||||
*
|
||||
* (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef RSEQ_COMPILER_H
|
||||
#define RSEQ_COMPILER_H
|
||||
|
||||
/*
|
||||
* gcc prior to 4.8.2 miscompiles asm goto.
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670
|
||||
*
|
||||
* gcc prior to 8.1.0 miscompiles asm goto at O1.
|
||||
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103908
|
||||
*
|
||||
* clang prior to version 13.0.1 miscompiles asm goto at O2.
|
||||
* https://github.com/llvm/llvm-project/issues/52735
|
||||
*
|
||||
* Work around these issues by adding a volatile inline asm with
|
||||
* memory clobber in the fallthrough after the asm goto and at each
|
||||
* label target. Emit this for all compilers in case other similar
|
||||
* issues are found in the future.
|
||||
*/
|
||||
#define rseq_after_asm_goto() asm volatile ("" : : : "memory")
|
||||
|
||||
#endif /* RSEQ_COMPILER_H_ */
|
||||
@ -159,7 +159,7 @@ unsigned int yield_mod_cnt, nr_abort;
|
||||
" cbnz " INJECT_ASM_REG ", 222b\n" \
|
||||
"333:\n"
|
||||
|
||||
#elif __PPC__
|
||||
#elif defined(__PPC__)
|
||||
|
||||
#define RSEQ_INJECT_INPUT \
|
||||
, [loop_cnt_1]"m"(loop_cnt[1]) \
|
||||
@ -366,9 +366,7 @@ void *test_percpu_spinlock_thread(void *arg)
|
||||
abort();
|
||||
reps = thread_data->reps;
|
||||
for (i = 0; i < reps; i++) {
|
||||
int cpu = rseq_cpu_start();
|
||||
|
||||
cpu = rseq_this_cpu_lock(&data->lock);
|
||||
int cpu = rseq_this_cpu_lock(&data->lock);
|
||||
data->c[cpu].count++;
|
||||
rseq_percpu_unlock(&data->lock, cpu);
|
||||
#ifndef BENCHMARK
|
||||
@ -549,7 +547,7 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list,
|
||||
for (;;) {
|
||||
struct percpu_list_node *head;
|
||||
intptr_t *targetptr, expectnot, *load;
|
||||
off_t offset;
|
||||
long offset;
|
||||
int ret;
|
||||
|
||||
cpu = rseq_cpu_start();
|
||||
|
||||
151
tools/testing/selftests/rseq/rseq-abi.h
Normal file
151
tools/testing/selftests/rseq/rseq-abi.h
Normal file
@ -0,0 +1,151 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
|
||||
#ifndef _RSEQ_ABI_H
|
||||
#define _RSEQ_ABI_H
|
||||
|
||||
/*
|
||||
* rseq-abi.h
|
||||
*
|
||||
* Restartable sequences system call API
|
||||
*
|
||||
* Copyright (c) 2015-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <asm/byteorder.h>
|
||||
|
||||
enum rseq_abi_cpu_id_state {
|
||||
RSEQ_ABI_CPU_ID_UNINITIALIZED = -1,
|
||||
RSEQ_ABI_CPU_ID_REGISTRATION_FAILED = -2,
|
||||
};
|
||||
|
||||
enum rseq_abi_flags {
|
||||
RSEQ_ABI_FLAG_UNREGISTER = (1 << 0),
|
||||
};
|
||||
|
||||
enum rseq_abi_cs_flags_bit {
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0,
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1,
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2,
|
||||
};
|
||||
|
||||
enum rseq_abi_cs_flags {
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT =
|
||||
(1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT),
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL =
|
||||
(1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT),
|
||||
RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE =
|
||||
(1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT),
|
||||
};
|
||||
|
||||
/*
|
||||
* struct rseq_abi_cs is aligned on 4 * 8 bytes to ensure it is always
|
||||
* contained within a single cache-line. It is usually declared as
|
||||
* link-time constant data.
|
||||
*/
|
||||
struct rseq_abi_cs {
|
||||
/* Version of this structure. */
|
||||
__u32 version;
|
||||
/* enum rseq_abi_cs_flags */
|
||||
__u32 flags;
|
||||
__u64 start_ip;
|
||||
/* Offset from start_ip. */
|
||||
__u64 post_commit_offset;
|
||||
__u64 abort_ip;
|
||||
} __attribute__((aligned(4 * sizeof(__u64))));
|
||||
|
||||
/*
|
||||
* struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always
|
||||
* contained within a single cache-line.
|
||||
*
|
||||
* A single struct rseq_abi per thread is allowed.
|
||||
*/
|
||||
struct rseq_abi {
|
||||
/*
|
||||
* Restartable sequences cpu_id_start field. Updated by the
|
||||
* kernel. Read by user-space with single-copy atomicity
|
||||
* semantics. This field should only be read by the thread which
|
||||
* registered this data structure. Aligned on 32-bit. Always
|
||||
* contains a value in the range of possible CPUs, although the
|
||||
* value may not be the actual current CPU (e.g. if rseq is not
|
||||
* initialized). This CPU number value should always be compared
|
||||
* against the value of the cpu_id field before performing a rseq
|
||||
* commit or returning a value read from a data structure indexed
|
||||
* using the cpu_id_start value.
|
||||
*/
|
||||
__u32 cpu_id_start;
|
||||
/*
|
||||
* Restartable sequences cpu_id field. Updated by the kernel.
|
||||
* Read by user-space with single-copy atomicity semantics. This
|
||||
* field should only be read by the thread which registered this
|
||||
* data structure. Aligned on 32-bit. Values
|
||||
* RSEQ_CPU_ID_UNINITIALIZED and RSEQ_CPU_ID_REGISTRATION_FAILED
|
||||
* have a special semantic: the former means "rseq uninitialized",
|
||||
* and latter means "rseq initialization failed". This value is
|
||||
* meant to be read within rseq critical sections and compared
|
||||
* with the cpu_id_start value previously read, before performing
|
||||
* the commit instruction, or read and compared with the
|
||||
* cpu_id_start value before returning a value loaded from a data
|
||||
* structure indexed using the cpu_id_start value.
|
||||
*/
|
||||
__u32 cpu_id;
|
||||
/*
|
||||
* Restartable sequences rseq_cs field.
|
||||
*
|
||||
* Contains NULL when no critical section is active for the current
|
||||
* thread, or holds a pointer to the currently active struct rseq_cs.
|
||||
*
|
||||
* Updated by user-space, which sets the address of the currently
|
||||
* active rseq_cs at the beginning of assembly instruction sequence
|
||||
* block, and set to NULL by the kernel when it restarts an assembly
|
||||
* instruction sequence block, as well as when the kernel detects that
|
||||
* it is preempting or delivering a signal outside of the range
|
||||
* targeted by the rseq_cs. Also needs to be set to NULL by user-space
|
||||
* before reclaiming memory that contains the targeted struct rseq_cs.
|
||||
*
|
||||
* Read and set by the kernel. Set by user-space with single-copy
|
||||
* atomicity semantics. This field should only be updated by the
|
||||
* thread which registered this data structure. Aligned on 64-bit.
|
||||
*/
|
||||
union {
|
||||
__u64 ptr64;
|
||||
|
||||
/*
|
||||
* The "arch" field provides architecture accessor for
|
||||
* the ptr field based on architecture pointer size and
|
||||
* endianness.
|
||||
*/
|
||||
struct {
|
||||
#ifdef __LP64__
|
||||
__u64 ptr;
|
||||
#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN)
|
||||
__u32 padding; /* Initialized to zero. */
|
||||
__u32 ptr;
|
||||
#else
|
||||
__u32 ptr;
|
||||
__u32 padding; /* Initialized to zero. */
|
||||
#endif
|
||||
} arch;
|
||||
} rseq_cs;
|
||||
|
||||
/*
|
||||
* Restartable sequences flags field.
|
||||
*
|
||||
* This field should only be updated by the thread which
|
||||
* registered this data structure. Read by the kernel.
|
||||
* Mainly used for single-stepping through rseq critical sections
|
||||
* with debuggers.
|
||||
*
|
||||
* - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT
|
||||
* Inhibit instruction sequence block restart on preemption
|
||||
* for this thread.
|
||||
* - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL
|
||||
* Inhibit instruction sequence block restart on signal
|
||||
* delivery for this thread.
|
||||
* - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE
|
||||
* Inhibit instruction sequence block restart on migration for
|
||||
* this thread.
|
||||
*/
|
||||
__u32 flags;
|
||||
} __attribute__((aligned(4 * sizeof(__u64))));
|
||||
|
||||
#endif /* _RSEQ_ABI_H */
|
||||
@ -147,14 +147,11 @@ do { \
|
||||
teardown \
|
||||
"b %l[" __rseq_str(cmpfail_label) "]\n\t"
|
||||
|
||||
#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -185,8 +182,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -198,30 +195,31 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -255,8 +253,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -270,19 +268,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -292,7 +292,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
@ -316,8 +315,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[count] "Ir" (count)
|
||||
RSEQ_INJECT_INPUT
|
||||
@ -328,14 +327,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -347,7 +347,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -381,8 +380,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -398,19 +397,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -422,7 +423,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -457,8 +457,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -474,19 +474,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -498,7 +500,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -537,8 +538,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -554,21 +555,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -582,7 +586,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -657,8 +660,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -678,21 +681,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -706,7 +709,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -782,8 +784,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -803,21 +805,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -230,8 +230,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "Qo" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -242,24 +242,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
@ -287,8 +291,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "Qo" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
[load] "Qo" (*load),
|
||||
@ -300,16 +304,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -337,8 +346,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "Qo" (*v),
|
||||
[count] "r" (count)
|
||||
RSEQ_INJECT_INPUT
|
||||
@ -348,12 +357,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -388,8 +400,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[expect] "r" (expect),
|
||||
[v] "Qo" (*v),
|
||||
[newv] "r" (newv),
|
||||
@ -402,17 +414,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -447,8 +463,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[expect] "r" (expect),
|
||||
[v] "Qo" (*v),
|
||||
[newv] "r" (newv),
|
||||
@ -461,17 +477,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -508,8 +528,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "Qo" (*v),
|
||||
[expect] "r" (expect),
|
||||
[v2] "Qo" (*v2),
|
||||
@ -522,19 +542,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -569,8 +594,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[expect] "r" (expect),
|
||||
[v] "Qo" (*v),
|
||||
[newv] "r" (newv),
|
||||
@ -584,17 +609,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -629,8 +658,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "Qo" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[expect] "r" (expect),
|
||||
[v] "Qo" (*v),
|
||||
[newv] "r" (newv),
|
||||
@ -644,17 +673,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
25
tools/testing/selftests/rseq/rseq-generic-thread-pointer.h
Normal file
25
tools/testing/selftests/rseq/rseq-generic-thread-pointer.h
Normal file
@ -0,0 +1,25 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
|
||||
/*
|
||||
* rseq-generic-thread-pointer.h
|
||||
*
|
||||
* (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef _RSEQ_GENERIC_THREAD_POINTER
|
||||
#define _RSEQ_GENERIC_THREAD_POINTER
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Use gcc builtin thread pointer. */
|
||||
static inline void *rseq_thread_pointer(void)
|
||||
{
|
||||
return __builtin_thread_pointer();
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@ -154,14 +154,11 @@ do { \
|
||||
teardown \
|
||||
"b %l[" __rseq_str(cmpfail_label) "]\n\t"
|
||||
|
||||
#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -190,8 +187,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -203,14 +200,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
@ -222,11 +216,10 @@ error2:
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -258,8 +251,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -273,14 +266,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
@ -295,7 +285,6 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
@ -319,8 +308,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[count] "Ir" (count)
|
||||
RSEQ_INJECT_INPUT
|
||||
@ -331,10 +320,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
@ -350,7 +337,6 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -382,8 +368,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -399,14 +385,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
@ -423,7 +406,6 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -456,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -473,14 +455,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
@ -497,7 +476,6 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -532,8 +510,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -549,14 +527,11 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
@ -577,7 +552,6 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -649,8 +623,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -670,21 +644,16 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -698,7 +667,6 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
|
||||
@ -771,8 +739,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -792,21 +760,16 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
30
tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h
Normal file
30
tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h
Normal file
@ -0,0 +1,30 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
|
||||
/*
|
||||
* rseq-ppc-thread-pointer.h
|
||||
*
|
||||
* (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef _RSEQ_PPC_THREAD_POINTER
|
||||
#define _RSEQ_PPC_THREAD_POINTER
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static inline void *rseq_thread_pointer(void)
|
||||
{
|
||||
#ifdef __powerpc64__
|
||||
register void *__result asm ("r13");
|
||||
#else
|
||||
register void *__result asm ("r2");
|
||||
#endif
|
||||
asm ("" : "=r" (__result));
|
||||
return __result;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@ -47,10 +47,13 @@ do { \
|
||||
|
||||
#ifdef __PPC64__
|
||||
|
||||
#define STORE_WORD "std "
|
||||
#define LOAD_WORD "ld "
|
||||
#define LOADX_WORD "ldx "
|
||||
#define CMP_WORD "cmpd "
|
||||
#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
|
||||
#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
|
||||
#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
|
||||
#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
|
||||
#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
|
||||
#define RSEQ_CMP_LONG "cmpd "
|
||||
#define RSEQ_CMP_LONG_INT "cmpdi "
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
|
||||
start_ip, post_commit_offset, abort_ip) \
|
||||
@ -89,10 +92,13 @@ do { \
|
||||
|
||||
#else /* #ifdef __PPC64__ */
|
||||
|
||||
#define STORE_WORD "stw "
|
||||
#define LOAD_WORD "lwz "
|
||||
#define LOADX_WORD "lwzx "
|
||||
#define CMP_WORD "cmpw "
|
||||
#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
|
||||
#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */
|
||||
#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
|
||||
#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
|
||||
#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
|
||||
#define RSEQ_CMP_LONG "cmpw "
|
||||
#define RSEQ_CMP_LONG_INT "cmpwi "
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
|
||||
start_ip, post_commit_offset, abort_ip) \
|
||||
@ -125,7 +131,7 @@ do { \
|
||||
RSEQ_INJECT_ASM(1) \
|
||||
"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
|
||||
"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
|
||||
"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
|
||||
RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
|
||||
__rseq_str(label) ":\n\t"
|
||||
|
||||
#endif /* #ifdef __PPC64__ */
|
||||
@ -136,7 +142,7 @@ do { \
|
||||
|
||||
#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
|
||||
RSEQ_INJECT_ASM(2) \
|
||||
"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
|
||||
RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
|
||||
"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
|
||||
"bne- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
@ -153,25 +159,25 @@ do { \
|
||||
* RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
|
||||
*/
|
||||
#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
|
||||
RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
|
||||
"bne- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
|
||||
RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
|
||||
"beq- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_STORE(value, var) \
|
||||
STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
|
||||
RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Load @var to r17 */
|
||||
#define RSEQ_ASM_OP_R_LOAD(var) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Store r17 to @var */
|
||||
#define RSEQ_ASM_OP_R_STORE(var) \
|
||||
STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Add @count to r17 */
|
||||
#define RSEQ_ASM_OP_R_ADD(count) \
|
||||
@ -179,11 +185,11 @@ do { \
|
||||
|
||||
/* Load (r17 + voffp) to r17 */
|
||||
#define RSEQ_ASM_OP_R_LOADX(voffp) \
|
||||
LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
|
||||
RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
|
||||
|
||||
/* TODO: implement a faster memcpy. */
|
||||
#define RSEQ_ASM_OP_R_MEMCPY() \
|
||||
"cmpdi %%r19, 0\n\t" \
|
||||
RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
|
||||
"beq 333f\n\t" \
|
||||
"addi %%r20, %%r20, -1\n\t" \
|
||||
"addi %%r21, %%r21, -1\n\t" \
|
||||
@ -191,16 +197,16 @@ do { \
|
||||
"lbzu %%r18, 1(%%r20)\n\t" \
|
||||
"stbu %%r18, 1(%%r21)\n\t" \
|
||||
"addi %%r19, %%r19, -1\n\t" \
|
||||
"cmpdi %%r19, 0\n\t" \
|
||||
RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
|
||||
"bne 222b\n\t" \
|
||||
"333:\n\t" \
|
||||
|
||||
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
|
||||
STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
__rseq_str(post_commit_label) ":\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
|
||||
STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
|
||||
RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
|
||||
__rseq_str(post_commit_label) ":\n\t"
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
@ -235,8 +241,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -248,23 +254,28 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
@ -301,8 +312,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -316,16 +327,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -359,8 +375,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[count] "r" (count)
|
||||
@ -372,12 +388,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -419,8 +438,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -436,16 +455,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -489,8 +513,8 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -506,16 +530,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -560,8 +589,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -577,18 +606,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -635,8 +670,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -653,16 +688,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -711,8 +751,8 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -729,23 +769,23 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
#undef STORE_WORD
|
||||
#undef LOAD_WORD
|
||||
#undef LOADX_WORD
|
||||
#undef CMP_WORD
|
||||
|
||||
#endif /* !RSEQ_SKIP_FASTPATH */
|
||||
|
||||
@ -165,8 +165,8 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -178,16 +178,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -198,7 +203,7 @@ error2:
|
||||
*/
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
@ -233,8 +238,8 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -248,16 +253,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -288,8 +298,8 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[count] "r" (count)
|
||||
@ -301,12 +311,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -347,8 +360,8 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -364,16 +377,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -426,8 +444,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -443,18 +461,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -534,8 +558,8 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
#endif
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
|
||||
[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -555,16 +579,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
19
tools/testing/selftests/rseq/rseq-thread-pointer.h
Normal file
19
tools/testing/selftests/rseq/rseq-thread-pointer.h
Normal file
@ -0,0 +1,19 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
|
||||
/*
|
||||
* rseq-thread-pointer.h
|
||||
*
|
||||
* (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef _RSEQ_THREAD_POINTER
|
||||
#define _RSEQ_THREAD_POINTER
|
||||
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
#include "rseq-x86-thread-pointer.h"
|
||||
#elif defined(__PPC__)
|
||||
#include "rseq-ppc-thread-pointer.h"
|
||||
#else
|
||||
#include "rseq-generic-thread-pointer.h"
|
||||
#endif
|
||||
|
||||
#endif
|
||||
40
tools/testing/selftests/rseq/rseq-x86-thread-pointer.h
Normal file
40
tools/testing/selftests/rseq/rseq-x86-thread-pointer.h
Normal file
@ -0,0 +1,40 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
|
||||
/*
|
||||
* rseq-x86-thread-pointer.h
|
||||
*
|
||||
* (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef _RSEQ_X86_THREAD_POINTER
|
||||
#define _RSEQ_X86_THREAD_POINTER
|
||||
|
||||
#include <features.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if __GNUC_PREREQ (11, 1)
|
||||
static inline void *rseq_thread_pointer(void)
|
||||
{
|
||||
return __builtin_thread_pointer();
|
||||
}
|
||||
#else
|
||||
static inline void *rseq_thread_pointer(void)
|
||||
{
|
||||
void *__result;
|
||||
|
||||
# ifdef __x86_64__
|
||||
__asm__ ("mov %%fs:0, %0" : "=r" (__result));
|
||||
# else
|
||||
__asm__ ("mov %%gs:0, %0" : "=r" (__result));
|
||||
# endif
|
||||
return __result;
|
||||
}
|
||||
#endif /* !GCC 11 */
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@ -28,6 +28,8 @@
|
||||
|
||||
#ifdef __x86_64__
|
||||
|
||||
#define RSEQ_ASM_TP_SEGMENT %%fs
|
||||
|
||||
#define rseq_smp_mb() \
|
||||
__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
|
||||
#define rseq_smp_rmb() rseq_barrier()
|
||||
@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
#endif
|
||||
@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -152,16 +154,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -172,7 +179,7 @@ error2:
|
||||
*/
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
@ -184,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"movq %[v], %%rbx\n\t"
|
||||
"cmpq %%rbx, %[expectnot]\n\t"
|
||||
"je %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"movq %[v], %%rbx\n\t"
|
||||
"cmpq %%rbx, %[expectnot]\n\t"
|
||||
"je %l[error2]\n\t"
|
||||
@ -207,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -220,16 +227,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -245,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
#endif
|
||||
/* final store */
|
||||
"addq %[count], %[v]\n\t"
|
||||
@ -258,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[count] "er" (count)
|
||||
@ -267,6 +279,66 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
: abort
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
|
||||
|
||||
/*
|
||||
* pval = *(ptr+off)
|
||||
* *pval += inc;
|
||||
*/
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
#endif
|
||||
/* get p+v */
|
||||
"movq %[ptr], %%rbx\n\t"
|
||||
"addq %[off], %%rbx\n\t"
|
||||
/* get pv */
|
||||
"movq (%%rbx), %%rcx\n\t"
|
||||
/* *pv += inc */
|
||||
"addq %[inc], (%%rcx)\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[ptr] "m" (*ptr),
|
||||
[off] "er" (off),
|
||||
[inc] "er" (inc)
|
||||
: "memory", "cc", "rax", "rbx", "rcx"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
@ -294,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
#endif
|
||||
@ -315,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -330,16 +402,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -369,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
@ -379,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
"cmpq %[v2], %[expect2]\n\t"
|
||||
@ -392,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -407,18 +484,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -443,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
"movq %[dst], %[rseq_scratch1]\n\t"
|
||||
"movq %[len], %[rseq_scratch2]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz 5f\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
|
||||
"cmpq %[v], %[expect]\n\t"
|
||||
"jnz 7f\n\t"
|
||||
#endif
|
||||
@ -498,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
#endif
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
@ -517,16 +600,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -543,7 +631,9 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
|
||||
#endif /* !RSEQ_SKIP_FASTPATH */
|
||||
|
||||
#elif __i386__
|
||||
#elif defined(__i386__)
|
||||
|
||||
#define RSEQ_ASM_TP_SEGMENT %%gs
|
||||
|
||||
#define rseq_smp_mb() \
|
||||
__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
|
||||
@ -644,14 +734,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
#endif
|
||||
@ -662,7 +752,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
@ -673,16 +763,21 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -693,7 +788,7 @@ error2:
|
||||
*/
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
long voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
@ -705,15 +800,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"movl %[v], %%ebx\n\t"
|
||||
"cmpl %%ebx, %[expectnot]\n\t"
|
||||
"je %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"movl %[v], %%ebx\n\t"
|
||||
"cmpl %%ebx, %[expectnot]\n\t"
|
||||
"je %l[error2]\n\t"
|
||||
@ -728,7 +823,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
@ -741,16 +836,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -766,11 +866,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
#endif
|
||||
/* final store */
|
||||
"addl %[count], %[v]\n\t"
|
||||
@ -779,7 +879,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[count] "ir" (count)
|
||||
@ -790,12 +890,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -815,14 +918,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
#endif
|
||||
@ -837,7 +940,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "m" (newv2),
|
||||
@ -852,16 +955,21 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -881,15 +989,15 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %[v], %%eax\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %[v], %%eax\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
@ -905,7 +1013,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
@ -920,16 +1028,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
|
||||
@ -951,8 +1064,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
|
||||
#endif
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
@ -961,7 +1074,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
"jnz %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||
"cmpl %[v], %[expect]\n\t"
|
||||
"jnz %l[error2]\n\t"
|
||||
"cmpl %[expect2], %[v2]\n\t"
|
||||
@ -975,7 +1088,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
@ -990,18 +1103,24 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -1027,15 +1146,15 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
"movl %[dst], %[rseq_scratch1]\n\t"
|
||||
"movl %[len], %[rseq_scratch2]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %%eax, %[v]\n\t"
|
||||
"jnz 5f\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %%eax, %[v]\n\t"
|
||||
"jnz 7f\n\t"
|
||||
@ -1085,7 +1204,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
#endif
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "m" (expect),
|
||||
@ -1104,16 +1223,21 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
@ -1139,15 +1263,15 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
"movl %[dst], %[rseq_scratch1]\n\t"
|
||||
"movl %[len], %[rseq_scratch2]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %%eax, %[v]\n\t"
|
||||
"jnz 5f\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
|
||||
"movl %[expect], %%eax\n\t"
|
||||
"cmpl %%eax, %[v]\n\t"
|
||||
"jnz 7f\n\t"
|
||||
@ -1198,7 +1322,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
#endif
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[rseq_abi] "r" (&__rseq_abi),
|
||||
[rseq_offset] "r" (rseq_offset),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "m" (expect),
|
||||
@ -1217,16 +1341,21 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_after_asm_goto();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_after_asm_goto();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_after_asm_goto();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_after_asm_goto();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -26,104 +26,114 @@
|
||||
#include <assert.h>
|
||||
#include <signal.h>
|
||||
#include <limits.h>
|
||||
#include <dlfcn.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "../kselftest.h"
|
||||
#include "rseq.h"
|
||||
|
||||
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
|
||||
static const ptrdiff_t *libc_rseq_offset_p;
|
||||
static const unsigned int *libc_rseq_size_p;
|
||||
static const unsigned int *libc_rseq_flags_p;
|
||||
|
||||
__thread volatile struct rseq __rseq_abi = {
|
||||
.cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
|
||||
};
|
||||
/* Offset from the thread pointer to the rseq area. */
|
||||
ptrdiff_t rseq_offset;
|
||||
|
||||
/*
|
||||
* Shared with other libraries. This library may take rseq ownership if it is
|
||||
* still 0 when executing the library constructor. Set to 1 by library
|
||||
* constructor when handling rseq. Set to 0 in destructor if handling rseq.
|
||||
*/
|
||||
int __rseq_handled;
|
||||
/* Size of the registered rseq area. 0 if the registration was
|
||||
unsuccessful. */
|
||||
unsigned int rseq_size = -1U;
|
||||
|
||||
/* Flags used during rseq registration. */
|
||||
unsigned int rseq_flags;
|
||||
|
||||
/* Whether this library have ownership of rseq registration. */
|
||||
static int rseq_ownership;
|
||||
|
||||
static __thread volatile uint32_t __rseq_refcount;
|
||||
static
|
||||
__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = {
|
||||
.cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED,
|
||||
};
|
||||
|
||||
static void signal_off_save(sigset_t *oldset)
|
||||
{
|
||||
sigset_t set;
|
||||
int ret;
|
||||
|
||||
sigfillset(&set);
|
||||
ret = pthread_sigmask(SIG_BLOCK, &set, oldset);
|
||||
if (ret)
|
||||
abort();
|
||||
}
|
||||
|
||||
static void signal_restore(sigset_t oldset)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
|
||||
if (ret)
|
||||
abort();
|
||||
}
|
||||
|
||||
static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len,
|
||||
static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len,
|
||||
int flags, uint32_t sig)
|
||||
{
|
||||
return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig);
|
||||
}
|
||||
|
||||
int rseq_available(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
rc = sys_rseq(NULL, 0, 0, 0);
|
||||
if (rc != -1)
|
||||
abort();
|
||||
switch (errno) {
|
||||
case ENOSYS:
|
||||
return 0;
|
||||
case EINVAL:
|
||||
return 1;
|
||||
default:
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
int rseq_register_current_thread(void)
|
||||
{
|
||||
int rc, ret = 0;
|
||||
sigset_t oldset;
|
||||
int rc;
|
||||
|
||||
if (!rseq_ownership)
|
||||
if (!rseq_ownership) {
|
||||
/* Treat libc's ownership as a successful registration. */
|
||||
return 0;
|
||||
signal_off_save(&oldset);
|
||||
if (__rseq_refcount == UINT_MAX) {
|
||||
ret = -1;
|
||||
goto end;
|
||||
}
|
||||
if (__rseq_refcount++)
|
||||
goto end;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG);
|
||||
if (!rc) {
|
||||
assert(rseq_current_cpu_raw() >= 0);
|
||||
goto end;
|
||||
}
|
||||
if (errno != EBUSY)
|
||||
__rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED;
|
||||
ret = -1;
|
||||
__rseq_refcount--;
|
||||
end:
|
||||
signal_restore(oldset);
|
||||
return ret;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG);
|
||||
if (rc)
|
||||
return -1;
|
||||
assert(rseq_current_cpu_raw() >= 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int rseq_unregister_current_thread(void)
|
||||
{
|
||||
int rc, ret = 0;
|
||||
sigset_t oldset;
|
||||
int rc;
|
||||
|
||||
if (!rseq_ownership)
|
||||
if (!rseq_ownership) {
|
||||
/* Treat libc's ownership as a successful unregistration. */
|
||||
return 0;
|
||||
signal_off_save(&oldset);
|
||||
if (!__rseq_refcount) {
|
||||
ret = -1;
|
||||
goto end;
|
||||
}
|
||||
if (--__rseq_refcount)
|
||||
goto end;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq),
|
||||
RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
|
||||
if (!rc)
|
||||
goto end;
|
||||
__rseq_refcount = 1;
|
||||
ret = -1;
|
||||
end:
|
||||
signal_restore(oldset);
|
||||
return ret;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
|
||||
if (rc)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static __attribute__((constructor))
|
||||
void rseq_init(void)
|
||||
{
|
||||
libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset");
|
||||
libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size");
|
||||
libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags");
|
||||
if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p) {
|
||||
/* rseq registration owned by glibc */
|
||||
rseq_offset = *libc_rseq_offset_p;
|
||||
rseq_size = *libc_rseq_size_p;
|
||||
rseq_flags = *libc_rseq_flags_p;
|
||||
return;
|
||||
}
|
||||
if (!rseq_available())
|
||||
return;
|
||||
rseq_ownership = 1;
|
||||
rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer();
|
||||
rseq_size = sizeof(struct rseq_abi);
|
||||
rseq_flags = 0;
|
||||
}
|
||||
|
||||
static __attribute__((destructor))
|
||||
void rseq_exit(void)
|
||||
{
|
||||
if (!rseq_ownership)
|
||||
return;
|
||||
rseq_offset = 0;
|
||||
rseq_size = -1U;
|
||||
rseq_ownership = 0;
|
||||
}
|
||||
|
||||
int32_t rseq_fallback_current_cpu(void)
|
||||
@ -137,20 +147,3 @@ int32_t rseq_fallback_current_cpu(void)
|
||||
}
|
||||
return cpu;
|
||||
}
|
||||
|
||||
void __attribute__((constructor)) rseq_init(void)
|
||||
{
|
||||
/* Check whether rseq is handled by another library. */
|
||||
if (__rseq_handled)
|
||||
return;
|
||||
__rseq_handled = 1;
|
||||
rseq_ownership = 1;
|
||||
}
|
||||
|
||||
void __attribute__((destructor)) rseq_fini(void)
|
||||
{
|
||||
if (!rseq_ownership)
|
||||
return;
|
||||
__rseq_handled = 0;
|
||||
rseq_ownership = 0;
|
||||
}
|
||||
|
||||
@ -16,7 +16,9 @@
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <linux/rseq.h>
|
||||
#include <stddef.h>
|
||||
#include "rseq-abi.h"
|
||||
#include "compiler.h"
|
||||
|
||||
/*
|
||||
* Empty code injection macros, override when testing.
|
||||
@ -43,8 +45,20 @@
|
||||
#define RSEQ_INJECT_FAILED
|
||||
#endif
|
||||
|
||||
extern __thread volatile struct rseq __rseq_abi;
|
||||
extern int __rseq_handled;
|
||||
#include "rseq-thread-pointer.h"
|
||||
|
||||
/* Offset from the thread pointer to the rseq area. */
|
||||
extern ptrdiff_t rseq_offset;
|
||||
/* Size of the registered rseq area. 0 if the registration was
|
||||
unsuccessful. */
|
||||
extern unsigned int rseq_size;
|
||||
/* Flags used during rseq registration. */
|
||||
extern unsigned int rseq_flags;
|
||||
|
||||
static inline struct rseq_abi *rseq_get_abi(void)
|
||||
{
|
||||
return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
|
||||
}
|
||||
|
||||
#define rseq_likely(x) __builtin_expect(!!(x), 1)
|
||||
#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
|
||||
@ -108,7 +122,7 @@ int32_t rseq_fallback_current_cpu(void);
|
||||
*/
|
||||
static inline int32_t rseq_current_cpu_raw(void)
|
||||
{
|
||||
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id);
|
||||
return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -124,7 +138,7 @@ static inline int32_t rseq_current_cpu_raw(void)
|
||||
*/
|
||||
static inline uint32_t rseq_cpu_start(void)
|
||||
{
|
||||
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start);
|
||||
return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start);
|
||||
}
|
||||
|
||||
static inline uint32_t rseq_current_cpu(void)
|
||||
@ -139,11 +153,7 @@ static inline uint32_t rseq_current_cpu(void)
|
||||
|
||||
static inline void rseq_clear_rseq_cs(void)
|
||||
{
|
||||
#ifdef __LP64__
|
||||
__rseq_abi.rseq_cs.ptr = 0;
|
||||
#else
|
||||
__rseq_abi.rseq_cs.ptr.ptr32 = 0;
|
||||
#endif
|
||||
RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user