2013-05-13 13:29:47 +02:00
|
|
|
/*
|
|
|
|
* Simple interface for atomic operations.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2013 Red Hat, Inc.
|
|
|
|
*
|
|
|
|
* Author: Paolo Bonzini <pbonzini@redhat.com>
|
|
|
|
*
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
|
|
* See the COPYING file in the top-level directory.
|
|
|
|
*
|
2021-05-17 17:16:59 +02:00
|
|
|
* See docs/devel/atomics.rst for discussion about the guarantees each
|
2016-01-28 11:15:17 +01:00
|
|
|
* atomic primitive is meant to provide.
|
2013-05-13 13:29:47 +02:00
|
|
|
*/
|
2010-02-22 17:57:54 +01:00
|
|
|
|
2016-06-29 13:47:03 +02:00
|
|
|
#ifndef QEMU_ATOMIC_H
|
|
|
|
#define QEMU_ATOMIC_H
|
2012-10-06 12:46:15 +02:00
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
/* Compiler barrier */
|
|
|
|
#define barrier() ({ asm volatile("" ::: "memory"); (void)0; })
|
|
|
|
|
2016-08-09 21:02:24 +02:00
|
|
|
/* The variable that receives the old value of an atomically-accessed
|
|
|
|
* variable must be non-qualified, because atomic builtins return values
|
|
|
|
* through a pointer-type argument as in __atomic_load(&var, &old, MODEL).
|
|
|
|
*
|
|
|
|
* This macro has to handle types smaller than int manually, because of
|
|
|
|
* implicit promotion. int and larger types, as well as pointers, can be
|
|
|
|
* converted to a non-qualified type just by applying a binary operator.
|
|
|
|
*/
|
|
|
|
#define typeof_strip_qual(expr) \
|
|
|
|
typeof( \
|
|
|
|
__builtin_choose_expr( \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), bool) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const bool) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), volatile bool) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const volatile bool), \
|
|
|
|
(bool)1, \
|
|
|
|
__builtin_choose_expr( \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), signed char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const signed char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), volatile signed char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const volatile signed char), \
|
|
|
|
(signed char)1, \
|
|
|
|
__builtin_choose_expr( \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), unsigned char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const unsigned char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), volatile unsigned char) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const volatile unsigned char), \
|
|
|
|
(unsigned char)1, \
|
|
|
|
__builtin_choose_expr( \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), signed short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const signed short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), volatile signed short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const volatile signed short), \
|
|
|
|
(signed short)1, \
|
|
|
|
__builtin_choose_expr( \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), unsigned short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const unsigned short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), volatile unsigned short) || \
|
|
|
|
__builtin_types_compatible_p(typeof(expr), const volatile unsigned short), \
|
|
|
|
(unsigned short)1, \
|
|
|
|
(expr)+0))))))
|
|
|
|
|
2016-01-28 11:15:17 +01:00
|
|
|
#ifdef __ATOMIC_RELAXED
|
|
|
|
/* For C11 atomic ops */
|
|
|
|
|
|
|
|
/* Manual memory barriers
|
|
|
|
*
|
|
|
|
*__atomic_thread_fence does not include a compiler barrier; instead,
|
|
|
|
* the barrier is part of __atomic_load/__atomic_store's "volatile-like"
|
|
|
|
* semantics. If smp_wmb() is a no-op, absence of the barrier means that
|
|
|
|
* the compiler is free to reorder stores on each side of the barrier.
|
|
|
|
* Add one here, and similarly in smp_rmb() and smp_read_barrier_depends().
|
|
|
|
*/
|
|
|
|
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb() ({ barrier(); __atomic_thread_fence(__ATOMIC_SEQ_CST); })
|
|
|
|
#define smp_mb_release() ({ barrier(); __atomic_thread_fence(__ATOMIC_RELEASE); })
|
|
|
|
#define smp_mb_acquire() ({ barrier(); __atomic_thread_fence(__ATOMIC_ACQUIRE); })
|
2016-01-28 11:15:17 +01:00
|
|
|
|
2016-05-24 22:06:13 +02:00
|
|
|
/* Most compilers currently treat consume and acquire the same, but really
|
|
|
|
* no processors except Alpha need a barrier here. Leave it in if
|
|
|
|
* using Thread Sanitizer to avoid warnings, otherwise optimize it away.
|
|
|
|
*/
|
|
|
|
#if defined(__SANITIZE_THREAD__)
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_read_barrier_depends() ({ barrier(); __atomic_thread_fence(__ATOMIC_CONSUME); })
|
2016-09-30 23:30:52 +02:00
|
|
|
#elif defined(__alpha__)
|
2016-05-24 22:06:13 +02:00
|
|
|
#define smp_read_barrier_depends() asm volatile("mb":::"memory")
|
|
|
|
#else
|
|
|
|
#define smp_read_barrier_depends() barrier()
|
|
|
|
#endif
|
|
|
|
|
2019-07-09 10:40:00 +02:00
|
|
|
/*
|
|
|
|
* A signal barrier forces all pending local memory ops to be observed before
|
|
|
|
* a SIGSEGV is delivered to the *same* thread. In practice this is exactly
|
|
|
|
* the same as barrier(), but since we have the correct builtin, use it.
|
|
|
|
*/
|
|
|
|
#define signal_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST)
|
|
|
|
|
2017-04-26 19:41:55 +02:00
|
|
|
/* Sanity check that the size of an atomic operation isn't "overly large".
|
|
|
|
* Despite the fact that e.g. i686 has 64-bit atomic operations, we do not
|
|
|
|
* want to use them because we ought not need them, and this lets us do a
|
|
|
|
* bit of sanity checking that other 32-bit hosts might build.
|
|
|
|
*
|
|
|
|
* That said, we have a problem on 64-bit ILP32 hosts in that in order to
|
|
|
|
* sync with TCG_OVERSIZED_GUEST, this must match TCG_TARGET_REG_BITS.
|
|
|
|
* We'd prefer not want to pull in everything else TCG related, so handle
|
|
|
|
* those few cases by hand.
|
|
|
|
*
|
2018-09-03 19:18:28 +02:00
|
|
|
* Note that x32 is fully detected with __x86_64__ + _ILP32, and that for
|
2018-12-27 16:32:11 +01:00
|
|
|
* Sparc we always force the use of sparcv9 in configure. MIPS n32 (ILP32) &
|
|
|
|
* n64 (LP64) ABIs are both detected using __mips64.
|
2017-04-26 19:41:55 +02:00
|
|
|
*/
|
2018-12-27 16:32:11 +01:00
|
|
|
#if defined(__x86_64__) || defined(__sparc__) || defined(__mips64)
|
2017-04-26 19:41:55 +02:00
|
|
|
# define ATOMIC_REG_SIZE 8
|
|
|
|
#else
|
|
|
|
# define ATOMIC_REG_SIZE sizeof(void *)
|
|
|
|
#endif
|
2016-01-28 11:15:17 +01:00
|
|
|
|
|
|
|
/* Weak atomic operations prevent the compiler moving other
|
|
|
|
* loads/stores past the atomic operation load/store. However there is
|
|
|
|
* no explicit memory barrier for the processor.
|
2016-09-30 23:30:53 +02:00
|
|
|
*
|
|
|
|
* The C11 memory model says that variables that are accessed from
|
|
|
|
* different threads should at least be done with __ATOMIC_RELAXED
|
|
|
|
* primitives or the result is undefined. Generally this has little to
|
|
|
|
* no effect on the generated code but not using the atomic primitives
|
|
|
|
* will get flagged by sanitizers as a violation.
|
2016-01-28 11:15:17 +01:00
|
|
|
*/
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_read__nocheck(ptr) \
|
2016-10-24 19:29:32 +02:00
|
|
|
__atomic_load_n(ptr, __ATOMIC_RELAXED)
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_read(ptr) \
|
|
|
|
({ \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_read__nocheck(ptr); \
|
2016-01-28 11:15:17 +01:00
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_set__nocheck(ptr, i) \
|
2016-10-24 19:29:32 +02:00
|
|
|
__atomic_store_n(ptr, i, __ATOMIC_RELAXED)
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_set(ptr, i) do { \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_set__nocheck(ptr, i); \
|
2016-01-28 11:15:17 +01:00
|
|
|
} while(0)
|
|
|
|
|
2016-05-24 22:06:14 +02:00
|
|
|
/* See above: most compilers currently treat consume and acquire the
|
2020-09-23 12:56:46 +02:00
|
|
|
* same, but this slows down qatomic_rcu_read unnecessarily.
|
2016-05-24 22:06:14 +02:00
|
|
|
*/
|
|
|
|
#ifdef __SANITIZE_THREAD__
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_read__nocheck(ptr, valptr) \
|
2016-05-24 22:06:14 +02:00
|
|
|
__atomic_load(ptr, valptr, __ATOMIC_CONSUME);
|
|
|
|
#else
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_read__nocheck(ptr, valptr) \
|
|
|
|
__atomic_load(ptr, valptr, __ATOMIC_RELAXED); \
|
2016-05-24 22:06:14 +02:00
|
|
|
smp_read_barrier_depends();
|
|
|
|
#endif
|
2016-01-28 11:15:17 +01:00
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_read(ptr) \
|
|
|
|
({ \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
typeof_strip_qual(*ptr) _val; \
|
|
|
|
qatomic_rcu_read__nocheck(ptr, &_val); \
|
|
|
|
_val; \
|
2016-01-28 11:15:17 +01:00
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_set(ptr, i) do { \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
__atomic_store_n(ptr, i, __ATOMIC_RELEASE); \
|
2016-01-28 11:15:17 +01:00
|
|
|
} while(0)
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_load_acquire(ptr) \
|
2016-01-28 11:15:17 +01:00
|
|
|
({ \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2016-08-09 21:02:24 +02:00
|
|
|
typeof_strip_qual(*ptr) _val; \
|
2016-09-19 11:36:44 +02:00
|
|
|
__atomic_load(ptr, &_val, __ATOMIC_ACQUIRE); \
|
2016-01-28 11:15:17 +01:00
|
|
|
_val; \
|
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_store_release(ptr, i) do { \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2016-09-19 11:36:44 +02:00
|
|
|
__atomic_store_n(ptr, i, __ATOMIC_RELEASE); \
|
2016-01-28 11:15:17 +01:00
|
|
|
} while(0)
|
|
|
|
|
|
|
|
|
|
|
|
/* All the remaining operations are fully sequentially consistent */
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg__nocheck(ptr, i) ({ \
|
2016-10-24 19:29:32 +02:00
|
|
|
__atomic_exchange_n(ptr, (i), __ATOMIC_SEQ_CST); \
|
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg(ptr, i) ({ \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_xchg__nocheck(ptr, i); \
|
2016-01-28 11:15:17 +01:00
|
|
|
})
|
|
|
|
|
|
|
|
/* Returns the eventual value, failed or not */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_cmpxchg__nocheck(ptr, old, new) ({ \
|
2016-08-29 19:17:01 +02:00
|
|
|
typeof_strip_qual(*ptr) _old = (old); \
|
2018-05-10 19:10:57 +02:00
|
|
|
(void)__atomic_compare_exchange_n(ptr, &_old, new, false, \
|
2016-01-28 11:15:17 +01:00
|
|
|
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \
|
|
|
|
_old; \
|
2016-10-24 19:29:32 +02:00
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_cmpxchg(ptr, old, new) ({ \
|
2017-04-26 19:41:55 +02:00
|
|
|
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_cmpxchg__nocheck(ptr, old, new); \
|
2016-10-24 19:29:32 +02:00
|
|
|
})
|
2016-01-28 11:15:17 +01:00
|
|
|
|
|
|
|
/* Provide shorter names for GCC atomic builtins, return old value */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_fetch_inc(ptr) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_fetch_dec(ptr) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST)
|
|
|
|
|
|
|
|
#define qatomic_fetch_add(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_fetch_or(ptr, n) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_fetch_xor(ptr, n) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
|
|
|
|
#define qatomic_inc_fetch(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_dec_fetch(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_add_fetch(ptr, n) __atomic_add_fetch(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_sub_fetch(ptr, n) __atomic_sub_fetch(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_and_fetch(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_or_fetch(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST)
|
|
|
|
#define qatomic_xor_fetch(ptr, n) __atomic_xor_fetch(ptr, n, __ATOMIC_SEQ_CST)
|
2016-06-27 21:01:54 +02:00
|
|
|
|
2016-01-28 11:15:17 +01:00
|
|
|
/* And even shorter names that return void. */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_inc(ptr) \
|
|
|
|
((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_dec(ptr) \
|
|
|
|
((void) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_add(ptr, n) \
|
|
|
|
((void) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_sub(ptr, n) \
|
|
|
|
((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_and(ptr, n) \
|
|
|
|
((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_or(ptr, n) \
|
|
|
|
((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST))
|
|
|
|
#define qatomic_xor(ptr, n) \
|
|
|
|
((void) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST))
|
2016-01-28 11:15:17 +01:00
|
|
|
|
|
|
|
#else /* __ATOMIC_RELAXED */
|
2013-02-22 17:36:25 +01:00
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
#ifdef __alpha__
|
|
|
|
#define smp_read_barrier_depends() asm volatile("mb":::"memory")
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 15:45:53 +02:00
|
|
|
#endif
|
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 15:45:53 +02:00
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
/*
|
|
|
|
* Because of the strongly ordered storage model, wmb() and rmb() are nops
|
|
|
|
* here (a compiler barrier only). QEMU doesn't do accesses to write-combining
|
|
|
|
* qemu memory or non-temporal load/stores from C code.
|
|
|
|
*/
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb_release() barrier()
|
|
|
|
#define smp_mb_acquire() barrier()
|
2013-05-13 13:29:47 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* __sync_lock_test_and_set() is documented to be an acquire barrier only,
|
|
|
|
* but it is a full barrier at the hardware level. Add a compiler barrier
|
|
|
|
* to make it a full barrier also at the compiler level.
|
|
|
|
*/
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg(ptr, i) (barrier(), __sync_lock_test_and_set(ptr, i))
|
2013-05-13 13:29:47 +02:00
|
|
|
|
2011-11-01 10:39:49 +01:00
|
|
|
#elif defined(_ARCH_PPC)
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
|
|
|
|
/*
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 15:45:53 +02:00
|
|
|
* We use an eieio() for wmb() on powerpc. This assumes we don't
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
* need to order cacheable and non-cacheable stores with respect to
|
2013-05-13 13:29:47 +02:00
|
|
|
* each other.
|
|
|
|
*
|
|
|
|
* smp_mb has the same problem as on x86 for not-very-new GCC
|
|
|
|
* (http://patchwork.ozlabs.org/patch/126184/, Nov 2011).
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
*/
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_wmb() ({ asm volatile("eieio" ::: "memory"); (void)0; })
|
2012-04-23 14:46:22 +02:00
|
|
|
#if defined(__powerpc64__)
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb_release() ({ asm volatile("lwsync" ::: "memory"); (void)0; })
|
|
|
|
#define smp_mb_acquire() ({ asm volatile("lwsync" ::: "memory"); (void)0; })
|
2012-04-23 14:46:22 +02:00
|
|
|
#else
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb_release() ({ asm volatile("sync" ::: "memory"); (void)0; })
|
|
|
|
#define smp_mb_acquire() ({ asm volatile("sync" ::: "memory"); (void)0; })
|
2012-04-23 14:46:22 +02:00
|
|
|
#endif
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb() ({ asm volatile("sync" ::: "memory"); (void)0; })
|
2012-04-23 14:46:22 +02:00
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
#endif /* _ARCH_PPC */
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* For (host) platforms we don't have explicit barrier definitions
|
|
|
|
* for, we use the gcc __sync_synchronize() primitive to generate a
|
|
|
|
* full barrier. This should be safe on all platforms, though it may
|
2016-09-19 10:50:38 +02:00
|
|
|
* be overkill for smp_mb_acquire() and smp_mb_release().
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
*/
|
2013-05-13 13:29:47 +02:00
|
|
|
#ifndef smp_mb
|
2016-09-19 10:50:38 +02:00
|
|
|
#define smp_mb() __sync_synchronize()
|
2013-05-13 13:29:47 +02:00
|
|
|
#endif
|
|
|
|
|
2016-09-19 10:50:38 +02:00
|
|
|
#ifndef smp_mb_acquire
|
|
|
|
#define smp_mb_acquire() __sync_synchronize()
|
2013-05-13 13:29:47 +02:00
|
|
|
#endif
|
|
|
|
|
2016-09-19 10:50:38 +02:00
|
|
|
#ifndef smp_mb_release
|
|
|
|
#define smp_mb_release() __sync_synchronize()
|
2013-05-13 13:29:47 +02:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef smp_read_barrier_depends
|
|
|
|
#define smp_read_barrier_depends() barrier()
|
|
|
|
#endif
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 04:05:21 +02:00
|
|
|
|
2019-07-09 10:40:00 +02:00
|
|
|
#ifndef signal_barrier
|
|
|
|
#define signal_barrier() barrier()
|
|
|
|
#endif
|
|
|
|
|
2016-01-28 11:15:17 +01:00
|
|
|
/* These will only be atomic if the processor does the fetch or store
|
|
|
|
* in a single issue memory operation
|
|
|
|
*/
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_read__nocheck(p) (*(__typeof__(*(p)) volatile*) (p))
|
|
|
|
#define qatomic_set__nocheck(p, i) ((*(__typeof__(*(p)) volatile*) (p)) = (i))
|
2016-10-24 19:29:32 +02:00
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_read(ptr) qatomic_read__nocheck(ptr)
|
|
|
|
#define qatomic_set(ptr, i) qatomic_set__nocheck(ptr,i)
|
2013-05-13 13:29:47 +02:00
|
|
|
|
2013-05-13 13:29:47 +02:00
|
|
|
/**
|
2020-09-23 12:56:46 +02:00
|
|
|
* qatomic_rcu_read - reads a RCU-protected pointer to a local variable
|
2013-05-13 13:29:47 +02:00
|
|
|
* into a RCU read-side critical section. The pointer can later be safely
|
|
|
|
* dereferenced within the critical section.
|
|
|
|
*
|
|
|
|
* This ensures that the pointer copy is invariant thorough the whole critical
|
|
|
|
* section.
|
|
|
|
*
|
|
|
|
* Inserts memory barriers on architectures that require them (currently only
|
|
|
|
* Alpha) and documents which pointers are protected by RCU.
|
|
|
|
*
|
2020-09-23 12:56:46 +02:00
|
|
|
* qatomic_rcu_read also includes a compiler barrier to ensure that
|
2016-01-28 11:15:17 +01:00
|
|
|
* value-speculative optimizations (e.g. VSS: Value Speculation
|
|
|
|
* Scheduling) does not perform the data read before the pointer read
|
|
|
|
* by speculating the value of the pointer.
|
2013-05-13 13:29:47 +02:00
|
|
|
*
|
2020-09-23 12:56:46 +02:00
|
|
|
* Should match qatomic_rcu_set(), qatomic_xchg(), qatomic_cmpxchg().
|
2013-05-13 13:29:47 +02:00
|
|
|
*/
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_read(ptr) ({ \
|
|
|
|
typeof(*ptr) _val = qatomic_read(ptr); \
|
2013-05-13 13:29:47 +02:00
|
|
|
smp_read_barrier_depends(); \
|
|
|
|
_val; \
|
|
|
|
})
|
|
|
|
|
|
|
|
/**
|
2020-09-23 12:56:46 +02:00
|
|
|
* qatomic_rcu_set - assigns (publicizes) a pointer to a new data structure
|
2013-05-13 13:29:47 +02:00
|
|
|
* meant to be read by RCU read-side critical sections.
|
|
|
|
*
|
|
|
|
* Documents which pointers will be dereferenced by RCU read-side critical
|
|
|
|
* sections and adds the required memory barriers on architectures requiring
|
|
|
|
* them. It also makes sure the compiler does not reorder code initializing the
|
|
|
|
* data structure before its publication.
|
|
|
|
*
|
2020-09-23 12:56:46 +02:00
|
|
|
* Should match qatomic_rcu_read().
|
2013-05-13 13:29:47 +02:00
|
|
|
*/
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_rcu_set(ptr, i) do { \
|
2013-05-13 13:29:47 +02:00
|
|
|
smp_wmb(); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_set(ptr, i); \
|
2013-05-13 13:29:47 +02:00
|
|
|
} while (0)
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_load_acquire(ptr) ({ \
|
|
|
|
typeof(*ptr) _val = qatomic_read(ptr); \
|
2016-09-19 10:50:38 +02:00
|
|
|
smp_mb_acquire(); \
|
2013-05-13 13:29:47 +02:00
|
|
|
_val; \
|
|
|
|
})
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_store_release(ptr, i) do { \
|
2016-09-19 10:50:38 +02:00
|
|
|
smp_mb_release(); \
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_set(ptr, i); \
|
2013-05-13 13:29:47 +02:00
|
|
|
} while (0)
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#ifndef qatomic_xchg
|
2013-10-22 11:58:41 +02:00
|
|
|
#if defined(__clang__)
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg(ptr, i) __sync_swap(ptr, i)
|
2013-05-13 13:29:47 +02:00
|
|
|
#else
|
|
|
|
/* __sync_lock_test_and_set() is documented to be an acquire barrier only. */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i))
|
2013-05-13 13:29:47 +02:00
|
|
|
#endif
|
|
|
|
#endif
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_xchg__nocheck qatomic_xchg
|
2013-05-13 13:29:47 +02:00
|
|
|
|
|
|
|
/* Provide shorter names for GCC atomic builtins. */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1)
|
|
|
|
#define qatomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1)
|
|
|
|
|
|
|
|
#define qatomic_fetch_add(ptr, n) __sync_fetch_and_add(ptr, n)
|
|
|
|
#define qatomic_fetch_sub(ptr, n) __sync_fetch_and_sub(ptr, n)
|
|
|
|
#define qatomic_fetch_and(ptr, n) __sync_fetch_and_and(ptr, n)
|
|
|
|
#define qatomic_fetch_or(ptr, n) __sync_fetch_and_or(ptr, n)
|
|
|
|
#define qatomic_fetch_xor(ptr, n) __sync_fetch_and_xor(ptr, n)
|
|
|
|
|
|
|
|
#define qatomic_inc_fetch(ptr) __sync_add_and_fetch(ptr, 1)
|
|
|
|
#define qatomic_dec_fetch(ptr) __sync_add_and_fetch(ptr, -1)
|
|
|
|
#define qatomic_add_fetch(ptr, n) __sync_add_and_fetch(ptr, n)
|
|
|
|
#define qatomic_sub_fetch(ptr, n) __sync_sub_and_fetch(ptr, n)
|
|
|
|
#define qatomic_and_fetch(ptr, n) __sync_and_and_fetch(ptr, n)
|
|
|
|
#define qatomic_or_fetch(ptr, n) __sync_or_and_fetch(ptr, n)
|
|
|
|
#define qatomic_xor_fetch(ptr, n) __sync_xor_and_fetch(ptr, n)
|
|
|
|
|
|
|
|
#define qatomic_cmpxchg(ptr, old, new) \
|
|
|
|
__sync_val_compare_and_swap(ptr, old, new)
|
|
|
|
#define qatomic_cmpxchg__nocheck(ptr, old, new) qatomic_cmpxchg(ptr, old, new)
|
2013-05-13 13:29:47 +02:00
|
|
|
|
|
|
|
/* And even shorter names that return void. */
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1))
|
|
|
|
#define qatomic_dec(ptr) ((void) __sync_fetch_and_add(ptr, -1))
|
|
|
|
#define qatomic_add(ptr, n) ((void) __sync_fetch_and_add(ptr, n))
|
|
|
|
#define qatomic_sub(ptr, n) ((void) __sync_fetch_and_sub(ptr, n))
|
|
|
|
#define qatomic_and(ptr, n) ((void) __sync_fetch_and_and(ptr, n))
|
|
|
|
#define qatomic_or(ptr, n) ((void) __sync_fetch_and_or(ptr, n))
|
|
|
|
#define qatomic_xor(ptr, n) ((void) __sync_fetch_and_xor(ptr, n))
|
2013-05-13 13:29:47 +02:00
|
|
|
|
2016-01-28 11:15:17 +01:00
|
|
|
#endif /* __ATOMIC_RELAXED */
|
2016-09-19 10:50:38 +02:00
|
|
|
|
|
|
|
#ifndef smp_wmb
|
|
|
|
#define smp_wmb() smp_mb_release()
|
|
|
|
#endif
|
|
|
|
#ifndef smp_rmb
|
|
|
|
#define smp_rmb() smp_mb_acquire()
|
|
|
|
#endif
|
|
|
|
|
2016-09-19 11:36:44 +02:00
|
|
|
/* This is more efficient than a store plus a fence. */
|
|
|
|
#if !defined(__SANITIZE_THREAD__)
|
|
|
|
#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_mb_set(ptr, i) ((void)qatomic_xchg(ptr, i))
|
2016-09-19 11:36:44 +02:00
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
/* qatomic_mb_read/set semantics map Java volatile variables. They are
|
2016-09-19 11:36:44 +02:00
|
|
|
* less expensive on some platforms (notably POWER) than fully
|
|
|
|
* sequentially consistent operations.
|
|
|
|
*
|
|
|
|
* As long as they are used as paired operations they are safe to
|
2021-05-17 17:16:59 +02:00
|
|
|
* use. See docs/devel/atomics.rst for more discussion.
|
2016-09-19 11:36:44 +02:00
|
|
|
*/
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#ifndef qatomic_mb_read
|
|
|
|
#define qatomic_mb_read(ptr) \
|
|
|
|
qatomic_load_acquire(ptr)
|
2016-09-19 11:36:44 +02:00
|
|
|
#endif
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#ifndef qatomic_mb_set
|
|
|
|
#define qatomic_mb_set(ptr, i) do { \
|
|
|
|
qatomic_store_release(ptr, i); \
|
2016-09-19 11:36:44 +02:00
|
|
|
smp_mb(); \
|
|
|
|
} while(0)
|
|
|
|
#endif
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
#define qatomic_fetch_inc_nonzero(ptr) ({ \
|
|
|
|
typeof_strip_qual(*ptr) _oldn = qatomic_read(ptr); \
|
|
|
|
while (_oldn && qatomic_cmpxchg(ptr, _oldn, _oldn + 1) != _oldn) { \
|
|
|
|
_oldn = qatomic_read(ptr); \
|
2017-09-21 14:32:47 +02:00
|
|
|
} \
|
|
|
|
_oldn; \
|
|
|
|
})
|
|
|
|
|
2018-09-11 01:27:42 +02:00
|
|
|
/* Abstractions to access atomically (i.e. "once") i64/u64 variables */
|
|
|
|
#ifdef CONFIG_ATOMIC64
|
2020-09-23 12:56:46 +02:00
|
|
|
static inline int64_t qatomic_read_i64(const int64_t *ptr)
|
2018-09-11 01:27:42 +02:00
|
|
|
{
|
|
|
|
/* use __nocheck because sizeof(void *) might be < sizeof(u64) */
|
2020-09-23 12:56:46 +02:00
|
|
|
return qatomic_read__nocheck(ptr);
|
2018-09-11 01:27:42 +02:00
|
|
|
}
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
static inline uint64_t qatomic_read_u64(const uint64_t *ptr)
|
2018-09-11 01:27:42 +02:00
|
|
|
{
|
2020-09-23 12:56:46 +02:00
|
|
|
return qatomic_read__nocheck(ptr);
|
2018-09-11 01:27:42 +02:00
|
|
|
}
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
static inline void qatomic_set_i64(int64_t *ptr, int64_t val)
|
2018-09-11 01:27:42 +02:00
|
|
|
{
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_set__nocheck(ptr, val);
|
2018-09-11 01:27:42 +02:00
|
|
|
}
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
static inline void qatomic_set_u64(uint64_t *ptr, uint64_t val)
|
2018-09-11 01:27:42 +02:00
|
|
|
{
|
2020-09-23 12:56:46 +02:00
|
|
|
qatomic_set__nocheck(ptr, val);
|
2018-09-11 01:27:42 +02:00
|
|
|
}
|
|
|
|
|
2020-09-23 12:56:46 +02:00
|
|
|
static inline void qatomic64_init(void)
|
2018-09-11 01:27:42 +02:00
|
|
|
{
|
|
|
|
}
|
|
|
|
#else /* !CONFIG_ATOMIC64 */
|
2020-09-23 12:56:46 +02:00
|
|
|
int64_t qatomic_read_i64(const int64_t *ptr);
|
|
|
|
uint64_t qatomic_read_u64(const uint64_t *ptr);
|
|
|
|
void qatomic_set_i64(int64_t *ptr, int64_t val);
|
|
|
|
void qatomic_set_u64(uint64_t *ptr, uint64_t val);
|
|
|
|
void qatomic64_init(void);
|
2018-09-11 01:27:42 +02:00
|
|
|
#endif /* !CONFIG_ATOMIC64 */
|
|
|
|
|
2016-06-29 13:47:03 +02:00
|
|
|
#endif /* QEMU_ATOMIC_H */
|