159 lines
4.1 KiB
C
159 lines
4.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
|
|
*/
|
|
|
|
#ifndef __ASM_ARC_CMPXCHG_H
|
|
#define __ASM_ARC_CMPXCHG_H
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <asm/barrier.h>
|
|
#include <asm/smp.h>
|
|
|
|
#ifdef CONFIG_ARC_HAS_LLSC
|
|
|
|
static inline unsigned long
|
|
__cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new)
|
|
{
|
|
unsigned long prev;
|
|
|
|
/*
|
|
* Explicit full memory barrier needed before/after as
|
|
* LLOCK/SCOND themselves don't provide any such semantics
|
|
*/
|
|
smp_mb();
|
|
|
|
__asm__ __volatile__(
|
|
"1: llock %0, [%1] \n"
|
|
" brne %0, %2, 2f \n"
|
|
" scond %3, [%1] \n"
|
|
" bnz 1b \n"
|
|
"2: \n"
|
|
: "=&r"(prev) /* Early clobber, to prevent reg reuse */
|
|
: "r"(ptr), /* Not "m": llock only supports reg direct addr mode */
|
|
"ir"(expected),
|
|
"r"(new) /* can't be "ir". scond can't take LIMM for "b" */
|
|
: "cc", "memory"); /* so that gcc knows memory is being written here */
|
|
|
|
smp_mb();
|
|
|
|
return prev;
|
|
}
|
|
|
|
#else /* !CONFIG_ARC_HAS_LLSC */
|
|
|
|
static inline unsigned long
|
|
__cmpxchg(volatile void *ptr, unsigned long expected, unsigned long new)
|
|
{
|
|
unsigned long flags;
|
|
int prev;
|
|
volatile unsigned long *p = ptr;
|
|
|
|
/*
|
|
* spin lock/unlock provide the needed smp_mb() before/after
|
|
*/
|
|
atomic_ops_lock(flags);
|
|
prev = *p;
|
|
if (prev == expected)
|
|
*p = new;
|
|
atomic_ops_unlock(flags);
|
|
return prev;
|
|
}
|
|
|
|
#endif
|
|
|
|
#define arch_cmpxchg(ptr, o, n) ({ \
|
|
(typeof(*(ptr)))__cmpxchg((ptr), \
|
|
(unsigned long)(o), \
|
|
(unsigned long)(n)); \
|
|
})
|
|
|
|
/*
|
|
* atomic_cmpxchg is same as cmpxchg
|
|
* LLSC: only different in data-type, semantics are exactly same
|
|
* !LLSC: cmpxchg() has to use an external lock atomic_ops_lock to guarantee
|
|
* semantics, and this lock also happens to be used by atomic_*()
|
|
*/
|
|
#define arch_atomic_cmpxchg(v, o, n) ((int)arch_cmpxchg(&((v)->counter), (o), (n)))
|
|
|
|
|
|
/*
|
|
* xchg (reg with memory) based on "Native atomic" EX insn
|
|
*/
|
|
static inline unsigned long __xchg(unsigned long val, volatile void *ptr,
|
|
int size)
|
|
{
|
|
extern unsigned long __xchg_bad_pointer(void);
|
|
|
|
switch (size) {
|
|
case 4:
|
|
smp_mb();
|
|
|
|
__asm__ __volatile__(
|
|
" ex %0, [%1] \n"
|
|
: "+r"(val)
|
|
: "r"(ptr)
|
|
: "memory");
|
|
|
|
smp_mb();
|
|
|
|
return val;
|
|
}
|
|
return __xchg_bad_pointer();
|
|
}
|
|
|
|
#define _xchg(ptr, with) ((typeof(*(ptr)))__xchg((unsigned long)(with), (ptr), \
|
|
sizeof(*(ptr))))
|
|
|
|
/*
|
|
* xchg() maps directly to ARC EX instruction which guarantees atomicity.
|
|
* However in !LLSC config, it also needs to be use @atomic_ops_lock spinlock
|
|
* due to a subtle reason:
|
|
* - For !LLSC, cmpxchg() needs to use that lock (see above) and there is lot
|
|
* of kernel code which calls xchg()/cmpxchg() on same data (see llist.h)
|
|
* Hence xchg() needs to follow same locking rules.
|
|
*
|
|
* Technically the lock is also needed for UP (boils down to irq save/restore)
|
|
* but we can cheat a bit since cmpxchg() atomic_ops_lock() would cause irqs to
|
|
* be disabled thus can't possibly be interrupted/preempted/clobbered by xchg()
|
|
* Other way around, xchg is one instruction anyways, so can't be interrupted
|
|
* as such
|
|
*/
|
|
|
|
#if !defined(CONFIG_ARC_HAS_LLSC) && defined(CONFIG_SMP)
|
|
|
|
#define arch_xchg(ptr, with) \
|
|
({ \
|
|
unsigned long flags; \
|
|
typeof(*(ptr)) old_val; \
|
|
\
|
|
atomic_ops_lock(flags); \
|
|
old_val = _xchg(ptr, with); \
|
|
atomic_ops_unlock(flags); \
|
|
old_val; \
|
|
})
|
|
|
|
#else
|
|
|
|
#define arch_xchg(ptr, with) _xchg(ptr, with)
|
|
|
|
#endif
|
|
|
|
/*
|
|
* "atomic" variant of xchg()
|
|
* REQ: It needs to follow the same serialization rules as other atomic_xxx()
|
|
* Since xchg() doesn't always do that, it would seem that following definition
|
|
* is incorrect. But here's the rationale:
|
|
* SMP : Even xchg() takes the atomic_ops_lock, so OK.
|
|
* LLSC: atomic_ops_lock are not relevant at all (even if SMP, since LLSC
|
|
* is natively "SMP safe", no serialization required).
|
|
* UP : other atomics disable IRQ, so no way a difft ctxt atomic_xchg()
|
|
* could clobber them. atomic_xchg() itself would be 1 insn, so it
|
|
* can't be clobbered by others. Thus no serialization required when
|
|
* atomic_xchg is involved.
|
|
*/
|
|
#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new))
|
|
|
|
#endif
|