2005-04-16 18:20:36 -04:00
|
|
|
/*
|
|
|
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
|
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
|
|
|
* for more details.
|
|
|
|
|
*
|
2007-02-16 12:18:50 -05:00
|
|
|
* Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
|
2005-04-16 18:20:36 -04:00
|
|
|
* Copyright (c) 1999, 2000 Silicon Graphics, Inc.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef _ASM_BITOPS_H
|
|
|
|
|
#define _ASM_BITOPS_H
|
|
|
|
|
|
2007-10-19 02:40:26 -04:00
|
|
|
#ifndef _LINUX_BITOPS_H
|
|
|
|
|
#error only <linux/bitops.h> can be included directly
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
#include <linux/compiler.h>
|
|
|
|
|
#include <linux/types.h>
|
2006-10-30 22:45:07 -05:00
|
|
|
#include <asm/barrier.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
#include <asm/byteorder.h> /* sigh ... */
|
2014-11-15 17:08:48 -05:00
|
|
|
#include <asm/compiler.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
#include <asm/cpu-features.h>
|
2016-04-15 04:25:33 -04:00
|
|
|
#include <asm/llsc.h>
|
2006-11-29 20:14:50 -05:00
|
|
|
#include <asm/sgidefs.h>
|
|
|
|
|
#include <asm/war.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2012-09-06 11:36:55 -04:00
|
|
|
/*
|
|
|
|
|
* These are the "slower" versions of the functions and are in bitops.c.
|
|
|
|
|
* These functions call raw_local_irq_{save,restore}().
|
|
|
|
|
*/
|
|
|
|
|
void __mips_set_bit(unsigned long nr, volatile unsigned long *addr);
|
|
|
|
|
void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr);
|
|
|
|
|
void __mips_change_bit(unsigned long nr, volatile unsigned long *addr);
|
|
|
|
|
int __mips_test_and_set_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr);
|
|
|
|
|
int __mips_test_and_set_bit_lock(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr);
|
|
|
|
|
int __mips_test_and_clear_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr);
|
|
|
|
|
int __mips_test_and_change_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr);
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
/*
|
|
|
|
|
* set_bit - Atomically set a bit in memory
|
|
|
|
|
* @nr: the bit to set
|
|
|
|
|
* @addr: the address to start counting from
|
|
|
|
|
*
|
|
|
|
|
* This function is atomic and may not be reordered. See __set_bit()
|
|
|
|
|
* if you do not require the atomic guarantees.
|
|
|
|
|
* Note that @nr may be almost arbitrarily large; this function is not
|
|
|
|
|
* restricted to acting on a single-word quantity.
|
|
|
|
|
*/
|
|
|
|
|
static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
|
|
|
|
|
{
|
|
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long temp;
|
|
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
"1: " __LL "%0, %1 # set_bit \n"
|
|
|
|
|
" or %0, %2 \n"
|
2005-06-14 13:35:03 -04:00
|
|
|
" " __SC "%0, %1 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
" beqzl %0, 1b \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2014-11-20 08:58:30 -05:00
|
|
|
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
|
MIPS: Loongson: Introduce and use loongson_llsc_mb()
On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions "before
each ll/lld" and "at the branch-target between ll/sc" to workaround.
Otherwise, this flaw will cause deadlock occasionally (e.g. when doing
heavy load test with LTP).
Below is the explaination of CPU designer:
"For Loongson 3 family, when a memory access instruction (load, store,
or prefetch)'s executing occurs between the execution of LL and SC, the
success or failure of SC is not predictable. Although programmer would
not insert memory access instructions between LL and SC, the memory
instructions before LL in program-order, may dynamically executed
between the execution of LL/SC, so a memory fence (SYNC) is needed
before LL/LLD to avoid this situation.
Since Loongson-3A R2 (3A2000), we have improved our hardware design to
handle this case. But we later deduce a rarely circumstance that some
speculatively executed memory instructions due to branch misprediction
between LL/SC still fall into the above case, so a memory fence (SYNC)
at branch-target (if its target is not between LL/SC) is needed for
Loongson 3A1000, 3B1500, 3A2000 and 3A3000.
Our processor is continually evolving and we aim to to remove all these
workaround-SYNCs around LL/SC for new-come processor."
Here is an example:
Both cpu1 and cpu2 simutaneously run atomic_add by 1 on same atomic var,
this bug cause both 'sc' run by two cpus (in atomic_add) succeed at same
time('sc' return 1), and the variable is only *added by 1*, sometimes,
which is wrong and unacceptable(it should be added by 2).
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's __ex_table section.
This patch fix all the cases in kernel, but:
+. the fix at the end of futex_atomic_cmpxchg_inatomic is for branch-target
of 'bne', there other cases which smp_mb__before_llsc() and smp_llsc_mb() fix
the ll and branch-target coincidently such as atomic_sub_if_positive/
cmpxchg/xchg, just like this one.
+. Loongson 3 does support CONFIG_EDAC_ATOMIC_SCRUB, so no need to touch
edac.h
+. local_ops and cmpxchg_local should not be affected by this bug since
only the owner can write.
+. mips_atomic_set for syscall.c is deprecated and rarely used, just let
it go
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Huang Pei <huangpei@loongson.cn>
[paul.burton@mips.com:
- Simplify the addition of -mno-fix-loongson3-llsc to cflags, and add
a comment describing why it's there.
- Make loongson_llsc_mb() a no-op when
CONFIG_CPU_LOONGSON3_WORKAROUNDS=n, rather than a compiler memory
barrier.
- Add a comment describing the bug & how loongson_llsc_mb() helps
in asm/barrier.h.]
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: ambrosehua@gmail.com
Cc: Steven J . Hill <Steven.Hill@cavium.com>
Cc: linux-mips@linux-mips.org
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: Li Xuefeng <lixuefeng@loongson.cn>
Cc: Xu Chenghua <xuchenghua@loongson.cn>
2019-01-15 03:04:54 -05:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
" " __LL "%0, %1 # set_bit \n"
|
|
|
|
|
" " __INS "%0, %3, %2, 1 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (bit), "r" (~0)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2014-11-20 08:58:30 -05:00
|
|
|
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
MIPS: Loongson: Introduce and use loongson_llsc_mb()
On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions "before
each ll/lld" and "at the branch-target between ll/sc" to workaround.
Otherwise, this flaw will cause deadlock occasionally (e.g. when doing
heavy load test with LTP).
Below is the explaination of CPU designer:
"For Loongson 3 family, when a memory access instruction (load, store,
or prefetch)'s executing occurs between the execution of LL and SC, the
success or failure of SC is not predictable. Although programmer would
not insert memory access instructions between LL and SC, the memory
instructions before LL in program-order, may dynamically executed
between the execution of LL/SC, so a memory fence (SYNC) is needed
before LL/LLD to avoid this situation.
Since Loongson-3A R2 (3A2000), we have improved our hardware design to
handle this case. But we later deduce a rarely circumstance that some
speculatively executed memory instructions due to branch misprediction
between LL/SC still fall into the above case, so a memory fence (SYNC)
at branch-target (if its target is not between LL/SC) is needed for
Loongson 3A1000, 3B1500, 3A2000 and 3A3000.
Our processor is continually evolving and we aim to to remove all these
workaround-SYNCs around LL/SC for new-come processor."
Here is an example:
Both cpu1 and cpu2 simutaneously run atomic_add by 1 on same atomic var,
this bug cause both 'sc' run by two cpus (in atomic_add) succeed at same
time('sc' return 1), and the variable is only *added by 1*, sometimes,
which is wrong and unacceptable(it should be added by 2).
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's __ex_table section.
This patch fix all the cases in kernel, but:
+. the fix at the end of futex_atomic_cmpxchg_inatomic is for branch-target
of 'bne', there other cases which smp_mb__before_llsc() and smp_llsc_mb() fix
the ll and branch-target coincidently such as atomic_sub_if_positive/
cmpxchg/xchg, just like this one.
+. Loongson 3 does support CONFIG_EDAC_ATOMIC_SCRUB, so no need to touch
edac.h
+. local_ops and cmpxchg_local should not be affected by this bug since
only the owner can write.
+. mips_atomic_set for syscall.c is deprecated and rarely used, just let
it go
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Huang Pei <huangpei@loongson.cn>
[paul.burton@mips.com:
- Simplify the addition of -mno-fix-loongson3-llsc to cflags, and add
a comment describing why it's there.
- Make loongson_llsc_mb() a no-op when
CONFIG_CPU_LOONGSON3_WORKAROUNDS=n, rather than a compiler memory
barrier.
- Add a comment describing the bug & how loongson_llsc_mb() helps
in asm/barrier.h.]
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: ambrosehua@gmail.com
Cc: Steven J . Hill <Steven.Hill@cavium.com>
Cc: linux-mips@linux-mips.org
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: Li Xuefeng <lixuefeng@loongson.cn>
Cc: Xu Chenghua <xuchenghua@loongson.cn>
2019-01-15 03:04:54 -05:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __LL "%0, %1 # set_bit \n"
|
|
|
|
|
" or %0, %2 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (1UL << bit)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
__mips_set_bit(nr, addr);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* clear_bit - Clears a bit in memory
|
|
|
|
|
* @nr: Bit to clear
|
|
|
|
|
* @addr: Address to start counting from
|
|
|
|
|
*
|
|
|
|
|
* clear_bit() is atomic and may not be reordered. However, it does
|
|
|
|
|
* not contain a memory barrier, so if it is used for locking purposes,
|
2014-03-13 14:00:36 -04:00
|
|
|
* you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
|
2005-04-16 18:20:36 -04:00
|
|
|
* in order to ensure changes are visible on other processors.
|
|
|
|
|
*/
|
|
|
|
|
static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
|
|
|
|
|
{
|
|
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long temp;
|
|
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
"1: " __LL "%0, %1 # clear_bit \n"
|
|
|
|
|
" and %0, %2 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
|
|
|
|
" beqzl %0, 1b \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (~(1UL << bit))
|
|
|
|
|
: __LLSC_CLOBBER);
|
2014-11-20 08:58:30 -05:00
|
|
|
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
|
MIPS: Loongson: Introduce and use loongson_llsc_mb()
On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions "before
each ll/lld" and "at the branch-target between ll/sc" to workaround.
Otherwise, this flaw will cause deadlock occasionally (e.g. when doing
heavy load test with LTP).
Below is the explaination of CPU designer:
"For Loongson 3 family, when a memory access instruction (load, store,
or prefetch)'s executing occurs between the execution of LL and SC, the
success or failure of SC is not predictable. Although programmer would
not insert memory access instructions between LL and SC, the memory
instructions before LL in program-order, may dynamically executed
between the execution of LL/SC, so a memory fence (SYNC) is needed
before LL/LLD to avoid this situation.
Since Loongson-3A R2 (3A2000), we have improved our hardware design to
handle this case. But we later deduce a rarely circumstance that some
speculatively executed memory instructions due to branch misprediction
between LL/SC still fall into the above case, so a memory fence (SYNC)
at branch-target (if its target is not between LL/SC) is needed for
Loongson 3A1000, 3B1500, 3A2000 and 3A3000.
Our processor is continually evolving and we aim to to remove all these
workaround-SYNCs around LL/SC for new-come processor."
Here is an example:
Both cpu1 and cpu2 simutaneously run atomic_add by 1 on same atomic var,
this bug cause both 'sc' run by two cpus (in atomic_add) succeed at same
time('sc' return 1), and the variable is only *added by 1*, sometimes,
which is wrong and unacceptable(it should be added by 2).
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's __ex_table section.
This patch fix all the cases in kernel, but:
+. the fix at the end of futex_atomic_cmpxchg_inatomic is for branch-target
of 'bne', there other cases which smp_mb__before_llsc() and smp_llsc_mb() fix
the ll and branch-target coincidently such as atomic_sub_if_positive/
cmpxchg/xchg, just like this one.
+. Loongson 3 does support CONFIG_EDAC_ATOMIC_SCRUB, so no need to touch
edac.h
+. local_ops and cmpxchg_local should not be affected by this bug since
only the owner can write.
+. mips_atomic_set for syscall.c is deprecated and rarely used, just let
it go
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Huang Pei <huangpei@loongson.cn>
[paul.burton@mips.com:
- Simplify the addition of -mno-fix-loongson3-llsc to cflags, and add
a comment describing why it's there.
- Make loongson_llsc_mb() a no-op when
CONFIG_CPU_LOONGSON3_WORKAROUNDS=n, rather than a compiler memory
barrier.
- Add a comment describing the bug & how loongson_llsc_mb() helps
in asm/barrier.h.]
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: ambrosehua@gmail.com
Cc: Steven J . Hill <Steven.Hill@cavium.com>
Cc: linux-mips@linux-mips.org
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: Li Xuefeng <lixuefeng@loongson.cn>
Cc: Xu Chenghua <xuchenghua@loongson.cn>
2019-01-15 03:04:54 -05:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
|
|
|
|
" " __LL "%0, %1 # clear_bit \n"
|
|
|
|
|
" " __INS "%0, $0, %2, 1 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (bit)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2014-11-20 08:58:30 -05:00
|
|
|
#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
MIPS: Loongson: Introduce and use loongson_llsc_mb()
On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions "before
each ll/lld" and "at the branch-target between ll/sc" to workaround.
Otherwise, this flaw will cause deadlock occasionally (e.g. when doing
heavy load test with LTP).
Below is the explaination of CPU designer:
"For Loongson 3 family, when a memory access instruction (load, store,
or prefetch)'s executing occurs between the execution of LL and SC, the
success or failure of SC is not predictable. Although programmer would
not insert memory access instructions between LL and SC, the memory
instructions before LL in program-order, may dynamically executed
between the execution of LL/SC, so a memory fence (SYNC) is needed
before LL/LLD to avoid this situation.
Since Loongson-3A R2 (3A2000), we have improved our hardware design to
handle this case. But we later deduce a rarely circumstance that some
speculatively executed memory instructions due to branch misprediction
between LL/SC still fall into the above case, so a memory fence (SYNC)
at branch-target (if its target is not between LL/SC) is needed for
Loongson 3A1000, 3B1500, 3A2000 and 3A3000.
Our processor is continually evolving and we aim to to remove all these
workaround-SYNCs around LL/SC for new-come processor."
Here is an example:
Both cpu1 and cpu2 simutaneously run atomic_add by 1 on same atomic var,
this bug cause both 'sc' run by two cpus (in atomic_add) succeed at same
time('sc' return 1), and the variable is only *added by 1*, sometimes,
which is wrong and unacceptable(it should be added by 2).
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's __ex_table section.
This patch fix all the cases in kernel, but:
+. the fix at the end of futex_atomic_cmpxchg_inatomic is for branch-target
of 'bne', there other cases which smp_mb__before_llsc() and smp_llsc_mb() fix
the ll and branch-target coincidently such as atomic_sub_if_positive/
cmpxchg/xchg, just like this one.
+. Loongson 3 does support CONFIG_EDAC_ATOMIC_SCRUB, so no need to touch
edac.h
+. local_ops and cmpxchg_local should not be affected by this bug since
only the owner can write.
+. mips_atomic_set for syscall.c is deprecated and rarely used, just let
it go
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Huang Pei <huangpei@loongson.cn>
[paul.burton@mips.com:
- Simplify the addition of -mno-fix-loongson3-llsc to cflags, and add
a comment describing why it's there.
- Make loongson_llsc_mb() a no-op when
CONFIG_CPU_LOONGSON3_WORKAROUNDS=n, rather than a compiler memory
barrier.
- Add a comment describing the bug & how loongson_llsc_mb() helps
in asm/barrier.h.]
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: ambrosehua@gmail.com
Cc: Steven J . Hill <Steven.Hill@cavium.com>
Cc: linux-mips@linux-mips.org
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: Li Xuefeng <lixuefeng@loongson.cn>
Cc: Xu Chenghua <xuchenghua@loongson.cn>
2019-01-15 03:04:54 -05:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __LL "%0, %1 # clear_bit \n"
|
|
|
|
|
" and %0, %2 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (~(1UL << bit))
|
|
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
__mips_clear_bit(nr, addr);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
2007-10-18 06:06:53 -04:00
|
|
|
/*
|
|
|
|
|
* clear_bit_unlock - Clears a bit in memory
|
|
|
|
|
* @nr: Bit to clear
|
|
|
|
|
* @addr: Address to start counting from
|
|
|
|
|
*
|
|
|
|
|
* clear_bit() is atomic and implies release semantics before the memory
|
|
|
|
|
* operation. It can be used for an unlock.
|
|
|
|
|
*/
|
|
|
|
|
static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
|
|
|
|
|
{
|
2014-03-13 14:00:36 -04:00
|
|
|
smp_mb__before_atomic();
|
2007-10-18 06:06:53 -04:00
|
|
|
clear_bit(nr, addr);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
/*
|
|
|
|
|
* change_bit - Toggle a bit in memory
|
|
|
|
|
* @nr: Bit to change
|
|
|
|
|
* @addr: Address to start counting from
|
|
|
|
|
*
|
|
|
|
|
* change_bit() is atomic and may not be reordered.
|
|
|
|
|
* Note that @nr may be almost arbitrarily large; this function is not
|
|
|
|
|
* restricted to acting on a single-word quantity.
|
|
|
|
|
*/
|
|
|
|
|
static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
|
|
|
|
|
{
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2007-03-04 19:56:15 -05:00
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
|
|
|
|
unsigned long temp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
"1: " __LL "%0, %1 # change_bit \n"
|
|
|
|
|
" xor %0, %2 \n"
|
2005-06-14 13:35:03 -04:00
|
|
|
" " __SC "%0, %1 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
" beqzl %0, 1b \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (1UL << bit)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
|
|
|
|
unsigned long temp;
|
|
|
|
|
|
MIPS: Loongson: Introduce and use loongson_llsc_mb()
On the Loongson-2G/2H/3A/3B there is a hardware flaw that ll/sc and
lld/scd is very weak ordering. We should add sync instructions "before
each ll/lld" and "at the branch-target between ll/sc" to workaround.
Otherwise, this flaw will cause deadlock occasionally (e.g. when doing
heavy load test with LTP).
Below is the explaination of CPU designer:
"For Loongson 3 family, when a memory access instruction (load, store,
or prefetch)'s executing occurs between the execution of LL and SC, the
success or failure of SC is not predictable. Although programmer would
not insert memory access instructions between LL and SC, the memory
instructions before LL in program-order, may dynamically executed
between the execution of LL/SC, so a memory fence (SYNC) is needed
before LL/LLD to avoid this situation.
Since Loongson-3A R2 (3A2000), we have improved our hardware design to
handle this case. But we later deduce a rarely circumstance that some
speculatively executed memory instructions due to branch misprediction
between LL/SC still fall into the above case, so a memory fence (SYNC)
at branch-target (if its target is not between LL/SC) is needed for
Loongson 3A1000, 3B1500, 3A2000 and 3A3000.
Our processor is continually evolving and we aim to to remove all these
workaround-SYNCs around LL/SC for new-come processor."
Here is an example:
Both cpu1 and cpu2 simutaneously run atomic_add by 1 on same atomic var,
this bug cause both 'sc' run by two cpus (in atomic_add) succeed at same
time('sc' return 1), and the variable is only *added by 1*, sometimes,
which is wrong and unacceptable(it should be added by 2).
Why disable fix-loongson3-llsc in compiler?
Because compiler fix will cause problems in kernel's __ex_table section.
This patch fix all the cases in kernel, but:
+. the fix at the end of futex_atomic_cmpxchg_inatomic is for branch-target
of 'bne', there other cases which smp_mb__before_llsc() and smp_llsc_mb() fix
the ll and branch-target coincidently such as atomic_sub_if_positive/
cmpxchg/xchg, just like this one.
+. Loongson 3 does support CONFIG_EDAC_ATOMIC_SCRUB, so no need to touch
edac.h
+. local_ops and cmpxchg_local should not be affected by this bug since
only the owner can write.
+. mips_atomic_set for syscall.c is deprecated and rarely used, just let
it go
Signed-off-by: Huacai Chen <chenhc@lemote.com>
Signed-off-by: Huang Pei <huangpei@loongson.cn>
[paul.burton@mips.com:
- Simplify the addition of -mno-fix-loongson3-llsc to cflags, and add
a comment describing why it's there.
- Make loongson_llsc_mb() a no-op when
CONFIG_CPU_LOONGSON3_WORKAROUNDS=n, rather than a compiler memory
barrier.
- Add a comment describing the bug & how loongson_llsc_mb() helps
in asm/barrier.h.]
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: ambrosehua@gmail.com
Cc: Steven J . Hill <Steven.Hill@cavium.com>
Cc: linux-mips@linux-mips.org
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: Li Xuefeng <lixuefeng@loongson.cn>
Cc: Xu Chenghua <xuchenghua@loongson.cn>
2019-01-15 03:04:54 -05:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __LL "%0, %1 # change_bit \n"
|
|
|
|
|
" xor %0, %2 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m)
|
2019-06-13 09:43:20 -04:00
|
|
|
: "ir" (1UL << bit)
|
|
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
__mips_change_bit(nr, addr);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* test_and_set_bit - Set a bit and return its old value
|
|
|
|
|
* @nr: Bit to set
|
|
|
|
|
* @addr: Address to count from
|
|
|
|
|
*
|
|
|
|
|
* This operation is atomic and cannot be reordered.
|
|
|
|
|
* It also implies a memory barrier.
|
|
|
|
|
*/
|
|
|
|
|
static inline int test_and_set_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr)
|
|
|
|
|
{
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long res;
|
2007-03-04 19:56:15 -05:00
|
|
|
|
2010-01-08 20:17:43 -05:00
|
|
|
smp_mb__before_llsc();
|
2007-10-18 06:06:52 -04:00
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
"1: " __LL "%0, %1 # test_and_set_bit \n"
|
|
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" " __SC "%2, %1 \n"
|
|
|
|
|
" beqzl %2, 1b \n"
|
|
|
|
|
" and %2, %0, %3 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2019-06-13 09:43:19 -04:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __LL "%0, %1 # test_and_set_bit \n"
|
|
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" " __SC "%2, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!res));
|
|
|
|
|
|
|
|
|
|
res = temp & (1UL << bit);
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
res = __mips_test_and_set_bit(nr, addr);
|
2006-10-30 22:45:07 -05:00
|
|
|
|
2007-07-14 08:24:05 -04:00
|
|
|
smp_llsc_mb();
|
2007-06-07 08:17:30 -04:00
|
|
|
|
|
|
|
|
return res != 0;
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
2007-10-18 06:06:53 -04:00
|
|
|
/*
|
|
|
|
|
* test_and_set_bit_lock - Set a bit and return its old value
|
|
|
|
|
* @nr: Bit to set
|
|
|
|
|
* @addr: Address to count from
|
|
|
|
|
*
|
|
|
|
|
* This operation is atomic and implies acquire ordering semantics
|
|
|
|
|
* after the memory operation.
|
|
|
|
|
*/
|
|
|
|
|
static inline int test_and_set_bit_lock(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr)
|
|
|
|
|
{
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2007-10-18 06:06:53 -04:00
|
|
|
unsigned long res;
|
|
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2007-10-18 06:06:53 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
|
|
|
|
unsigned long temp;
|
|
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2007-10-18 06:06:53 -04:00
|
|
|
"1: " __LL "%0, %1 # test_and_set_bit \n"
|
|
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" " __SC "%2, %1 \n"
|
|
|
|
|
" beqzl %2, 1b \n"
|
|
|
|
|
" and %2, %0, %3 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
: "=&r" (temp), "+m" (*m), "=&r" (res)
|
|
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
2007-10-18 06:06:53 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
|
|
|
|
unsigned long temp;
|
|
|
|
|
|
2019-06-13 09:43:19 -04:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __LL "%0, %1 # test_and_set_bit \n"
|
|
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" " __SC "%2, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!res));
|
|
|
|
|
|
|
|
|
|
res = temp & (1UL << bit);
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
res = __mips_test_and_set_bit_lock(nr, addr);
|
2007-10-18 06:06:53 -04:00
|
|
|
|
|
|
|
|
smp_llsc_mb();
|
|
|
|
|
|
|
|
|
|
return res != 0;
|
|
|
|
|
}
|
2005-04-16 18:20:36 -04:00
|
|
|
/*
|
|
|
|
|
* test_and_clear_bit - Clear a bit and return its old value
|
|
|
|
|
* @nr: Bit to clear
|
|
|
|
|
* @addr: Address to count from
|
|
|
|
|
*
|
|
|
|
|
* This operation is atomic and cannot be reordered.
|
|
|
|
|
* It also implies a memory barrier.
|
|
|
|
|
*/
|
|
|
|
|
static inline int test_and_clear_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr)
|
|
|
|
|
{
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long res;
|
2007-03-04 19:56:15 -05:00
|
|
|
|
2010-01-08 20:17:43 -05:00
|
|
|
smp_mb__before_llsc();
|
2007-10-18 06:06:52 -04:00
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-13 11:56:31 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
"1: " __LL "%0, %1 # test_and_clear_bit \n"
|
|
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" xor %2, %3 \n"
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __SC "%2, %1 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
" beqzl %2, 1b \n"
|
|
|
|
|
" and %2, %0, %3 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2014-11-20 08:58:30 -05:00
|
|
|
#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
|
2007-02-16 12:18:50 -05:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2007-02-16 12:18:50 -05:00
|
|
|
|
2019-06-13 09:43:19 -04:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __LL "%0, %1 # test_and_clear_bit \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" " __EXT "%2, %0, %3, 1 \n"
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __INS "%0, $0, %3, 1 \n"
|
|
|
|
|
" " __SC "%0, %1 \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "ir" (bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!temp));
|
2007-02-16 12:18:50 -05:00
|
|
|
#endif
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2019-06-13 09:43:19 -04:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __LL "%0, %1 # test_and_clear_bit \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" or %2, %0, %3 \n"
|
|
|
|
|
" xor %2, %3 \n"
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __SC "%2, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!res));
|
|
|
|
|
|
|
|
|
|
res = temp & (1UL << bit);
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
res = __mips_test_and_clear_bit(nr, addr);
|
2006-10-30 22:45:07 -05:00
|
|
|
|
2007-07-14 08:24:05 -04:00
|
|
|
smp_llsc_mb();
|
2007-06-07 08:17:30 -04:00
|
|
|
|
|
|
|
|
return res != 0;
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* test_and_change_bit - Change a bit and return its old value
|
|
|
|
|
* @nr: Bit to change
|
|
|
|
|
* @addr: Address to count from
|
|
|
|
|
*
|
|
|
|
|
* This operation is atomic and cannot be reordered.
|
|
|
|
|
* It also implies a memory barrier.
|
|
|
|
|
*/
|
|
|
|
|
static inline int test_and_change_bit(unsigned long nr,
|
|
|
|
|
volatile unsigned long *addr)
|
|
|
|
|
{
|
2012-09-06 11:36:54 -04:00
|
|
|
int bit = nr & SZLONG_MASK;
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long res;
|
2007-03-04 19:56:15 -05:00
|
|
|
|
2010-01-08 20:17:43 -05:00
|
|
|
smp_mb__before_llsc();
|
2007-10-18 06:06:52 -04:00
|
|
|
|
2009-07-13 14:15:19 -04:00
|
|
|
if (kernel_uses_llsc && R10000_LLSC_WAR) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-03-30 07:20:10 -04:00
|
|
|
" .set arch=r4000 \n"
|
2005-06-14 13:35:03 -04:00
|
|
|
"1: " __LL "%0, %1 # test_and_change_bit \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
" xor %2, %0, %3 \n"
|
2005-06-14 13:35:03 -04:00
|
|
|
" " __SC "%2, %1 \n"
|
2005-04-16 18:20:36 -04:00
|
|
|
" beqzl %2, 1b \n"
|
|
|
|
|
" and %2, %0, %3 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2009-07-13 14:15:19 -04:00
|
|
|
} else if (kernel_uses_llsc) {
|
2005-04-16 18:20:36 -04:00
|
|
|
unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
|
2007-06-07 08:17:30 -04:00
|
|
|
unsigned long temp;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2019-06-13 09:43:19 -04:00
|
|
|
loongson_llsc_mb();
|
2010-10-29 14:08:24 -04:00
|
|
|
do {
|
|
|
|
|
__asm__ __volatile__(
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_ARCH_LEVEL" \n"
|
2013-01-22 06:59:30 -05:00
|
|
|
" " __LL "%0, %1 # test_and_change_bit \n"
|
2010-10-29 14:08:24 -04:00
|
|
|
" xor %2, %0, %3 \n"
|
|
|
|
|
" " __SC "\t%2, %1 \n"
|
2018-11-08 15:14:38 -05:00
|
|
|
" .set pop \n"
|
2015-01-26 07:44:11 -05:00
|
|
|
: "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res)
|
2010-10-29 14:08:24 -04:00
|
|
|
: "r" (1UL << bit)
|
2019-06-13 09:43:20 -04:00
|
|
|
: __LLSC_CLOBBER);
|
2010-10-29 14:08:24 -04:00
|
|
|
} while (unlikely(!res));
|
|
|
|
|
|
|
|
|
|
res = temp & (1UL << bit);
|
2012-09-06 11:36:55 -04:00
|
|
|
} else
|
|
|
|
|
res = __mips_test_and_change_bit(nr, addr);
|
2006-10-30 22:45:07 -05:00
|
|
|
|
2007-07-14 08:24:05 -04:00
|
|
|
smp_llsc_mb();
|
2007-06-07 08:17:30 -04:00
|
|
|
|
|
|
|
|
return res != 0;
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
[PATCH] bitops: mips: use generic bitops
- remove __{,test_and_}{set,clear,change}_bit() and test_bit()
- unless defined(CONFIG_CPU_MIPS32) or defined(CONFIG_CPU_MIPS64)
- remove __ffs()
- remove ffs()
- remove ffz()
- remove fls()
- remove fls64()
- remove find_{next,first}{,_zero}_bit()
- remove sched_find_first_bit()
- remove generic_hweight64()
- remove generic_hweight{32,16,8}()
- remove ext2_{set,clear,test,find_first_zero,find_next_zero}_bit()
- remove ext2_{set,clear}_bit_atomic()
- remove minix_{test,set,test_and_clear,test,find_first_zero}_bit()
Signed-off-by: Akinobu Mita <mita@miraclelinux.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-26 04:39:30 -05:00
|
|
|
#include <asm-generic/bitops/non-atomic.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2007-10-18 06:06:53 -04:00
|
|
|
/*
|
|
|
|
|
* __clear_bit_unlock - Clears a bit in memory
|
|
|
|
|
* @nr: Bit to clear
|
|
|
|
|
* @addr: Address to start counting from
|
|
|
|
|
*
|
|
|
|
|
* __clear_bit() is non-atomic and implies release semantics before the memory
|
|
|
|
|
* operation. It can be used for an unlock if no other CPUs can concurrently
|
|
|
|
|
* modify other bits in the word.
|
|
|
|
|
*/
|
|
|
|
|
static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
|
|
|
|
|
{
|
2015-06-01 20:09:52 -04:00
|
|
|
smp_mb__before_llsc();
|
2007-10-18 06:06:53 -04:00
|
|
|
__clear_bit(nr, addr);
|
2017-09-15 13:31:28 -04:00
|
|
|
nudge_writes();
|
2007-10-18 06:06:53 -04:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
/*
|
2005-10-07 11:58:15 -04:00
|
|
|
* Return the bit position (0..63) of the most significant 1 bit in a word
|
2005-07-12 08:50:30 -04:00
|
|
|
* Returns -1 if no 1 bit exists
|
|
|
|
|
*/
|
2019-05-14 18:42:07 -04:00
|
|
|
static __always_inline unsigned long __fls(unsigned long word)
|
2005-07-12 08:50:30 -04:00
|
|
|
{
|
2008-10-28 05:40:35 -04:00
|
|
|
int num;
|
2005-07-12 08:50:30 -04:00
|
|
|
|
2015-04-03 18:25:00 -04:00
|
|
|
if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) &&
|
2009-04-18 21:21:22 -04:00
|
|
|
__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
|
2007-10-11 18:46:15 -04:00
|
|
|
__asm__(
|
2005-10-07 11:58:15 -04:00
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_LEVEL" \n"
|
2005-10-07 11:58:15 -04:00
|
|
|
" clz %0, %1 \n"
|
|
|
|
|
" .set pop \n"
|
2008-10-28 05:40:35 -04:00
|
|
|
: "=r" (num)
|
|
|
|
|
: "r" (word));
|
2005-07-12 08:50:30 -04:00
|
|
|
|
2008-10-28 05:40:35 -04:00
|
|
|
return 31 - num;
|
2005-10-07 11:58:15 -04:00
|
|
|
}
|
|
|
|
|
|
2015-04-03 18:25:00 -04:00
|
|
|
if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) &&
|
2008-10-28 05:40:35 -04:00
|
|
|
__builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
|
|
|
|
|
__asm__(
|
|
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_LEVEL" \n"
|
2008-10-28 05:40:35 -04:00
|
|
|
" dclz %0, %1 \n"
|
|
|
|
|
" .set pop \n"
|
|
|
|
|
: "=r" (num)
|
|
|
|
|
: "r" (word));
|
2005-07-12 08:50:30 -04:00
|
|
|
|
2008-10-28 05:40:35 -04:00
|
|
|
return 63 - num;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
num = BITS_PER_LONG - 1;
|
2005-07-12 08:50:30 -04:00
|
|
|
|
2008-10-28 05:40:35 -04:00
|
|
|
#if BITS_PER_LONG == 64
|
|
|
|
|
if (!(word & (~0ul << 32))) {
|
|
|
|
|
num -= 32;
|
|
|
|
|
word <<= 32;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
|
|
|
|
|
num -= 16;
|
|
|
|
|
word <<= 16;
|
|
|
|
|
}
|
|
|
|
|
if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
|
|
|
|
|
num -= 8;
|
|
|
|
|
word <<= 8;
|
|
|
|
|
}
|
|
|
|
|
if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
|
|
|
|
|
num -= 4;
|
|
|
|
|
word <<= 4;
|
|
|
|
|
}
|
|
|
|
|
if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
|
|
|
|
|
num -= 2;
|
|
|
|
|
word <<= 2;
|
|
|
|
|
}
|
|
|
|
|
if (!(word & (~0ul << (BITS_PER_LONG-1))))
|
|
|
|
|
num -= 1;
|
|
|
|
|
return num;
|
2005-07-12 08:50:30 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* __ffs - find first bit in word.
|
2005-04-16 18:20:36 -04:00
|
|
|
* @word: The word to search
|
|
|
|
|
*
|
2005-07-12 08:50:30 -04:00
|
|
|
* Returns 0..SZLONG-1
|
|
|
|
|
* Undefined if no bit exists, so code should check against 0 first.
|
2005-04-16 18:20:36 -04:00
|
|
|
*/
|
2019-05-14 18:42:07 -04:00
|
|
|
static __always_inline unsigned long __ffs(unsigned long word)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
2008-05-04 09:53:53 -04:00
|
|
|
return __fls(word & -word);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
2006-04-17 08:19:12 -04:00
|
|
|
* fls - find last bit set.
|
2005-04-16 18:20:36 -04:00
|
|
|
* @word: The word to search
|
|
|
|
|
*
|
2006-04-17 08:19:12 -04:00
|
|
|
* This is defined the same way as ffs.
|
|
|
|
|
* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
|
2005-04-16 18:20:36 -04:00
|
|
|
*/
|
2019-01-03 18:26:41 -05:00
|
|
|
static inline int fls(unsigned int x)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
2008-10-28 05:40:35 -04:00
|
|
|
int r;
|
2005-07-12 08:50:30 -04:00
|
|
|
|
2015-04-03 18:25:00 -04:00
|
|
|
if (!__builtin_constant_p(x) &&
|
|
|
|
|
__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
|
2014-06-28 19:26:20 -04:00
|
|
|
__asm__(
|
|
|
|
|
" .set push \n"
|
2014-11-20 08:58:30 -05:00
|
|
|
" .set "MIPS_ISA_LEVEL" \n"
|
2014-06-28 19:26:20 -04:00
|
|
|
" clz %0, %1 \n"
|
|
|
|
|
" .set pop \n"
|
|
|
|
|
: "=r" (x)
|
|
|
|
|
: "r" (x));
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2008-10-28 05:40:35 -04:00
|
|
|
return 32 - x;
|
|
|
|
|
}
|
2006-04-17 08:19:12 -04:00
|
|
|
|
2008-10-28 05:40:35 -04:00
|
|
|
r = 32;
|
|
|
|
|
if (!x)
|
|
|
|
|
return 0;
|
|
|
|
|
if (!(x & 0xffff0000u)) {
|
|
|
|
|
x <<= 16;
|
|
|
|
|
r -= 16;
|
|
|
|
|
}
|
|
|
|
|
if (!(x & 0xff000000u)) {
|
|
|
|
|
x <<= 8;
|
|
|
|
|
r -= 8;
|
|
|
|
|
}
|
|
|
|
|
if (!(x & 0xf0000000u)) {
|
|
|
|
|
x <<= 4;
|
|
|
|
|
r -= 4;
|
|
|
|
|
}
|
|
|
|
|
if (!(x & 0xc0000000u)) {
|
|
|
|
|
x <<= 2;
|
|
|
|
|
r -= 2;
|
|
|
|
|
}
|
|
|
|
|
if (!(x & 0x80000000u)) {
|
|
|
|
|
x <<= 1;
|
|
|
|
|
r -= 1;
|
|
|
|
|
}
|
|
|
|
|
return r;
|
2005-07-12 08:50:30 -04:00
|
|
|
}
|
2008-10-28 05:40:35 -04:00
|
|
|
|
2006-04-17 08:19:12 -04:00
|
|
|
#include <asm-generic/bitops/fls64.h>
|
2005-07-12 08:50:30 -04:00
|
|
|
|
|
|
|
|
/*
|
2006-04-17 08:19:12 -04:00
|
|
|
* ffs - find first bit set.
|
2005-07-12 08:50:30 -04:00
|
|
|
* @word: The word to search
|
|
|
|
|
*
|
2006-04-17 08:19:12 -04:00
|
|
|
* This is defined the same way as
|
|
|
|
|
* the libc and compiler builtin ffs routines, therefore
|
|
|
|
|
* differs in spirit from the above ffz (man ffs).
|
2005-07-12 08:50:30 -04:00
|
|
|
*/
|
2006-04-17 08:19:12 -04:00
|
|
|
static inline int ffs(int word)
|
2005-07-12 08:50:30 -04:00
|
|
|
{
|
2006-04-17 08:19:12 -04:00
|
|
|
if (!word)
|
|
|
|
|
return 0;
|
2006-01-30 12:14:41 -05:00
|
|
|
|
2006-04-17 08:19:12 -04:00
|
|
|
return fls(word & -word);
|
2005-07-12 08:50:30 -04:00
|
|
|
}
|
|
|
|
|
|
2006-04-17 08:19:12 -04:00
|
|
|
#include <asm-generic/bitops/ffz.h>
|
[PATCH] bitops: mips: use generic bitops
- remove __{,test_and_}{set,clear,change}_bit() and test_bit()
- unless defined(CONFIG_CPU_MIPS32) or defined(CONFIG_CPU_MIPS64)
- remove __ffs()
- remove ffs()
- remove ffz()
- remove fls()
- remove fls64()
- remove find_{next,first}{,_zero}_bit()
- remove sched_find_first_bit()
- remove generic_hweight64()
- remove generic_hweight{32,16,8}()
- remove ext2_{set,clear,test,find_first_zero,find_next_zero}_bit()
- remove ext2_{set,clear}_bit_atomic()
- remove minix_{test,set,test_and_clear,test,find_first_zero}_bit()
Signed-off-by: Akinobu Mita <mita@miraclelinux.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-26 04:39:30 -05:00
|
|
|
#include <asm-generic/bitops/find.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
|
|
[PATCH] bitops: mips: use generic bitops
- remove __{,test_and_}{set,clear,change}_bit() and test_bit()
- unless defined(CONFIG_CPU_MIPS32) or defined(CONFIG_CPU_MIPS64)
- remove __ffs()
- remove ffs()
- remove ffz()
- remove fls()
- remove fls64()
- remove find_{next,first}{,_zero}_bit()
- remove sched_find_first_bit()
- remove generic_hweight64()
- remove generic_hweight{32,16,8}()
- remove ext2_{set,clear,test,find_first_zero,find_next_zero}_bit()
- remove ext2_{set,clear}_bit_atomic()
- remove minix_{test,set,test_and_clear,test,find_first_zero}_bit()
Signed-off-by: Akinobu Mita <mita@miraclelinux.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-26 04:39:30 -05:00
|
|
|
#include <asm-generic/bitops/sched.h>
|
2010-06-25 19:46:07 -04:00
|
|
|
|
|
|
|
|
#include <asm/arch_hweight.h>
|
|
|
|
|
#include <asm-generic/bitops/const_hweight.h>
|
|
|
|
|
|
bitops: introduce little-endian bitops for most architectures
Introduce little-endian bit operations to the big-endian architectures
which do not have native little-endian bit operations and the
little-endian architectures. (alpha, avr32, blackfin, cris, frv, h8300,
ia64, m32r, mips, mn10300, parisc, sh, sparc, tile, x86, xtensa)
These architectures can just include generic implementation
(asm-generic/bitops/le.h).
Signed-off-by: Akinobu Mita <akinobu.mita@gmail.com>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Mikael Starvik <starvik@axis.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Cc: "Luck, Tony" <tony.luck@intel.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Matthew Wilcox <willy@debian.org>
Cc: Grant Grundler <grundler@parisc-linux.org>
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: Kazumoto Kojima <kkojima@rr.iij4u.or.jp>
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Chris Zankel <chris@zankel.net>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Hans-Christian Egtvedt <hans-christian.egtvedt@atmel.com>
Acked-by: "H. Peter Anvin" <hpa@zytor.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-03-23 19:42:02 -04:00
|
|
|
#include <asm-generic/bitops/le.h>
|
[PATCH] bitops: mips: use generic bitops
- remove __{,test_and_}{set,clear,change}_bit() and test_bit()
- unless defined(CONFIG_CPU_MIPS32) or defined(CONFIG_CPU_MIPS64)
- remove __ffs()
- remove ffs()
- remove ffz()
- remove fls()
- remove fls64()
- remove find_{next,first}{,_zero}_bit()
- remove sched_find_first_bit()
- remove generic_hweight64()
- remove generic_hweight{32,16,8}()
- remove ext2_{set,clear,test,find_first_zero,find_next_zero}_bit()
- remove ext2_{set,clear}_bit_atomic()
- remove minix_{test,set,test_and_clear,test,find_first_zero}_bit()
Signed-off-by: Akinobu Mita <mita@miraclelinux.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-26 04:39:30 -05:00
|
|
|
#include <asm-generic/bitops/ext2-atomic.h>
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
|
#endif /* __KERNEL__ */
|
|
|
|
|
|
|
|
|
|
#endif /* _ASM_BITOPS_H */
|
