/** * @file rtems/score/cpuatomic.h * * This include file implements the atomic operations for PowerPC and defines * atomic data types which are used by the atomic operations API file. This * file should use fixed name cpuatomic.h and should be included in atomic * operations API file atomic.h. Most of the parts of implementations are * imported from FreeBSD kernel. */ /* * Copyright (c) 2008 Marcel Moolenaar * Copyright (c) 2001 Benno Rice * Copyright (c) 2001 David E. O'Brien * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _RTEMS_SCORE_ATOMIC_CPU_H #define _RTEMS_SCORE_ATOMIC_CPU_H #include #ifdef __cplusplus extern "C" { #endif /** * @defgroup RTEMS atomic implementation * */ /**@{*/ #define __ATOMIC_BARRIER \ __asm __volatile("sync" : : : "memory") #define mb() __ATOMIC_BARRIER #define wmb() mb() #define rmb() mb() /* * Atomic_Fetch_add(p, v) * { *p += v; } */ #define __CPU_Atomic_Fetch_add_int(p, v, t) \ __asm __volatile( \ "1:lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_add_int */ #define __CPU_Atomic_Fetch_add_long(p, v, t) \ __asm __volatile( \ "1:lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_add_long */ #define _ATOMIC_ADD(typename, type) \ static __inline void \ _CPU_Atomic_Fetch_add_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_add_##typename(p, v, t); \ } \ \ static __inline void \ _CPU_Atomic_Fetch_add_acq_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_add_##typename(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ _CPU_Atomic_Fetch_add_rel_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __ATOMIC_BARRIER; \ __CPU_Atomic_Fetch_add_##typename(p, v, t); \ } \ /* _ATOMIC_ADD */ _ATOMIC_ADD(int, Int) _ATOMIC_ADD(long, Long) #define _CPU_Atomic_Fetch_add_32(p, v) \ _CPU_Atomic_Fetch_add_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_add_acq_32(p, v) \ _CPU_Atomic_Fetch_add_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_add_rel_32(p, v) \ _CPU_Atomic_Fetch_add_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_add_ptr(p, v) \ _CPU_Atomic_Fetch_add_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_add_acq_ptr(p, v) \ _CPU_Atomic_Fetch_add_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_add_rel_ptr(p, v) \ _CPU_Atomic_Fetch_add_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #undef _ATOMIC_ADD #undef __CPU_Atomic_Fetch_add_long #undef __CPU_Atomic_Fetch_add_int /* * Atomic_Fetch_and(p, v) * { *p &= v; } */ #define __CPU_Atomic_Fetch_and_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " and %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* _CPU_Atomic_Fetch_and_int */ #define __CPU_Atomic_Fetch_and_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " and %0, %0, %3\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* _CPU_Atomic_Fetch_and_long */ #define _ATOMIC_AND(typename, type) \ static __inline void \ _CPU_Atomic_Fetch_and_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_and_##typename(p, v, t); \ } \ \ static __inline void \ _CPU_Atomic_Fetch_and_acq_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_and_##typename(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ _CPU_Atomic_Fetch_and_rel_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __ATOMIC_BARRIER; \ __CPU_Atomic_Fetch_and_##typename(p, v, t); \ } \ /* _ATOMIC_AND */ _ATOMIC_AND(int, Int) _ATOMIC_AND(long, Long) #define _CPU_Atomic_Fetch_and_32(p, v) \ _CPU_Atomic_Fetch_and_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_and_acq_32(p, v) \ _CPU_Atomic_Fetch_and_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_and_rel_32(p, v) \ _CPU_Atomic_Fetch_and_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_and_ptr(p, v) \ _CPU_Atomic_Fetch_and_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_and_acq_ptr(p, v) \ _CPU_Atomic_Fetch_and_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_and_rel_ptr(p, v) \ _CPU_Atomic_Fetch_and_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #undef _ATOMIC_AND #undef __CPU_Atomic_Fetch_and_long #undef __CPU_Atomic_Fetch_and_int /* * Atomic_Fetch_or(p, v) * { *p |= v; } */ #define __CPU_Atomic_Fetch_or_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_or_int */ #define __CPU_Atomic_Fetch_or_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_or_long */ #define _ATOMIC_OR(typename, type) \ static __inline void \ _CPU_Atomic_Fetch_or_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_or_##typename(p, v, t); \ } \ \ static __inline void \ _CPU_Atomic_Fetch_or_acq_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_or_##typename(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ _CPU_Atomic_Fetch_or_rel_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __ATOMIC_BARRIER; \ __CPU_Atomic_Fetch_or_##typename(p, v, t); \ } \ /* _ATOMIC_OR */ _ATOMIC_OR(int, Int) _ATOMIC_OR(long, Long) #define _CPU_Atomic_Fetch_or_32(p, v) \ _CPU_Atomic_Fetch_or_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_or_acq_32(p, v) \ _CPU_Atomic_Fetch_or_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_or_rel_32(p, v) \ _CPU_Atomic_Fetch_or_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_or_ptr(p, v) \ _CPU_Atomic_Fetch_or_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_or_acq_ptr(p, v) \ _CPU_Atomic_Fetch_or_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_or_rel_ptr(p, v) \ _CPU_Atomic_Fetch_or_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #undef _ATOMIC_OR #undef __CPU_Atomic_Fetch_or_long #undef __CPU_Atomic_Fetch_or_int /* * Atomic_Fetch_sub(p, v) * { *p -= v; } */ #define __CPU_Atomic_Fetch_sub_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_sub_int */ #define __CPU_Atomic_Fetch_sub_long(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ " stwcx. %0, 0, %2\n" \ " bne- 1b\n" \ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ /* __CPU_Atomic_Fetch_sub_long */ #define _ATOMIC_SUB(typename, type) \ static __inline void \ _CPU_Atomic_Fetch_sub_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_sub_##typename(p, v, t); \ } \ \ static __inline void \ _CPU_Atomic_Fetch_sub_acq_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __CPU_Atomic_Fetch_sub_##typename(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ _CPU_Atomic_Fetch_sub_rel_##typename(volatile Atomic_##type *p, Atomic_##type v) { \ Atomic_##type t; \ __ATOMIC_BARRIER; \ __CPU_Atomic_Fetch_sub_##typename(p, v, t); \ } \ /* _ATOMIC_SUB */ _ATOMIC_SUB(int, Int) _ATOMIC_SUB(long, Long) #define _CPU_Atomic_Fetch_sub_32(p, v) \ _CPU_Atomic_Fetch_sub_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_sub_acq_32(p, v) \ _CPU_Atomic_Fetch_sub_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_sub_rel_32(p, v) \ _CPU_Atomic_Fetch_sub_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_sub_ptr(p, v) \ _CPU_Atomic_Fetch_sub_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_sub_acq_ptr(p, v) \ _CPU_Atomic_Fetch_sub_acq_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Fetch_sub_rel_ptr(p, v) \ _CPU_Atomic_Fetch_sub_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #undef _ATOMIC_SUB #undef __CPU_Atomic_Fetch_sub_long #undef __CPU_Atomic_Fetch_sub_int /* * We assume that a = b will do atomic loads and stores. */ #define ATOMIC_STORE_LOAD(TYPENAME, TYPE) \ static __inline Atomic_##TYPE \ _CPU_Atomic_Load_##TYPENAME(volatile Atomic_##TYPE *p) \ { \ Atomic_##TYPE v; \ \ v = *p; \ return (v); \ } \ static __inline Atomic_##TYPE \ _CPU_Atomic_Load_acq_##TYPENAME(volatile Atomic_##TYPE *p) \ { \ Atomic_##TYPE v; \ \ v = *p; \ __ATOMIC_BARRIER; \ return (v); \ } \ \ static __inline void \ _CPU_Atomic_Store_##TYPENAME(volatile Atomic_##TYPE *p, Atomic_##TYPE v) \ { \ *p = v; \ } \ static __inline void \ _CPU_Atomic_Store_rel_##TYPENAME(volatile Atomic_##TYPE *p, Atomic_##TYPE v) \ { \ __ATOMIC_BARRIER; \ *p = v; \ } ATOMIC_STORE_LOAD(int, Int) #define _CPU_Atomic_Load_32(p) \ _CPU_Atomic_Load_int((volatile Atomic_Int *)(p)) #define _CPU_Atomic_Load_acq_32(p) \ _CPU_Atomic_Load_acq_int((volatile Atomic_Int *)(p)) #define _CPU_Atomic_Store_32(p, v) \ _CPU_Atomic_Store_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) #define _CPU_Atomic_Store_rel_32(p, v) \ _CPU_Atomic_Store_rel_int((volatile Atomic_Int *)(p), (Atomic_Int)(v)) static __inline Atomic_Long _CPU_Atomic_Load_long(volatile Atomic_Long *addr) { return ((Atomic_Long)_CPU_Atomic_Load_int((volatile Atomic_Int *)addr)); } static __inline Atomic_Long _CPU_Atomic_Load_acq_long(volatile Atomic_Long *addr) { return ((Atomic_Long)_CPU_Atomic_Load_acq_int((volatile Atomic_Int *)addr)); } static __inline void _CPU_Atomic_Store_long(volatile Atomic_Long *addr, Atomic_Long val) { _CPU_Atomic_Store_int((volatile Atomic_Int *)addr, (Atomic_Int)val); } static __inline void _CPU_Atomic_Store_rel_long(volatile Atomic_Long *addr, Atomic_Long val) { _CPU_Atomic_Store_rel_int((volatile Atomic_Int *)addr, (Atomic_Int)val); } #define _CPU_Atomic_Load_ptr(p) \ _CPU_Atomic_Load_int((volatile Atomic_Int *)(p)) #define _CPU_Atomic_Load_acq_ptr(p) \ _CPU_Atomic_Load_acq_int((volatile Atomic_Int *)(p)) #define _CPU_Atomic_Store_ptr(p, v) \ _CPU_Atomic_Store_int((volatile Atomic_Int *)(p), (v)) #define _CPU_Atomic_Store_rel_ptr(p, v) \ _CPU_Atomic_Store_rel_int((volatile Atomic_Int *)(p), (v)) #undef ATOMIC_STORE_LOAD /* * * Atomically compare the value stored at *p with cmpval and if the * * two values are equal, update the value of *p with newval. Returns * * zero if the compare failed, nonzero otherwise. * */ static __inline int _CPU_Atomic_Compare_exchange_int(volatile Atomic_Int* p, Atomic_Int cmpval, Atomic_Int newval) { int ret; __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cc", "memory"); return (ret); } static __inline int _CPU_Atomic_Compare_exchange_long(volatile Atomic_Long* p, Atomic_Long cmpval, Atomic_Long newval) { int ret; __asm __volatile ( "1:\tlwarx %0, 0, %2\n\t" /* load old value */ "cmplw %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stwcx. %4, 0, %2\n\t" /* attempt to store */ "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) : "cc", "memory"); return (ret); } static __inline int _CPU_Atomic_Compare_exchange_acq_int(volatile Atomic_Int *p, Atomic_Int cmpval, Atomic_Int newval) { int retval; retval = _CPU_Atomic_Compare_exchange_int(p, cmpval, newval); __ATOMIC_BARRIER; return (retval); } static __inline int _CPU_Atomic_Compare_exchange_rel_int(volatile Atomic_Int *p, Atomic_Int cmpval, Atomic_Int newval) { __ATOMIC_BARRIER; return (_CPU_Atomic_Compare_exchange_int(p, cmpval, newval)); } static __inline int _CPU_Atomic_Compare_exchange_acq_long(volatile Atomic_Long *p, Atomic_Long cmpval, Atomic_Long newval) { Atomic_Long retval; retval = _CPU_Atomic_Compare_exchange_long(p, cmpval, newval); __ATOMIC_BARRIER; return (retval); } static __inline int _CPU_Atomic_Compare_exchange_rel_long(volatile Atomic_Long *p, Atomic_Long cmpval, Atomic_Long newval) { __ATOMIC_BARRIER; return (_CPU_Atomic_Compare_exchange_long(p, cmpval, newval)); } #define _CPU_Atomic_Compare_exchange_32(dst, old, new) \ _CPU_Atomic_Compare_exchange_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), (Atomic_Int)(new)) #define _CPU_Atomic_Compare_exchange_acq_32(dst, old, new) \ _CPU_Atomic_Compare_exchange_acq_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), (Atomic_Int)(new)) #define _CPU_Atomic_Compare_exchange_rel_32(dst, old, new) \ _CPU_Atomic_Compare_exchange_rel_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), (Atomic_Int)(new)) #define _CPU_Atomic_Compare_exchange_ptr(dst, old, new) \ _CPU_Atomic_Compare_exchange_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), (Atomic_Int)(new)) #define _CPU_Atomic_Compare_exchange_acq_ptr(dst, old, new) \ _CPU_Atomic_Compare_exchange_acq_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), \ (Atomic_Int)(new)) #define _CPU_Atomic_Compare_exchange_rel_ptr(dst, old, new) \ _CPU_Atomic_Compare_exchange_rel_int((volatile Atomic_Int *)(dst), (Atomic_Int)(old), \ (Atomic_Int)(new)) #ifdef __cplusplus } #endif /**@}*/ #endif /* end of include file */