From 1029b22feb8d9768df9e8b7b0344e2961bf82659 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Mon, 4 Sep 2017 15:40:21 +0400 Subject: [PATCH] MDEV-13728 - Import MySQL 5.7 atomic operations for MSVC and Solaris gcc_sync.h, solaris.h, generic-msvc.h copied verbatim. --- include/atomic/gcc_builtins.h | 20 +--- include/atomic/gcc_sync.h | 106 ++++++++++++++++++++ include/atomic/generic-msvc.h | 175 ++++++++++++++++++---------------- include/atomic/solaris.h | 116 ++++++++++++++++------ include/my_atomic.h | 112 +--------------------- unittest/mysys/thr_template.c | 2 +- 6 files changed, 287 insertions(+), 244 deletions(-) create mode 100644 include/atomic/gcc_sync.h diff --git a/include/atomic/gcc_builtins.h b/include/atomic/gcc_builtins.h index b6d7ec6d28c..99e6f628b50 100644 --- a/include/atomic/gcc_builtins.h +++ b/include/atomic/gcc_builtins.h @@ -1,7 +1,7 @@ #ifndef ATOMIC_GCC_BUILTINS_INCLUDED #define ATOMIC_GCC_BUILTINS_INCLUDED -/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2017 MariaDB Foundation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -16,8 +16,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#if defined(HAVE_GCC_C11_ATOMICS) -#define MY_ATOMIC_MODE "gcc-atomics-smp" #define MY_MEMORY_ORDER_RELAXED __ATOMIC_RELAXED #define MY_MEMORY_ORDER_CONSUME __ATOMIC_CONSUME @@ -76,21 +74,5 @@ __atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) #define my_atomic_casptr(P, E, D) \ __atomic_compare_exchange_n((P), (E), (D), 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) -#else -#define MY_ATOMIC_MODE "gcc-builtins-smp" -#define make_atomic_load_body(S) \ - ret= __sync_fetch_and_or(a, 0); -#define make_atomic_store_body(S) \ - (void) __sync_lock_test_and_set(a, v); -#define make_atomic_add_body(S) \ - v= __sync_fetch_and_add(a, v); -#define make_atomic_fas_body(S) \ - v= __sync_lock_test_and_set(a, v); -#define make_atomic_cas_body(S) \ - int ## S sav; \ - int ## S cmp_val= *cmp; \ - sav= __sync_val_compare_and_swap(a, cmp_val, set);\ - if (!(ret= (sav == cmp_val))) *cmp= sav -#endif #endif /* ATOMIC_GCC_BUILTINS_INCLUDED */ diff --git a/include/atomic/gcc_sync.h b/include/atomic/gcc_sync.h new file mode 100644 index 00000000000..82eea35b2ce --- /dev/null +++ b/include/atomic/gcc_sync.h @@ -0,0 +1,106 @@ +#ifndef GCC_SYNC_INCLUDED +#define GCC_SYNC_INCLUDED + +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* Old GCC __sync builtins introduced in GCC 4.1 */ + +static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set) +{ + int32 cmp_val= *cmp; + int32 sav= __sync_val_compare_and_swap(a, cmp_val, set); + int ret= (sav == cmp_val); + if (!ret) + *cmp = sav; + return ret; +} + +static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set) +{ + int64 cmp_val= *cmp; + int64 sav= __sync_val_compare_and_swap(a, cmp_val, set); + int ret= (sav == cmp_val); + if (!ret) + *cmp = sav; + return ret; +} + +static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set) +{ + void *cmp_val= *cmp; + void *sav= __sync_val_compare_and_swap(a, cmp_val, set); + int ret= (sav == cmp_val); + if (!ret) + *cmp = sav; + return ret; +} + +static inline int32 my_atomic_add32(int32 volatile *a, int32 v) +{ + return __sync_fetch_and_add(a, v); +} + +static inline int64 my_atomic_add64(int64 volatile *a, int64 v) +{ + return __sync_fetch_and_add(a, v); +} + +static inline int32 my_atomic_fas32(int32 volatile *a, int32 v) +{ + return __sync_lock_test_and_set(a, v); +} + +static inline int64 my_atomic_fas64(int64 volatile *a, int64 v) +{ + return __sync_lock_test_and_set(a, v); +} + +static inline void * my_atomic_fasptr(void * volatile *a, void * v) +{ + return __sync_lock_test_and_set(a, v); +} + +static inline int32 my_atomic_load32(int32 volatile *a) +{ + return __sync_fetch_and_or(a, 0); +} + +static inline int64 my_atomic_load64(int64 volatile *a) +{ + return __sync_fetch_and_or(a, 0); +} + +static inline void* my_atomic_loadptr(void * volatile *a) +{ + return __sync_fetch_and_or(a, 0); +} + +static inline void my_atomic_store32(int32 volatile *a, int32 v) +{ + (void) __sync_lock_test_and_set(a, v); +} + +static inline void my_atomic_store64(int64 volatile *a, int64 v) +{ + (void) __sync_lock_test_and_set(a, v); +} + +static inline void my_atomic_storeptr(void * volatile *a, void *v) +{ + (void) __sync_lock_test_and_set(a, v); +} + +#endif /* GCC_SYNC_INCLUDED */ diff --git a/include/atomic/generic-msvc.h b/include/atomic/generic-msvc.h index d06229ce5ef..9fe2cddd7ae 100644 --- a/include/atomic/generic-msvc.h +++ b/include/atomic/generic-msvc.h @@ -1,5 +1,7 @@ -/* Copyright (c) 2006-2008 MySQL AB, 2009 Sun Microsystems, Inc. - Use is subject to license terms. +#ifndef ATOMIC_MSC_INCLUDED +#define ATOMIC_MSC_INCLUDED + +/* Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,79 +16,101 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef _atomic_h_cleanup_ -#define _atomic_h_cleanup_ "atomic/generic-msvc.h" - #include -/* - x86 compilers (both VS2003 or VS2005) never use instrinsics, but generate - function calls to kernel32 instead, even in the optimized build. - We force intrinsics as described in MSDN documentation for - _InterlockedCompareExchange. -*/ -#ifdef _M_IX86 -#if (_MSC_VER >= 1500) -#include -#else -C_MODE_START -/*Visual Studio 2003 and earlier do not have prototypes for atomic intrinsics*/ -LONG _InterlockedCompareExchange (LONG volatile *Target, LONG Value, LONG Comp); -LONGLONG _InterlockedCompareExchange64 (LONGLONG volatile *Target, - LONGLONG Value, LONGLONG Comp); -C_MODE_END +static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set) +{ + int32 initial_cmp= *cmp; + int32 initial_a= InterlockedCompareExchange((volatile LONG*)a, + set, initial_cmp); + int ret= (initial_a == initial_cmp); + if (!ret) + *cmp= initial_a; + return ret; +} -#pragma intrinsic(_InterlockedCompareExchange) -#pragma intrinsic(_InterlockedCompareExchange64) -#endif +static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set) +{ + int64 initial_cmp= *cmp; + int64 initial_a= InterlockedCompareExchange64((volatile LONGLONG*)a, + (LONGLONG)set, + (LONGLONG)initial_cmp); + int ret= (initial_a == initial_cmp); + if (!ret) + *cmp= initial_a; + return ret; +} -#define InterlockedCompareExchange _InterlockedCompareExchange -#define InterlockedCompareExchange64 _InterlockedCompareExchange64 -/* - No need to do something special for InterlockedCompareExchangePointer - as it is a #define to InterlockedCompareExchange. The same applies to - InterlockedExchangePointer. -*/ -#endif /*_M_IX86*/ +static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set) +{ + void *initial_cmp= *cmp; + void *initial_a= InterlockedCompareExchangePointer(a, set, initial_cmp); + int ret= (initial_a == initial_cmp); + if (!ret) + *cmp= initial_a; + return ret; +} -#define MY_ATOMIC_MODE "msvc-intrinsics" -/* Implement using CAS on WIN32 */ -#define IL_COMP_EXCHG32(X,Y,Z) \ - InterlockedCompareExchange((volatile LONG *)(X),(Y),(Z)) -#define IL_COMP_EXCHG64(X,Y,Z) \ - InterlockedCompareExchange64((volatile LONGLONG *)(X), \ - (LONGLONG)(Y),(LONGLONG)(Z)) -#define IL_COMP_EXCHGptr InterlockedCompareExchangePointer +static inline int32 my_atomic_add32(int32 volatile *a, int32 v) +{ + return (int32)InterlockedExchangeAdd((volatile LONG*)a, v); +} -#define make_atomic_cas_body(S) \ - int ## S initial_cmp= *cmp; \ - int ## S initial_a= IL_COMP_EXCHG ## S (a, set, initial_cmp); \ - if (!(ret= (initial_a == initial_cmp))) *cmp= initial_a; +static inline int64 my_atomic_add64(int64 volatile *a, int64 v) +{ + return (int64)InterlockedExchangeAdd64((volatile LONGLONG*)a, (LONGLONG)v); +} + +static inline int32 my_atomic_load32(int32 volatile *a) +{ + return (int32)InterlockedCompareExchange((volatile LONG *)a, 0, 0); +} + +static inline int64 my_atomic_load64(int64 volatile *a) +{ + return (int64)InterlockedCompareExchange64((volatile LONGLONG *)a, 0, 0); +} + +static inline void* my_atomic_loadptr(void * volatile *a) +{ + return InterlockedCompareExchangePointer(a, 0, 0); +} + +static inline int32 my_atomic_fas32(int32 volatile *a, int32 v) +{ + return (int32)InterlockedExchange((volatile LONG*)a, v); +} + +static inline int64 my_atomic_fas64(int64 volatile *a, int64 v) +{ + return (int64)InterlockedExchange64((volatile LONGLONG*)a, v); +} + +static inline void * my_atomic_fasptr(void * volatile *a, void * v) +{ + return InterlockedExchangePointer(a, v); +} + +static inline void my_atomic_store32(int32 volatile *a, int32 v) +{ + (void)InterlockedExchange((volatile LONG*)a, v); +} + +static inline void my_atomic_store64(int64 volatile *a, int64 v) +{ + (void)InterlockedExchange64((volatile LONGLONG*)a, v); +} + +static inline void my_atomic_storeptr(void * volatile *a, void *v) +{ + (void)InterlockedExchangePointer(a, v); +} -#ifndef _M_IX86 -/* Use full set of optimised functions on WIN64 */ -#define IL_EXCHG_ADD32(X,Y) \ - InterlockedExchangeAdd((volatile LONG *)(X),(Y)) -#define IL_EXCHG_ADD64(X,Y) \ - InterlockedExchangeAdd64((volatile LONGLONG *)(X),(LONGLONG)(Y)) -#define IL_EXCHG32(X,Y) \ - InterlockedExchange((volatile LONG *)(X),(Y)) -#define IL_EXCHG64(X,Y) \ - InterlockedExchange64((volatile LONGLONG *)(X),(LONGLONG)(Y)) -#define IL_EXCHGptr InterlockedExchangePointer -#define make_atomic_add_body(S) \ - v= IL_EXCHG_ADD ## S (a, v) -#define make_atomic_swap_body(S) \ - v= IL_EXCHG ## S (a, v) -#define make_atomic_load_body(S) \ - ret= 0; /* avoid compiler warning */ \ - ret= IL_COMP_EXCHG ## S (a, ret, ret); -#endif /* my_yield_processor (equivalent of x86 PAUSE instruction) should be used to improve performance on hyperthreaded CPUs. Intel recommends to use it in - spin loops also on non-HT machines to reduce power consumption (see e.g + spin loops also on non-HT machines to reduce power consumption (see e.g http://softwarecommunity.intel.com/articles/eng/2004.htm) Running benchmarks for spinlocks implemented with InterlockedCompareExchange @@ -94,35 +118,18 @@ C_MODE_END YieldProcessor in a loop - that is, yielding longer. On Intel boxes setting loop count in the range 200-300 brought best results. */ -#ifndef YIELD_LOOPS #define YIELD_LOOPS 200 -#endif -static __inline int my_yield_processor() +static inline int my_yield_processor() { int i; - for(i=0; i -#define MY_ATOMIC_MODE "solaris-atomic" - #if defined(__GNUC__) #define atomic_typeof(T,V) __typeof__(V) #else #define atomic_typeof(T,V) T #endif -#define uintptr_t void * -#define atomic_or_ptr_nv(X,Y) (void *)atomic_or_ulong_nv((volatile ulong_t *)X, Y) - -#define make_atomic_cas_body(S) \ - atomic_typeof(uint ## S ## _t, *cmp) sav; \ - sav = atomic_cas_ ## S( \ - (volatile uint ## S ## _t *)a, \ - (uint ## S ## _t)*cmp, \ - (uint ## S ## _t)set); \ - if (! (ret= (sav == *cmp))) \ +static inline int my_atomic_cas32(int32 volatile *a, int32 *cmp, int32 set) +{ + int ret; + atomic_typeof(uint32_t, *cmp) sav; + sav= atomic_cas_32((volatile uint32_t *)a, (uint32_t)*cmp, (uint32_t)set); + ret= (sav == *cmp); + if (!ret) *cmp= sav; + return ret; +} -#define make_atomic_add_body(S) \ - int ## S nv; /* new value */ \ - nv= atomic_add_ ## S ## _nv((volatile uint ## S ## _t *)a, v); \ - v= nv - v +static inline int my_atomic_cas64(int64 volatile *a, int64 *cmp, int64 set) +{ + int ret; + atomic_typeof(uint64_t, *cmp) sav; + sav= atomic_cas_64((volatile uint64_t *)a, (uint64_t)*cmp, (uint64_t)set); + ret= (sav == *cmp); + if (!ret) + *cmp= sav; + return ret; +} -/* ------------------------------------------------------------------------ */ +static inline int my_atomic_casptr(void * volatile *a, void **cmp, void *set) +{ + int ret; + atomic_typeof(void *, *cmp) sav; + sav= atomic_cas_ptr((volatile void **)a, (void *)*cmp, (void *)set); + ret= (sav == *cmp); + if (!ret) + *cmp= sav; + return ret; +} -#define make_atomic_load_body(S) \ - ret= atomic_or_ ## S ## _nv((volatile uint ## S ## _t *)a, 0) +static inline int32 my_atomic_add32(int32 volatile *a, int32 v) +{ + int32 nv= atomic_add_32_nv((volatile uint32_t *)a, v); + return nv - v; +} -#define make_atomic_store_body(S) \ - (void) atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v) +static inline int64 my_atomic_add64(int64 volatile *a, int64 v) +{ + int64 nv= atomic_add_64_nv((volatile uint64_t *)a, v); + return nv - v; +} -#define make_atomic_fas_body(S) \ - v= atomic_swap_ ## S((volatile uint ## S ## _t *)a, (uint ## S ## _t)v) +static inline int32 my_atomic_fas32(int32 volatile *a, int32 v) +{ + return atomic_swap_32((volatile uint32_t *)a, (uint32_t)v); +} -#else /* cleanup */ +static inline int64 my_atomic_fas64(int64 volatile *a, int64 v) +{ + return atomic_swap_64((volatile uint64_t *)a, (uint64_t)v); +} -#undef uintptr_t -#undef atomic_or_ptr_nv +static inline void * my_atomic_fasptr(void * volatile *a, void * v) +{ + return atomic_swap_ptr(a, v); +} -#endif +static inline int32 my_atomic_load32(int32 volatile *a) +{ + return atomic_or_32_nv((volatile uint32_t *)a, 0); +} +static inline int64 my_atomic_load64(int64 volatile *a) +{ + return atomic_or_64_nv((volatile uint64_t *)a, 0); +} + +static inline void* my_atomic_loadptr(void * volatile *a) +{ + return atomic_add_ptr_nv(a, 0); +} + +static inline void my_atomic_store32(int32 volatile *a, int32 v) +{ + (void) atomic_swap_32((volatile uint32_t *)a, (uint32_t)v); +} + +static inline void my_atomic_store64(int64 volatile *a, int64 v) +{ + (void) atomic_swap_64((volatile uint64_t *)a, (uint64_t)v); +} + +static inline void my_atomic_storeptr(void * volatile *a, void *v) +{ + (void) atomic_swap_ptr((volatile void **)a, (void *)v); +} + +#endif /* ATOMIC_SOLARIS_INCLUDED */ diff --git a/include/my_atomic.h b/include/my_atomic.h index bcb6005a8c2..32c9d6b4736 100644 --- a/include/my_atomic.h +++ b/include/my_atomic.h @@ -110,119 +110,13 @@ #include "atomic/generic-msvc.h" #elif defined(HAVE_SOLARIS_ATOMIC) #include "atomic/solaris.h" -#elif defined(HAVE_GCC_ATOMIC_BUILTINS) || defined(HAVE_GCC_C11_ATOMICS) +#elif defined(HAVE_GCC_C11_ATOMICS) #include "atomic/gcc_builtins.h" +#elif defined(HAVE_GCC_ATOMIC_BUILTINS) +#include "atomic/gcc_sync.h" #endif -#ifndef HAVE_GCC_C11_ATOMICS -#ifndef make_atomic_cas_body -/* nolock.h was not able to generate even a CAS function, fall back */ -#error atomic ops for this platform are not implemented -#endif - -#define intptr void * - -/* define missing functions by using the already generated ones */ -#ifndef make_atomic_add_body -#define make_atomic_add_body(S) \ - int ## S tmp=*a; \ - while (!my_atomic_cas ## S(a, &tmp, tmp+v)) ; \ - v=tmp; -#endif -#ifndef make_atomic_fas_body -#define make_atomic_fas_body(S) \ - int ## S tmp=*a; \ - while (!my_atomic_cas ## S(a, &tmp, v)) ; \ - v=tmp; -#endif -#ifndef make_atomic_load_body -#define make_atomic_load_body(S) \ - ret= 0; /* avoid compiler warning */ \ - (void)(my_atomic_cas ## S(a, &ret, ret)); -#endif -#ifndef make_atomic_store_body -#define make_atomic_store_body(S) \ - (void)(my_atomic_fas ## S (a, v)); -#endif - -#define make_atomic_cas(S) \ -static inline int my_atomic_cas ## S(int ## S volatile *a, \ - int ## S *cmp, int ## S set) \ -{ \ - int8 ret; \ - make_atomic_cas_body(S); \ - return ret; \ -} - -#define make_atomic_add(S) \ -static inline int ## S my_atomic_add ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_add_body(S); \ - return v; \ -} - -#define make_atomic_fas(S) \ -static inline int ## S my_atomic_fas ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_fas_body(S); \ - return v; \ -} - -#define make_atomic_load(S) \ -static inline int ## S my_atomic_load ## S(int ## S volatile *a)\ -{ \ - int ## S ret; \ - make_atomic_load_body(S); \ - return ret; \ -} - -#define make_atomic_store(S) \ -static inline void my_atomic_store ## S( \ - int ## S volatile *a, int ## S v) \ -{ \ - make_atomic_store_body(S); \ -} - -make_atomic_cas(32) -make_atomic_cas(64) -make_atomic_cas(ptr) - -make_atomic_add(32) -make_atomic_add(64) - -make_atomic_load(32) -make_atomic_load(64) -make_atomic_load(ptr) - -make_atomic_fas(32) -make_atomic_fas(64) -make_atomic_fas(ptr) - -make_atomic_store(32) -make_atomic_store(64) -make_atomic_store(ptr) - -#ifdef _atomic_h_cleanup_ -#include _atomic_h_cleanup_ -#undef _atomic_h_cleanup_ -#endif - -#undef make_atomic_add -#undef make_atomic_cas -#undef make_atomic_load -#undef make_atomic_store -#undef make_atomic_fas -#undef make_atomic_add_body -#undef make_atomic_cas_body -#undef make_atomic_load_body -#undef make_atomic_store_body -#undef make_atomic_fas_body -#undef intptr -#endif - /* the macro below defines (as an expression) the code that will be run in spin-loops. Intel manuals recummend to have PAUSE there. diff --git a/unittest/mysys/thr_template.c b/unittest/mysys/thr_template.c index 38999022da0..83c7432c823 100644 --- a/unittest/mysys/thr_template.c +++ b/unittest/mysys/thr_template.c @@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char **argv) #define CYCLES 3000 #define THREADS 30 - diag("N CPUs: %d, atomic ops: %s", my_getncpus(), MY_ATOMIC_MODE); + diag("N CPUs: %d", my_getncpus()); do_tests();