diff --git a/src/os/unix/ngx_gcc_atomic_ppc.h b/src/os/unix/ngx_gcc_atomic_ppc.h index cfa58f92a..e82e75574 100644 --- a/src/os/unix/ngx_gcc_atomic_ppc.h +++ b/src/os/unix/ngx_gcc_atomic_ppc.h @@ -15,6 +15,11 @@ * any register except r0. The r0 register always has a zero value and * could not be used in "addi r0, r0, 1". * The "=&b" means that no input registers can be used. + * + * "sync" read and write barriers + * "isync" read barrier, is faster than "sync" + * "eieio" write barrier, is faster than "sync" + * "lwsync" write barrier, is faster than "eieio" on ppc64 */ #if (NGX_PTR_SIZE == 8) @@ -28,6 +33,7 @@ ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, __asm__ volatile ( " li %0, 0 \n" /* preset "0" to "res" */ + " lwsync \n" /* write barrier */ "1: \n" " ldarx %1, 0, %2 \n" /* load from [lock] into "temp" */ /* and store reservation */ @@ -36,6 +42,7 @@ ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, " stdcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */ /* is not cleared */ " bne- 1b \n" /* the reservation was cleared */ + " isync \n" /* read barrier */ " li %0, 1 \n" /* set "1" to "res" */ "2: \n" @@ -54,12 +61,14 @@ ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) __asm__ volatile ( + " lwsync \n" /* write barrier */ "1: ldarx %0, 0, %2 \n" /* load from [value] into "res" */ /* and store reservation */ " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */ " stdcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */ /* is not cleared */ " bne- 1b \n" /* try again if reservation was cleared */ + " isync \n" /* read barrier */ : "=&b" (res), "=&b" (temp) : "b" (value), "b" (add) @@ -70,7 +79,8 @@ ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) #if (NGX_SMP) -#define ngx_memory_barrier() __asm__ volatile ("lwsync\n" ::: "memory") +#define ngx_memory_barrier() \ + __asm__ volatile ("isync \n lwsync \n" ::: "memory") #else #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory") #endif @@ -86,6 +96,7 @@ ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, __asm__ volatile ( " li %0, 0 \n" /* preset "0" to "res" */ + " eieio \n" /* write barrier */ "1: \n" " lwarx %1, 0, %2 \n" /* load from [lock] into "temp" */ /* and store reservation */ @@ -94,6 +105,7 @@ ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old, " stwcx. %4, 0, %2 \n" /* store "set" into [lock] if reservation */ /* is not cleared */ " bne- 1b \n" /* the reservation was cleared */ + " isync \n" /* read barrier */ " li %0, 1 \n" /* set "1" to "res" */ "2: \n" @@ -112,12 +124,14 @@ ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) __asm__ volatile ( + " eieio \n" /* write barrier */ "1: lwarx %0, 0, %2 \n" /* load from [value] into "res" */ /* and store reservation */ " add %1, %0, %3 \n" /* "res" + "add" store in "temp" */ " stwcx. %1, 0, %2 \n" /* store "temp" into [value] if reservation */ /* is not cleared */ " bne- 1b \n" /* try again if reservation was cleared */ + " isync \n" /* read barrier */ : "=&b" (res), "=&b" (temp) : "b" (value), "b" (add) @@ -128,7 +142,8 @@ ngx_atomic_fetch_add(ngx_atomic_t *value, ngx_atomic_int_t add) #if (NGX_SMP) -#define ngx_memory_barrier() __asm__ volatile ("sync\n" ::: "memory") +#define ngx_memory_barrier() \ + __asm__ volatile ("isync \n eieio \n" ::: "memory") #else #define ngx_memory_barrier() __asm__ volatile ("" ::: "memory") #endif