Modified: trunk/Source/WTF/ChangeLog (205920 => 205921)
--- trunk/Source/WTF/ChangeLog 2016-09-14 18:46:32 UTC (rev 205920)
+++ trunk/Source/WTF/ChangeLog 2016-09-14 18:51:26 UTC (rev 205921)
@@ -1,5 +1,44 @@
2016-09-14 JF Bastien <[email protected]>
+ Atomics on ARM don't require full-system fencing, and other minutiae
+ https://bugs.webkit.org/show_bug.cgi?id=161928
+
+ Reviewed by Geoffrey Garen.
+
+ Add cmpxchg versions with both success and failure memory
+ ordering. In some interesting cases we can craft code which needs
+ barriers which aren't as strong.
+
+ weakCompareAndSwap is super dubious, its 3 uses seem
+ questionable... but for now I'm just adding debug asserts.
+
+ Rename armv7_dmb* functions to arm_dmb* because they apply to v7
+ and v8 (or more precisely; to ARMv7's ARM and Thumb2, as well as
+ ARMv8's aarch32 A32/T32 and aarch64).
+
+ Use inner-shareability domain for ARM barriers instead of
+ full-system. This is what C++ uses.
+
+ The default case for barriers simply used a compiler barrier. This
+ is generally wrong, e.g. for MIPS.
+
+ * wtf/Atomics.h:
+ (WTF::Atomic::compareExchangeWeak): offer two-order version
+ (WTF::Atomic::compareExchangeStrong): offer two-order version
+ (WTF::weakCompareAndSwap): a few assertions
+ (WTF::arm_dmb): rename since it applies to ARMv7 and v8; make it innser-shareable
+ (WTF::arm_dmb_st): rename since it applies to ARMv7 and v8; make it innser-shareable
+ (WTF::loadLoadFence): incorrect generally
+ (WTF::loadStoreFence): incorrect generally
+ (WTF::storeLoadFence): incorrect generally
+ (WTF::storeStoreFence): incorrect generally
+ (WTF::memoryBarrierAfterLock): incorrect generally
+ (WTF::memoryBarrierBeforeUnlock): incorrect generally
+ (WTF::armV7_dmb): Deleted.
+ (WTF::armV7_dmb_st): Deleted.
+
+2016-09-14 JF Bastien <[email protected]>
+
Alwasys inline atomic operations
https://bugs.webkit.org/show_bug.cgi?id=155371
Modified: trunk/Source/WTF/wtf/Atomics.h (205920 => 205921)
--- trunk/Source/WTF/wtf/Atomics.h 2016-09-14 18:46:32 UTC (rev 205920)
+++ trunk/Source/WTF/wtf/Atomics.h 2016-09-14 18:51:26 UTC (rev 205921)
@@ -67,6 +67,18 @@
return value.compare_exchange_weak(expectedOrActual, desired, order);
}
+ ALWAYS_INLINE bool compareExchangeWeak(T expected, T desired, std::memory_order order_success, std::memory_order order_failure)
+ {
+#if OS(WINDOWS)
+ // Windows makes strange assertions about the argument to compare_exchange_weak, and anyway,
+ // Windows is X86 so seq_cst is cheap.
+ order_success = std::memory_order_seq_cst;
+ order_failure = std::memory_order_seq_cst;
+#endif
+ T expectedOrActual = expected;
+ return value.compare_exchange_weak(expectedOrActual, desired, order_success, order_failure);
+ }
+
ALWAYS_INLINE bool compareExchangeStrong(T expected, T desired, std::memory_order order = std::memory_order_seq_cst)
{
#if OS(WINDOWS)
@@ -76,7 +88,18 @@
T expectedOrActual = expected;
return value.compare_exchange_strong(expectedOrActual, desired, order);
}
-
+
+ ALWAYS_INLINE bool compareExchangeStrong(T expected, T desired, std::memory_order order_success, std::memory_order order_failure)
+ {
+#if OS(WINDOWS)
+ // See above.
+ order_success = std::memory_order_seq_cst;
+ order_failure = std::memory_order_seq_cst;
+#endif
+ T expectedOrActual = expected;
+ return value.compare_exchange_strong(expectedOrActual, desired, order_success, order_failure);
+ }
+
template<typename U>
ALWAYS_INLINE T exchangeAndAdd(U addend, std::memory_order order = std::memory_order_seq_cst)
{
@@ -103,6 +126,8 @@
template<typename T>
inline bool weakCompareAndSwap(volatile T* location, T expected, T newValue)
{
+ ASSERT(isPointerTypeAlignmentOkay(location) && "natural alignment required");
+ ASSERT(bitwise_cast<std::atomic<T>*>(location)->is_lock_free() && "expected lock-free type");
return bitwise_cast<Atomic<T>*>(location)->compareExchangeWeak(expected, newValue, std::memory_order_relaxed);
}
@@ -122,23 +147,23 @@
// Full memory fence. No accesses will float above this, and no accesses will sink
// below it.
-inline void armV7_dmb()
+inline void arm_dmb()
{
- asm volatile("dmb sy" ::: "memory");
+ asm volatile("dmb ish" ::: "memory");
}
// Like the above, but only affects stores.
-inline void armV7_dmb_st()
+inline void arm_dmb_st()
{
- asm volatile("dmb st" ::: "memory");
+ asm volatile("dmb ishst" ::: "memory");
}
-inline void loadLoadFence() { armV7_dmb(); }
-inline void loadStoreFence() { armV7_dmb(); }
-inline void storeLoadFence() { armV7_dmb(); }
-inline void storeStoreFence() { armV7_dmb_st(); }
-inline void memoryBarrierAfterLock() { armV7_dmb(); }
-inline void memoryBarrierBeforeUnlock() { armV7_dmb(); }
+inline void loadLoadFence() { arm_dmb(); }
+inline void loadStoreFence() { arm_dmb(); }
+inline void storeLoadFence() { arm_dmb(); }
+inline void storeStoreFence() { arm_dmb_st(); }
+inline void memoryBarrierAfterLock() { arm_dmb(); }
+inline void memoryBarrierBeforeUnlock() { arm_dmb(); }
#elif CPU(X86) || CPU(X86_64)
@@ -164,12 +189,12 @@
#else
-inline void loadLoadFence() { compilerFence(); }
-inline void loadStoreFence() { compilerFence(); }
-inline void storeLoadFence() { compilerFence(); }
-inline void storeStoreFence() { compilerFence(); }
-inline void memoryBarrierAfterLock() { compilerFence(); }
-inline void memoryBarrierBeforeUnlock() { compilerFence(); }
+inline void loadLoadFence() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void loadStoreFence() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void storeLoadFence() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void storeStoreFence() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void memoryBarrierAfterLock() { std::atomic_thread_fence(std::memory_order_seq_cst); }
+inline void memoryBarrierBeforeUnlock() { std::atomic_thread_fence(std::memory_order_seq_cst); }
#endif