Diff
Modified: trunk/JSTests/ChangeLog (251874 => 251875)
--- trunk/JSTests/ChangeLog 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/JSTests/ChangeLog 2019-10-31 20:50:56 UTC (rev 251875)
@@ -1,3 +1,14 @@
+2019-10-31 Saam Barati <sbar...@apple.com>
+
+ Don't use memmove/memcpy/memset for memory that can be scanned concurrently
+ https://bugs.webkit.org/show_bug.cgi?id=203228
+ <rdar://problem/56401852>
+
+ Reviewed by Robin Morisset.
+
+ * stress/torn-js-value-concurrent-collector.js: Added.
+ (foo):
+
2019-10-30 Yusuke Suzuki <ysuz...@apple.com>
[JSC] Date functions should have intrinsic
Added: trunk/JSTests/stress/torn-js-value-concurrent-collector.js (0 => 251875)
--- trunk/JSTests/stress/torn-js-value-concurrent-collector.js (rev 0)
+++ trunk/JSTests/stress/torn-js-value-concurrent-collector.js 2019-10-31 20:50:56 UTC (rev 251875)
@@ -0,0 +1,18 @@
+const a = [];
+function bar(a0) {
+ let j = 0;
+ while (j++ < 1000) {
+ a.splice(a0, 1000, 0, {}, {}, {}, {});
+ }
+}
+
+function foo() {
+ for (let k = 0; k < 10; k++) {
+ bar(2**25);
+ }
+ bar();
+}
+
+for (let i = 0; i < 30; i++) {
+ foo();
+}
Modified: trunk/Source/_javascript_Core/CMakeLists.txt (251874 => 251875)
--- trunk/Source/_javascript_Core/CMakeLists.txt 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/CMakeLists.txt 2019-10-31 20:50:56 UTC (rev 251875)
@@ -577,6 +577,7 @@
heap/GCIncomingRefCountedInlines.h
heap/GCIncomingRefCountedSet.h
heap/GCLogging.h
+ heap/GCMemoryOperations.h
heap/GCRequest.h
heap/GCSegmentedArray.h
heap/Handle.h
Modified: trunk/Source/_javascript_Core/ChangeLog (251874 => 251875)
--- trunk/Source/_javascript_Core/ChangeLog 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/ChangeLog 2019-10-31 20:50:56 UTC (rev 251875)
@@ -1,3 +1,62 @@
+2019-10-31 Saam Barati <sbar...@apple.com>
+
+ Don't use memmove/memcpy/memset for memory that can be scanned concurrently
+ https://bugs.webkit.org/show_bug.cgi?id=203228
+ <rdar://problem/56401852>
+
+ Reviewed by Robin Morisset.
+
+ We had code inside various places of the runtime which would call into system
+ memcpy/memmove/memset when updating a live butterfly. This means that the
+ concurrent collector could be scanning such butterflies while a memcpy/memmove/memset
+ was running. Those functions don't guarantee anything about the minimum
+ alignment of the stores they do. And implementations for them frequently have
+ byte copy loops for low byte copy counts. This lead to us seeing torn JSValues
+ inside the concurrent collector during Array.prototype.splice. This patch
+ introduces new functions for doing memcpy/memmove/memset for data structures
+ which may be concurrently scanned. The loops are written using inline assembly
+ for gcc compatible compilers on 64 bit platforms. The inline assembly
+ ensures we never write to memory using instructions that store fewer
+ than 8 bytes. On other platforms, we just use a volatile pointer to
+ ensure the compiler doesn't turn the loop into a function call or a
+ series of stores which may be smaller than 8 bytes.
+
+ * CMakeLists.txt:
+ * _javascript_Core.xcodeproj/project.pbxproj:
+ * heap/GCMemoryOperations.h: Added.
+ (JSC::gcSafeMemcpy):
+ (JSC::gcSafeMemmove):
+ (JSC::gcSafeZeroMemory):
+ * heap/Heap.h:
+ * runtime/ArrayConventions.cpp:
+ (JSC::clearArrayMemset):
+ * runtime/ArrayPrototype.cpp:
+ (JSC::copyElements):
+ * runtime/ButterflyInlines.h:
+ (JSC::Butterfly::tryCreate):
+ (JSC::Butterfly::createOrGrowPropertyStorage):
+ (JSC::Butterfly::growArrayRight):
+ (JSC::Butterfly::reallocArrayRightIfPossible):
+ (JSC::Butterfly::resizeArray):
+ (JSC::Butterfly::unshift):
+ (JSC::Butterfly::shift):
+ * runtime/JSArray.cpp:
+ (JSC::JSArray::unshiftCountSlowCase):
+ (JSC::JSArray::appendMemcpy):
+ (JSC::JSArray::fastSlice):
+ (JSC::JSArray::shiftCountWithArrayStorage):
+ (JSC::JSArray::shiftCountWithAnyIndexingType):
+ (JSC::JSArray::unshiftCountWithArrayStorage):
+ * runtime/JSObject.cpp:
+ (JSC::JSObject::constructConvertedArrayStorageWithoutCopyingElements):
+ (JSC::JSObject::convertFromCopyOnWrite):
+ (JSC::JSObject::shiftButterflyAfterFlattening):
+ * runtime/JSObject.h:
+ * runtime/RegExpMatchesArray.h:
+ (JSC::createRegExpMatchesArray):
+ * runtime/Structure.cpp:
+ (JSC::Structure::flattenDictionaryStructure):
+
2019-10-31 Devin Rousso <drou...@apple.com>
Web Inspector: Debugger: make sure the blackbox config is removed before iterating all existing scripts
Modified: trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj (251874 => 251875)
--- trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2019-10-31 20:50:56 UTC (rev 251875)
@@ -890,6 +890,7 @@
4BAA07CEB81F49A296E02203 /* WasmSignatureInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 30A5F403F11C4F599CD596D5 /* WasmSignatureInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
521131F71F82BF14007CCEEE /* PolyProtoAccessChain.h in Headers */ = {isa = PBXBuildFile; fileRef = 521131F61F82BF11007CCEEE /* PolyProtoAccessChain.h */; settings = {ATTRIBUTES = (Private, ); }; };
521322461ECBCE8200F65615 /* WebAssemblyFunctionBase.h in Headers */ = {isa = PBXBuildFile; fileRef = 521322441ECBCE8200F65615 /* WebAssemblyFunctionBase.h */; };
+ 522927D5235FD0B9005CB169 /* GCMemoryOperations.h in Headers */ = {isa = PBXBuildFile; fileRef = 5272987B235FC8BA005C982C /* GCMemoryOperations.h */; settings = {ATTRIBUTES = (Private, ); }; };
523FD88E225566C9003B3DCC /* WebAssemblyFunctionHeapCellType.h in Headers */ = {isa = PBXBuildFile; fileRef = 523FD88C225566C3003B3DCC /* WebAssemblyFunctionHeapCellType.h */; };
524E9D7322092B5200A6BEEE /* AirAllocateRegistersAndStackAndGenerateCode.h in Headers */ = {isa = PBXBuildFile; fileRef = 524E9D7222092B4600A6BEEE /* AirAllocateRegistersAndStackAndGenerateCode.h */; };
5250D2D21E8DA05A0029A932 /* WasmThunks.h in Headers */ = {isa = PBXBuildFile; fileRef = 5250D2D01E8DA05A0029A932 /* WasmThunks.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -3535,6 +3536,7 @@
52678F901A04177C006A306D /* ControlFlowProfiler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ControlFlowProfiler.h; sourceTree = "<group>"; };
526AC4B41E977C5D003500E1 /* WasmCodeBlock.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = WasmCodeBlock.cpp; sourceTree = "<group>"; };
526AC4B51E977C5D003500E1 /* WasmCodeBlock.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WasmCodeBlock.h; sourceTree = "<group>"; };
+ 5272987B235FC8BA005C982C /* GCMemoryOperations.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GCMemoryOperations.h; sourceTree = "<group>"; };
527773DD1AAF83AC00BDE7E8 /* RuntimeType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RuntimeType.cpp; sourceTree = "<group>"; };
527CE35222555FDD00C6F382 /* JSToWasmICCallee.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = JSToWasmICCallee.cpp; path = js/JSToWasmICCallee.cpp; sourceTree = "<group>"; };
527CE35322555FDD00C6F382 /* JSToWasmICCallee.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = JSToWasmICCallee.h; path = js/JSToWasmICCallee.h; sourceTree = "<group>"; };
@@ -6134,6 +6136,7 @@
0F2B66AB17B6B53D00A7AE3F /* GCIncomingRefCountedSetInlines.h */,
2ADFA26218EF3540004F9FCC /* GCLogging.cpp */,
2AABCDE618EF294200002096 /* GCLogging.h */,
+ 5272987B235FC8BA005C982C /* GCMemoryOperations.h */,
0F97152E1EB28BE900A1645D /* GCRequest.cpp */,
0F97152F1EB28BE900A1645D /* GCRequest.h */,
2A343F7418A1748B0039B085 /* GCSegmentedArray.h */,
@@ -8898,6 +8901,7 @@
0F725CA81C503DED00AD943A /* B3EliminateCommonSubexpressions.h in Headers */,
3395C70722555F6D00BDBFAD /* B3EliminateDeadCode.h in Headers */,
0F5BF1711F23A5A10029D91D /* B3EnsureLoopPreHeaders.h in Headers */,
+ 522927D5235FD0B9005CB169 /* GCMemoryOperations.h in Headers */,
5318045C22EAAC4B004A7342 /* B3ExtractValue.h in Headers */,
0F6971EA1D92F42400BA02A5 /* B3FenceValue.h in Headers */,
0F6B8AE51C4EFE1700969052 /* B3FixSSA.h in Headers */,
Added: trunk/Source/_javascript_Core/heap/GCMemoryOperations.h (0 => 251875)
--- trunk/Source/_javascript_Core/heap/GCMemoryOperations.h (rev 0)
+++ trunk/Source/_javascript_Core/heap/GCMemoryOperations.h 2019-10-31 20:50:56 UTC (rev 251875)
@@ -0,0 +1,306 @@
+/*
+ * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "CPU.h"
+#include "JSCJSValue.h"
+
+namespace JSC {
+
+// We use these memory operations when modifying memory that might be scanned by the concurrent collector.
+// We don't call the default operations because they're not guaranteed to store to memory in eight byte aligned
+// chunks. If we happened to fall into the system's normal byte copy loop, we may see a torn JSValue in the
+// concurrent collector.
+
+constexpr size_t smallCutoff = 30 * 8;
+constexpr size_t mediumCutoff = 4 * 1024;
+
+// This is a forwards loop so gcSafeMemmove can rely on the direction.
+template <typename T>
+ALWAYS_INLINE void gcSafeMemcpy(T* dst, T* src, size_t bytes)
+{
+ static_assert(sizeof(T) == sizeof(JSValue));
+ RELEASE_ASSERT(bytes % 8 == 0);
+
+#if USE(JSVALUE64)
+
+ auto slowPathForwardMemcpy = [&] {
+ size_t count = bytes / 8;
+ for (unsigned i = 0; i < count; ++i)
+ bitwise_cast<volatile uint64_t*>(dst)[i] = bitwise_cast<volatile uint64_t*>(src)[i];
+ };
+
+#if COMPILER(GCC_COMPATIBLE) && USE(JSVALUE64)
+ if (bytes <= smallCutoff)
+ slowPathForwardMemcpy();
+ else if (isARM64() || bytes <= mediumCutoff) {
+#if CPU(X86_64)
+ size_t alignedBytes = (bytes / 64) * 64;
+ size_t tmp;
+ size_t offset = 0;
+ asm volatile(
+ ".balign 32\t\n"
+ "1:\t\n"
+ "cmpq %q[offset], %q[alignedBytes]\t\n"
+ "je 2f\t\n"
+ "movups (%q[src], %q[offset], 1), %%xmm0\t\n"
+ "movups 16(%q[src], %q[offset], 1), %%xmm1\t\n"
+ "movups 32(%q[src], %q[offset], 1), %%xmm2\t\n"
+ "movups 48(%q[src], %q[offset], 1), %%xmm3\t\n"
+ "movups %%xmm0, (%q[dst], %q[offset], 1)\t\n"
+ "movups %%xmm1, 16(%q[dst], %q[offset], 1)\t\n"
+ "movups %%xmm2, 32(%q[dst], %q[offset], 1)\t\n"
+ "movups %%xmm3, 48(%q[dst], %q[offset], 1)\t\n"
+ "addq $64, %q[offset]\t\n"
+ "jmp 1b\t\n"
+
+ "2:\t\n"
+ "cmpq %q[offset], %q[bytes]\t\n"
+ "je 3f\t\n"
+ "movq (%q[src], %q[offset], 1), %q[tmp]\t\n"
+ "movq %q[tmp], (%q[dst], %q[offset], 1)\t\n"
+ "addq $8, %q[offset]\t\n"
+ "jmp 2b\t\n"
+
+ "3:\t\n"
+
+ : [alignedBytes] "+r" (alignedBytes), [bytes] "+r" (bytes), [tmp] "+r" (tmp), [offset] "+r" (offset), [dst] "+r" (dst), [src] "+r" (src)
+ :
+ : "xmm0", "xmm1", "xmm2", "xmm3", "memory", "cc"
+ );
+#elif CPU(ARM64)
+ size_t alignedBytes = (bytes / 16) * 16;
+ size_t offset = 0;
+
+ asm volatile(
+ "1:\t\n"
+ "cmp %x[offset], %x[alignedBytes]\t\n"
+ "b.eq 2f\t\n"
+ "ldr q0, [%x[src], %x[offset]]\t\n"
+ "str q0, [%x[dst], %x[offset]]\t\n"
+ "add %x[offset], %x[offset], #0x10\t\n"
+ "b 1b\t\n"
+
+ "2:\t\n"
+ "cmp %x[offset], %x[bytes]\t\n"
+ "b.eq 3f\t\n"
+ "ldr d0, [%x[src], %x[offset]]\t\n"
+ "str d0, [%x[dst], %x[offset]]\t\n"
+ "add %x[offset], %x[offset], #0x8\t\n"
+ "b 2b\t\n"
+
+ "3:\t\n"
+
+ : [alignedBytes] "+r" (alignedBytes), [bytes] "+r" (bytes), [offset] "+r" (offset), [dst] "+r" (dst), [src] "+r" (src)
+ :
+ : "d0", "d1", "memory"
+ );
+#else
+#error "Unknown architecture."
+#endif // CPU(X86_64)
+ } else {
+ RELEASE_ASSERT(isX86_64());
+#if CPU(X86_64)
+ size_t count = bytes / 8;
+ asm volatile(
+ ".balign 16\t\n"
+ "cld\t\n"
+ "rep movsq\t\n"
+ : "+D" (dst), "+S" (src), "+c" (count)
+ :
+ : "memory");
+#endif // CPU(X86_64)
+ }
+#else
+ slowPathForwardMemcpy();
+#endif // COMPILER(GCC_COMPATIBLE)
+#else
+ memcpy(dst, src, bytes);
+#endif // USE(JSVALUE64)
+}
+
+template <typename T>
+ALWAYS_INLINE void gcSafeMemmove(T* dst, T* src, size_t bytes)
+{
+ static_assert(sizeof(T) == sizeof(JSValue));
+ RELEASE_ASSERT(bytes % 8 == 0);
+#if USE(JSVALUE64)
+ if (bitwise_cast<uintptr_t>(src) >= bitwise_cast<uintptr_t>(dst)) {
+ // This is written to do a forwards loop, so calling it is ok.
+ gcSafeMemcpy(dst, src, bytes);
+ return;
+ }
+
+ if ((bitwise_cast<uintptr_t>(src) + bytes) <= bitwise_cast<uintptr_t>(dst)) {
+ gcSafeMemcpy(dst, src, bytes);
+ return;
+ }
+
+#if COMPILER(GCC_COMPATIBLE)
+
+ auto slowPathBackwardsMemmove = [&] {
+ size_t count = bytes / 8;
+ for (size_t i = count; i--; )
+ bitwise_cast<volatile uint64_t*>(dst)[i] = bitwise_cast<volatile uint64_t*>(src)[i];
+ };
+
+ if (bytes <= smallCutoff)
+ slowPathBackwardsMemmove();
+ else {
+#if CPU(X86_64)
+ size_t alignedBytes = (bytes / 64) * 64;
+
+ size_t tail = alignedBytes;
+ size_t tmp;
+ asm volatile(
+ "2:\t\n"
+ "cmpq %q[tail], %q[bytes]\t\n"
+ "je 1f\t\n"
+ "addq $-8, %q[bytes]\t\n"
+ "movq (%q[src], %q[bytes], 1), %q[tmp]\t\n"
+ "movq %q[tmp], (%q[dst], %q[bytes], 1)\t\n"
+ "jmp 2b\t\n"
+
+ "1:\t\n"
+ "test %q[alignedBytes], %q[alignedBytes]\t\n"
+ "jz 3f\t\n"
+
+ ".balign 32\t\n"
+ "100:\t\n"
+
+ "movups -64(%q[src], %q[alignedBytes], 1), %%xmm0\t\n"
+ "movups -48(%q[src], %q[alignedBytes], 1), %%xmm1\t\n"
+ "movups -32(%q[src], %q[alignedBytes], 1), %%xmm2\t\n"
+ "movups -16(%q[src], %q[alignedBytes], 1), %%xmm3\t\n"
+ "movups %%xmm0, -64(%q[dst], %q[alignedBytes], 1)\t\n"
+ "movups %%xmm1, -48(%q[dst], %q[alignedBytes], 1)\t\n"
+ "movups %%xmm2, -32(%q[dst], %q[alignedBytes], 1)\t\n"
+ "movups %%xmm3, -16(%q[dst], %q[alignedBytes], 1)\t\n"
+ "addq $-64, %q[alignedBytes]\t\n"
+ "jnz 100b\t\n"
+
+ "3:\t\n"
+
+ : [alignedBytes] "+r" (alignedBytes), [tail] "+r" (tail), [bytes] "+r" (bytes), [tmp] "+r" (tmp), [dst] "+r" (dst), [src] "+r" (src)
+ :
+ : "xmm0", "xmm1", "xmm2", "xmm3", "memory", "cc"
+ );
+#elif CPU(ARM64)
+ size_t alignedBytes = (bytes / 16) * 16;
+
+ asm volatile(
+ "1:\t\n"
+ "cmp %x[alignedBytes], %x[bytes]\t\n"
+ "b.eq 2f\t\n"
+ "sub %x[bytes], %x[bytes], #0x8\t\n"
+ "ldr d0, [%x[src], %x[bytes]]\t\n"
+ "str d0, [%x[dst], %x[bytes]]\t\n"
+ "b 1b\t\n"
+
+ "2:\t\n"
+ "cbz %x[alignedBytes], 3f\t\n"
+ "sub %x[alignedBytes], %x[alignedBytes], #0x10\t\n"
+ "ldr q0, [%x[src], %x[alignedBytes]]\t\n"
+ "str q0, [%x[dst], %x[alignedBytes]]\t\n"
+ "b 2b\t\n"
+
+ "3:\t\n"
+
+ : [alignedBytes] "+r" (alignedBytes), [bytes] "+r" (bytes), [dst] "+r" (dst), [src] "+r" (src)
+ :
+ : "d0", "d1", "memory"
+ );
+#else
+#error "Unknown architecture."
+#endif // CPU(X86_64)
+ }
+#else
+ slowPathBackwardsMemmove();
+#endif // COMPILER(GCC_COMPATIBLE)
+#else
+ memmove(dst, src, bytes);
+#endif // USE(JSVALUE64)
+}
+
+template <typename T>
+ALWAYS_INLINE void gcSafeZeroMemory(T* dst, size_t bytes)
+{
+ static_assert(sizeof(T) == sizeof(JSValue));
+ RELEASE_ASSERT(bytes % 8 == 0);
+#if USE(JSVALUE64)
+#if COMPILER(GCC_COMPATIBLE)
+#if CPU(X86_64)
+ uint64_t zero = 0;
+ size_t count = bytes / 8;
+ asm volatile (
+ "rep stosq\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+#elif CPU(ARM64)
+ size_t alignedBytes = (bytes / 64) * 64;
+ char* end = bitwise_cast<char*>(dst) + bytes;
+ char* alignedEnd = bitwise_cast<char*>(dst) + alignedBytes;
+ asm volatile(
+ "movi d0, #0\t\n"
+ "movi d1, #0\t\n"
+
+ ".p2align 4\t\n"
+ "2:\t\n"
+ "cmp %x[dst], %x[alignedEnd]\t\n"
+ "b.eq 4f\t\n"
+ "stnp q0, q0, [%x[dst]]\t\n"
+ "stnp q0, q0, [%x[dst], #0x20]\t\n"
+ "add %x[dst], %x[dst], #0x40\t\n"
+ "b 2b\t\n"
+
+ "4:\t\n"
+ "cmp %x[dst], %x[end]\t\n"
+ "b.eq 5f\t\n"
+ "str d0, [%x[dst]], #0x10\t\n"
+ "b 4b\t\n"
+
+ "5:\t\n"
+
+ : [alignedBytes] "+r" (alignedBytes), [bytes] "+r" (bytes), [dst] "+r" (dst), [end] "+r" (end), [alignedEnd] "+r" (alignedEnd)
+ :
+ : "d0", "d1", "memory"
+ );
+#else
+#error "Unknown architecture."
+#endif // CPU(X86_64)
+#else
+ size_t count = bytes / 8;
+ for (size_t i = 0; i < count; ++i)
+ bitwise_cast<volatile uint64_t*>(dst)[i] = 0;
+#endif // COMPILER(GCC_COMPATIBLE)
+#else
+ memset(dst, 0, bytes);
+#endif // USE(JSVALUE64)
+}
+
+} // namespace JSC
Modified: trunk/Source/_javascript_Core/heap/Heap.h (251874 => 251875)
--- trunk/Source/_javascript_Core/heap/Heap.h 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/heap/Heap.h 2019-10-31 20:50:56 UTC (rev 251875)
@@ -28,6 +28,7 @@
#include "DeleteAllCodeEffort.h"
#include "GCConductor.h"
#include "GCIncomingRefCountedSet.h"
+#include "GCMemoryOperations.h"
#include "GCRequest.h"
#include "HandleSet.h"
#include "HeapFinalizerCallback.h"
Modified: trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp 2019-10-31 20:50:56 UTC (rev 251875)
@@ -33,17 +33,7 @@
#if USE(JSVALUE64)
void clearArrayMemset(WriteBarrier<Unknown>* base, unsigned count)
{
-#if CPU(X86_64) && COMPILER(GCC_COMPATIBLE)
- uint64_t zero = 0;
- asm volatile (
- "rep stosq\n\t"
- : "+D"(base), "+c"(count)
- : "a"(zero)
- : "memory"
- );
-#else // not CPU(X86_64)
- memset(base, 0, count * sizeof(WriteBarrier<Unknown>));
-#endif // generic CPU
+ gcSafeZeroMemory(base, count * sizeof(WriteBarrier<Unknown>));
}
void clearArrayMemset(double* base, unsigned count)
Modified: trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp 2019-10-31 20:50:56 UTC (rev 251875)
@@ -1487,10 +1487,10 @@
}
template<typename T>
-ALWAYS_INLINE void copyElements(T* buffer, unsigned offset, void* source, unsigned sourceSize, IndexingType sourceType)
+ALWAYS_INLINE void copyElements(T* buffer, unsigned offset, T* source, unsigned sourceSize, IndexingType sourceType)
{
if (sourceType != ArrayWithUndecided) {
- memcpy(buffer + offset, source, sizeof(JSValue) * sourceSize);
+ gcSafeMemcpy(buffer + offset, source, sizeof(JSValue) * sourceSize);
return;
}
Modified: trunk/Source/_javascript_Core/runtime/ButterflyInlines.h (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/ButterflyInlines.h 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/ButterflyInlines.h 2019-10-31 20:50:56 UTC (rev 251875)
@@ -104,7 +104,7 @@
Butterfly* result = fromBase(base, preCapacity, propertyCapacity);
if (hasIndexingHeader)
*result->indexingHeader() = indexingHeader;
- memset(result->propertyStorage() - propertyCapacity, 0, propertyCapacity * sizeof(EncodedJSValue));
+ gcSafeZeroMemory(result->propertyStorage() - propertyCapacity, propertyCapacity * sizeof(EncodedJSValue));
return result;
}
@@ -139,13 +139,12 @@
size_t indexingPayloadSizeInBytes = oldButterfly->indexingHeader()->indexingPayloadSizeInBytes(structure);
bool hasIndexingHeader = structure->hasIndexingHeader(intendedOwner);
Butterfly* result = createUninitialized(vm, intendedOwner, preCapacity, newPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes);
- memcpy(
+ gcSafeMemcpy(
result->propertyStorage() - oldPropertyCapacity,
oldButterfly->propertyStorage() - oldPropertyCapacity,
totalSize(0, oldPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes));
- memset(
+ gcSafeZeroMemory(
result->propertyStorage() - newPropertyCapacity,
- 0,
(newPropertyCapacity - oldPropertyCapacity) * sizeof(EncodedJSValue));
return result;
}
@@ -179,7 +178,7 @@
if (!newBase)
return nullptr;
// FIXME: This probably shouldn't be a memcpy.
- memcpy(newBase, theBase, oldSize);
+ gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(theBase), oldSize);
return fromBase(newBase, 0, propertyCapacity);
}
@@ -221,7 +220,7 @@
void* newBase = vm.jsValueGigacageAuxiliarySpace.allocateNonVirtual(vm, newSize, &deferralContext, AllocationFailureMode::ReturnNull);
if (!newBase)
return nullptr;
- memcpy(newBase, theBase, oldSize);
+ gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(theBase), oldSize);
return fromBase(newBase, 0, propertyCapacity);
}
@@ -238,7 +237,7 @@
size_t size = std::min(
totalSize(0, propertyCapacity, oldHasIndexingHeader, oldIndexingPayloadSizeInBytes),
totalSize(0, propertyCapacity, newHasIndexingHeader, newIndexingPayloadSizeInBytes));
- memcpy(to, from, size);
+ gcSafeMemcpy(static_cast<JSValue*>(to), static_cast<JSValue*>(from), size);
return result;
}
@@ -265,7 +264,7 @@
// inline memmove (particularly since the size argument is likely to be variable), nor can
// we rely on the compiler to recognize the ordering of the pointer arguments (since
// propertyCapacity is variable and could cause wrap-around as far as the compiler knows).
- memmove(
+ gcSafeMemmove(
propertyStorage() - numberOfSlots - propertyCapacity,
propertyStorage() - propertyCapacity,
sizeof(EncodedJSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
@@ -277,7 +276,7 @@
ASSERT(hasAnyArrayStorage(structure->indexingType()));
unsigned propertyCapacity = structure->outOfLineCapacity();
// FIXME: See comment in unshift(), above.
- memmove(
+ gcSafeMemmove(
propertyStorage() - propertyCapacity + numberOfSlots,
propertyStorage() - propertyCapacity,
sizeof(EncodedJSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
Modified: trunk/Source/_javascript_Core/runtime/JSArray.cpp (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/JSArray.cpp 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/JSArray.cpp 2019-10-31 20:50:56 UTC (rev 251875)
@@ -421,11 +421,11 @@
if (addToFront) {
ASSERT(count + usedVectorLength <= newVectorLength);
- memmove(newButterfly->arrayStorage()->m_vector + count, storage->m_vector, sizeof(JSValue) * usedVectorLength);
- memmove(newButterfly->propertyStorage() - propertySize, butterfly->propertyStorage() - propertySize, sizeof(JSValue) * propertySize + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
+ gcSafeMemmove(newButterfly->arrayStorage()->m_vector + count, storage->m_vector, sizeof(JSValue) * usedVectorLength);
+ gcSafeMemmove(newButterfly->propertyStorage() - propertySize, butterfly->propertyStorage() - propertySize, sizeof(JSValue) * propertySize + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
// We don't need to zero the pre-capacity for the concurrent GC because it is not available to use as property storage.
- memset(newButterfly->base(0, propertyCapacity), 0, (propertyCapacity - propertySize) * sizeof(JSValue));
+ gcSafeZeroMemory(static_cast<JSValue*>(newButterfly->base(0, propertyCapacity)), (propertyCapacity - propertySize) * sizeof(JSValue));
if (allocatedNewStorage) {
// We will set the vectorLength to newVectorLength. We populated requiredVectorLength
@@ -434,8 +434,8 @@
newButterfly->arrayStorage()->m_vector[i].clear();
}
} else if ((newAllocBase != butterfly->base(structure)) || (preCapacity != storage->m_indexBias)) {
- memmove(newButterfly->propertyStorage() - propertyCapacity, butterfly->propertyStorage() - propertyCapacity, sizeof(JSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
- memmove(newButterfly->arrayStorage()->m_vector, storage->m_vector, sizeof(JSValue) * usedVectorLength);
+ gcSafeMemmove(newButterfly->propertyStorage() - propertyCapacity, butterfly->propertyStorage() - propertyCapacity, sizeof(JSValue) * propertyCapacity + sizeof(IndexingHeader) + ArrayStorage::sizeFor(0));
+ gcSafeMemmove(newButterfly->arrayStorage()->m_vector, storage->m_vector, sizeof(JSValue) * usedVectorLength);
for (unsigned i = requiredVectorLength; i < newVectorLength; i++)
newButterfly->arrayStorage()->m_vector[i].clear();
@@ -569,9 +569,9 @@
butterfly->contiguousInt32().at(this, i).setWithoutWriteBarrier(JSValue());
}
} else if (type == ArrayWithDouble)
- memcpy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), sizeof(JSValue) * otherLength);
+ gcSafeMemcpy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), sizeof(JSValue) * otherLength);
else {
- memcpy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), sizeof(JSValue) * otherLength);
+ gcSafeMemcpy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), sizeof(JSValue) * otherLength);
vm.heap.writeBarrier(this);
}
@@ -789,9 +789,9 @@
auto& resultButterfly = *resultArray->butterfly();
if (arrayType == ArrayWithDouble)
- memcpy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, sizeof(JSValue) * count);
+ gcSafeMemcpy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, sizeof(JSValue) * count);
else
- memcpy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, sizeof(JSValue) * count);
+ gcSafeMemcpy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, sizeof(JSValue) * count);
ASSERT(resultButterfly.publicLength() == count);
return resultArray;
@@ -849,7 +849,7 @@
// after the shift region, so we move the elements before to the right.
if (numElementsBeforeShiftRegion) {
RELEASE_ASSERT(count + startIndex <= vectorLength);
- memmove(storage->m_vector + count,
+ gcSafeMemmove(storage->m_vector + count,
storage->m_vector,
sizeof(JSValue) * startIndex);
}
@@ -867,7 +867,7 @@
} else {
// The number of elements before the shift region is greater than or equal to the number
// of elements after the shift region, so we move the elements after the shift region to the left.
- memmove(storage->m_vector + startIndex,
+ gcSafeMemmove(storage->m_vector + startIndex,
storage->m_vector + firstIndexAfterShiftRegion,
sizeof(JSValue) * numElementsAfterShiftRegion);
@@ -927,7 +927,7 @@
butterfly->contiguous().at(this, i).setWithoutWriteBarrier(v);
}
} else {
- memmove(butterfly->contiguous().data() + startIndex,
+ gcSafeMemmove(butterfly->contiguous().data() + startIndex,
butterfly->contiguous().data() + startIndex + count,
sizeof(JSValue) * (end - startIndex));
}
@@ -969,7 +969,7 @@
butterfly->contiguousDouble().at(this, i) = v;
}
} else {
- memmove(butterfly->contiguousDouble().data() + startIndex,
+ gcSafeMemmove(butterfly->contiguousDouble().data() + startIndex,
butterfly->contiguousDouble().data() + startIndex + count,
sizeof(JSValue) * (end - startIndex));
}
@@ -1033,9 +1033,9 @@
if (startIndex) {
if (moveFront)
- memmove(vector, vector + count, startIndex * sizeof(JSValue));
+ gcSafeMemmove(vector, vector + count, startIndex * sizeof(JSValue));
else if (length - startIndex)
- memmove(vector + startIndex + count, vector + startIndex, (length - startIndex) * sizeof(JSValue));
+ gcSafeMemmove(vector + startIndex + count, vector + startIndex, (length - startIndex) * sizeof(JSValue));
}
for (unsigned i = 0; i < count; i++)
Modified: trunk/Source/_javascript_Core/runtime/JSObject.cpp (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/JSObject.cpp 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/JSObject.cpp 2019-10-31 20:50:56 UTC (rev 251875)
@@ -1210,9 +1210,9 @@
Butterfly* newButterfly = Butterfly::createUninitialized(vm, this, 0, propertyCapacity, true, ArrayStorage::sizeFor(neededLength));
- memcpy(
- newButterfly->base(0, propertyCapacity),
- m_butterfly->base(0, propertyCapacity),
+ gcSafeMemcpy(
+ static_cast<JSValue*>(newButterfly->base(0, propertyCapacity)),
+ static_cast<JSValue*>(m_butterfly->base(0, propertyCapacity)),
propertyCapacity * sizeof(EncodedJSValue));
ArrayStorage* newStorage = newButterfly->arrayStorage();
@@ -1503,7 +1503,7 @@
unsigned newVectorLength = Butterfly::optimalContiguousVectorLength(propertyCapacity, std::min(oldButterfly->vectorLength() * 2, MAX_STORAGE_VECTOR_LENGTH));
Butterfly* newButterfly = Butterfly::createUninitialized(vm, this, 0, propertyCapacity, hasIndexingHeader, newVectorLength * sizeof(JSValue));
- memcpy(newButterfly->propertyStorage(), oldButterfly->propertyStorage(), oldButterfly->vectorLength() * sizeof(JSValue) + sizeof(IndexingHeader));
+ gcSafeMemcpy(newButterfly->propertyStorage(), oldButterfly->propertyStorage(), oldButterfly->vectorLength() * sizeof(JSValue) + sizeof(IndexingHeader));
WTF::storeStoreFence();
NonPropertyTransition transition = ([&] () {
@@ -3782,7 +3782,7 @@
void* currentBase = oldButterfly->base(0, outOfLineCapacityAfter);
void* newBase = newButterfly->base(0, outOfLineCapacityAfter);
- memcpy(newBase, currentBase, Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes));
+ gcSafeMemcpy(static_cast<JSValue*>(newBase), static_cast<JSValue*>(currentBase), Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes));
setButterfly(vm, newButterfly);
}
Modified: trunk/Source/_javascript_Core/runtime/JSObject.h (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/JSObject.h 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/JSObject.h 2019-10-31 20:50:56 UTC (rev 251875)
@@ -1180,7 +1180,7 @@
explicit JSFinalObject(VM& vm, Structure* structure, Butterfly* butterfly = nullptr)
: JSObject(vm, structure, butterfly)
{
- memset(inlineStorageUnsafe(), 0, structure->inlineCapacity() * sizeof(EncodedJSValue));
+ gcSafeZeroMemory(inlineStorageUnsafe(), structure->inlineCapacity() * sizeof(EncodedJSValue));
}
};
Modified: trunk/Source/_javascript_Core/runtime/RegExpMatchesArray.h (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/RegExpMatchesArray.h 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/RegExpMatchesArray.h 2019-10-31 20:50:56 UTC (rev 251875)
@@ -98,7 +98,7 @@
ASSERT(!array->butterfly()->indexingHeader()->preCapacity(matchStructure));
auto capacity = matchStructure->outOfLineCapacity();
auto size = matchStructure->outOfLineSize();
- memset(array->butterfly()->base(0, capacity), 0, (capacity - size) * sizeof(JSValue));
+ gcSafeZeroMemory(static_cast<JSValue*>(array->butterfly()->base(0, capacity)), (capacity - size) * sizeof(JSValue));
};
if (UNLIKELY(globalObject->isHavingABadTime())) {
Modified: trunk/Source/_javascript_Core/runtime/Structure.cpp (251874 => 251875)
--- trunk/Source/_javascript_Core/runtime/Structure.cpp 2019-10-31 20:41:56 UTC (rev 251874)
+++ trunk/Source/_javascript_Core/runtime/Structure.cpp 2019-10-31 20:50:56 UTC (rev 251875)
@@ -777,9 +777,8 @@
// We need to zero our unused property space; otherwise the GC might see a
// stale pointer when we add properties in the future.
- memset(
+ gcSafeZeroMemory(
object->inlineStorageUnsafe() + inlineSize(),
- 0,
(inlineCapacity() - inlineSize()) * sizeof(EncodedJSValue));
Butterfly* butterfly = object->butterfly();
@@ -786,7 +785,7 @@
size_t preCapacity = butterfly->indexingHeader()->preCapacity(this);
void* base = butterfly->base(preCapacity, beforeOutOfLineCapacity);
void* startOfPropertyStorageSlots = reinterpret_cast<EncodedJSValue*>(base) + preCapacity;
- memset(startOfPropertyStorageSlots, 0, (beforeOutOfLineCapacity - outOfLineSize()) * sizeof(EncodedJSValue));
+ gcSafeZeroMemory(static_cast<JSValue*>(startOfPropertyStorageSlots), (beforeOutOfLineCapacity - outOfLineSize()) * sizeof(EncodedJSValue));
checkOffsetConsistency();
}
