Diff
Modified: trunk/Source/_javascript_Core/ChangeLog (228305 => 228306)
--- trunk/Source/_javascript_Core/ChangeLog 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/ChangeLog 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,3 +1,46 @@
+2018-02-08 Filip Pizlo <[email protected]>
+
+ Experiment with alternative implementation of memcpy/memset
+ https://bugs.webkit.org/show_bug.cgi?id=182563
+
+ Reviewed by Michael Saboff and Mark Lam.
+
+ This adopts new fastCopy/fastZeroFill calls for calls to memcpy/memset that do not take a
+ constant size argument.
+
+ * assembler/AssemblerBuffer.h:
+ (JSC::AssemblerBuffer::append):
+ * runtime/ArrayBuffer.cpp:
+ (JSC::ArrayBufferContents::tryAllocate):
+ (JSC::ArrayBufferContents::copyTo):
+ (JSC::ArrayBuffer::createInternal):
+ * runtime/ArrayBufferView.h:
+ (JSC::ArrayBufferView::zeroRangeImpl):
+ * runtime/ArrayConventions.cpp:
+ * runtime/ArrayConventions.h:
+ (JSC::clearArray):
+ * runtime/ArrayPrototype.cpp:
+ (JSC::arrayProtoPrivateFuncConcatMemcpy):
+ * runtime/ButterflyInlines.h:
+ (JSC::Butterfly::tryCreate):
+ (JSC::Butterfly::createOrGrowPropertyStorage):
+ (JSC::Butterfly::growArrayRight):
+ (JSC::Butterfly::resizeArray):
+ * runtime/GenericTypedArrayViewInlines.h:
+ (JSC::GenericTypedArrayView<Adaptor>::create):
+ * runtime/JSArray.cpp:
+ (JSC::JSArray::appendMemcpy):
+ (JSC::JSArray::fastSlice):
+ * runtime/JSArrayBufferView.cpp:
+ (JSC::JSArrayBufferView::ConstructionContext::ConstructionContext):
+ * runtime/JSGenericTypedArrayViewInlines.h:
+ (JSC::JSGenericTypedArrayView<Adaptor>::set):
+ * runtime/JSObject.cpp:
+ (JSC::JSObject::constructConvertedArrayStorageWithoutCopyingElements):
+ (JSC::JSObject::shiftButterflyAfterFlattening):
+ * runtime/PropertyTable.cpp:
+ (JSC::PropertyTable::PropertyTable):
+
2018-02-08 Don Olmstead <[email protected]>
Remove _javascript_Core/ForwardingHeaders directory
Modified: trunk/Source/_javascript_Core/assembler/AssemblerBuffer.h (228305 => 228306)
--- trunk/Source/_javascript_Core/assembler/AssemblerBuffer.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/assembler/AssemblerBuffer.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2008, 2012, 2014 Apple Inc. All rights reserved.
+ * Copyright (C) 2008-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -276,7 +276,7 @@
if (!isAvailable(size))
grow(size);
- memcpy(m_storage.buffer() + m_index, data, size);
+ fastCopyBytes(m_storage.buffer() + m_index, data, size);
m_index += size;
}
Modified: trunk/Source/_javascript_Core/heap/LargeAllocation.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/heap/LargeAllocation.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/heap/LargeAllocation.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -45,7 +45,7 @@
return nullptr;
// Make sure that the padding does not contain useful things.
- memset(static_cast<char*>(space) + sizeBeforeDistancing, 0, distancing);
+ fastZeroFillBytes(static_cast<char*>(space) + sizeBeforeDistancing, distancing);
if (scribbleFreeCells())
scribble(space, size);
Modified: trunk/Source/_javascript_Core/heap/MarkedBlock.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/heap/MarkedBlock.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/heap/MarkedBlock.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -493,7 +493,7 @@
if (m_securityOriginToken == securityOriginToken)
return;
- memset(&block(), 0, endAtom * atomSize);
+ fastZeroFillBytes(&block(), endAtom * atomSize);
m_securityOriginToken = securityOriginToken;
}
Modified: trunk/Source/_javascript_Core/runtime/ArrayBuffer.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ArrayBuffer.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ArrayBuffer.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -113,7 +113,7 @@
}
if (policy == ZeroInitialize)
- memset(m_data.get(), 0, size);
+ fastZeroFillBytes(m_data.get(), size);
m_sizeInBytes = numElements * elementByteSize;
m_destructor = [] (void* p) { Gigacage::free(Gigacage::Primitive, p); };
@@ -141,7 +141,7 @@
other.tryAllocate(m_sizeInBytes, sizeof(char), ArrayBufferContents::DontInitialize);
if (!other.m_data)
return;
- memcpy(other.m_data.get(), m_data.get(), m_sizeInBytes);
+ fastCopyBytes(other.m_data.get(), m_data.get(), m_sizeInBytes);
other.m_sizeInBytes = m_sizeInBytes;
}
@@ -246,7 +246,7 @@
{
ASSERT(!byteLength || source);
auto buffer = adoptRef(*new ArrayBuffer(WTFMove(contents)));
- memcpy(buffer->data(), source, byteLength);
+ fastCopyBytes(buffer->data(), source, byteLength);
return buffer;
}
Modified: trunk/Source/_javascript_Core/runtime/ArrayBufferView.h (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ArrayBufferView.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ArrayBufferView.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -215,7 +215,7 @@
}
uint8_t* base = static_cast<uint8_t*>(baseAddress());
- memset(base + byteOffset, 0, rangeByteLength);
+ fastZeroFillBytes(base + byteOffset, rangeByteLength);
return true;
}
Modified: trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ArrayConventions.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -31,21 +31,6 @@
namespace JSC {
#if USE(JSVALUE64)
-void clearArrayMemset(WriteBarrier<Unknown>* base, unsigned count)
-{
-#if CPU(X86_64) && COMPILER(GCC_OR_CLANG)
- uint64_t zero = 0;
- asm volatile (
- "rep stosq\n\t"
- : "+D"(base), "+c"(count)
- : "a"(zero)
- : "memory"
- );
-#else // not CPU(X86_64)
- memset(base, 0, count * sizeof(WriteBarrier<Unknown>));
-#endif // generic CPU
-}
-
void clearArrayMemset(double* base, unsigned count)
{
#if CPU(X86_64) && COMPILER(GCC_OR_CLANG)
Modified: trunk/Source/_javascript_Core/runtime/ArrayConventions.h (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ArrayConventions.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ArrayConventions.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -117,7 +117,6 @@
}
#if USE(JSVALUE64)
-JS_EXPORT_PRIVATE void clearArrayMemset(WriteBarrier<Unknown>* base, unsigned count);
JS_EXPORT_PRIVATE void clearArrayMemset(double* base, unsigned count);
#endif // USE(JSVALUE64)
@@ -124,15 +123,11 @@
ALWAYS_INLINE void clearArray(WriteBarrier<Unknown>* base, unsigned count)
{
#if USE(JSVALUE64)
- const unsigned minCountForMemset = 100;
- if (count >= minCountForMemset) {
- clearArrayMemset(base, count);
- return;
- }
-#endif
-
+ fastZeroFill(base, count);
+#else
for (unsigned i = count; i--;)
base[i].clear();
+#endif
}
ALWAYS_INLINE void clearArray(double* base, unsigned count)
Modified: trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ArrayPrototype.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1341,19 +1341,18 @@
if (type == ArrayWithDouble) {
double* buffer = result->butterfly()->contiguousDouble().data();
- memcpy(buffer, firstButterfly->contiguousDouble().data(), sizeof(JSValue) * firstArraySize);
- memcpy(buffer + firstArraySize, secondButterfly->contiguousDouble().data(), sizeof(JSValue) * secondArraySize);
+ fastCopy(buffer, firstButterfly->contiguousDouble().data(), firstArraySize);
+ fastCopy(buffer + firstArraySize, secondButterfly->contiguousDouble().data(), secondArraySize);
} else if (type != ArrayWithUndecided) {
WriteBarrier<Unknown>* buffer = result->butterfly()->contiguous().data();
auto copy = [&] (unsigned offset, void* source, unsigned size, IndexingType type) {
if (type != ArrayWithUndecided) {
- memcpy(buffer + offset, source, sizeof(JSValue) * size);
+ fastCopy(buffer + offset, static_cast<WriteBarrier<Unknown>*>(source), size);
return;
}
- for (unsigned i = size; i--;)
- buffer[i + offset].clear();
+ clearArray(buffer + offset, size);
};
copy(0, firstButterfly->contiguous().data(), firstArraySize, firstType);
Modified: trunk/Source/_javascript_Core/runtime/ButterflyInlines.h (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/ButterflyInlines.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/ButterflyInlines.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2012-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -93,7 +93,7 @@
Butterfly* result = fromBase(base, preCapacity, propertyCapacity);
if (hasIndexingHeader)
*result->indexingHeader() = indexingHeader;
- memset(result->propertyStorage() - propertyCapacity, 0, propertyCapacity * sizeof(EncodedJSValue));
+ fastZeroFill(result->propertyStorage() - propertyCapacity, propertyCapacity);
return result;
}
@@ -129,14 +129,13 @@
bool hasIndexingHeader = structure->hasIndexingHeader(intendedOwner);
Butterfly* result = createUninitialized(
vm, intendedOwner, preCapacity, newPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes);
- memcpy(
+ fastCopyBytes(
result->propertyStorage() - oldPropertyCapacity,
oldButterfly->propertyStorage() - oldPropertyCapacity,
totalSize(0, oldPropertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes));
- memset(
+ fastZeroFill(
result->propertyStorage() - newPropertyCapacity,
- 0,
- (newPropertyCapacity - oldPropertyCapacity) * sizeof(EncodedJSValue));
+ newPropertyCapacity - oldPropertyCapacity);
return result;
}
@@ -168,8 +167,7 @@
void* newBase = vm.jsValueGigacageAuxiliarySpace.allocateNonVirtual(vm, newSize, nullptr, AllocationFailureMode::ReturnNull);
if (!newBase)
return nullptr;
- // FIXME: This probably shouldn't be a memcpy.
- memcpy(newBase, theBase, oldSize);
+ fastCopyBytes(newBase, theBase, oldSize);
return fromBase(newBase, 0, propertyCapacity);
}
@@ -199,7 +197,7 @@
size_t size = std::min(
totalSize(0, propertyCapacity, oldHasIndexingHeader, oldIndexingPayloadSizeInBytes),
totalSize(0, propertyCapacity, newHasIndexingHeader, newIndexingPayloadSizeInBytes));
- memcpy(to, from, size);
+ fastCopyBytes(to, from, size);
return result;
}
Modified: trunk/Source/_javascript_Core/runtime/GenericTypedArrayViewInlines.h (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/GenericTypedArrayViewInlines.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/GenericTypedArrayViewInlines.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2013, 2016 Apple Inc. All rights reserved.
+ * Copyright (C) 2013-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -52,7 +52,7 @@
const typename Adaptor::Type* array, unsigned length)
{
RefPtr<GenericTypedArrayView> result = create(length);
- memcpy(result->data(), array, length * sizeof(typename Adaptor::Type));
+ fastCopy(result->data(), array, length);
return result;
}
Modified: trunk/Source/_javascript_Core/runtime/JSArray.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/JSArray.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/JSArray.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -553,9 +553,9 @@
butterfly->contiguousInt32().at(this, i).setWithoutWriteBarrier(JSValue());
}
} else if (type == ArrayWithDouble)
- memcpy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), sizeof(JSValue) * otherLength);
+ fastCopy(butterfly()->contiguousDouble().data() + startIndex, otherArray->butterfly()->contiguousDouble().data(), otherLength);
else
- memcpy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), sizeof(JSValue) * otherLength);
+ fastCopy(butterfly()->contiguous().data() + startIndex, otherArray->butterfly()->contiguous().data(), otherLength);
return true;
}
@@ -761,9 +761,9 @@
auto& resultButterfly = *resultArray->butterfly();
if (arrayType == ArrayWithDouble)
- memcpy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, sizeof(JSValue) * count);
+ fastCopy(resultButterfly.contiguousDouble().data(), butterfly()->contiguousDouble().data() + startIndex, count);
else
- memcpy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, sizeof(JSValue) * count);
+ fastCopy(resultButterfly.contiguous().data(), butterfly()->contiguous().data() + startIndex, count);
resultButterfly.setPublicLength(count);
return resultArray;
Modified: trunk/Source/_javascript_Core/runtime/JSArrayBufferView.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/JSArrayBufferView.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/JSArrayBufferView.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -94,7 +94,7 @@
if (!m_vector)
return;
if (mode == ZeroFill)
- memset(m_vector.get(), 0, size);
+ fastZeroFillBytes(m_vector.get(), size);
vm.heap.reportExtraMemoryAllocated(static_cast<size_t>(length) * elementSize);
Modified: trunk/Source/_javascript_Core/runtime/JSGenericTypedArrayViewInlines.h (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/JSGenericTypedArrayViewInlines.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/JSGenericTypedArrayViewInlines.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -246,7 +246,7 @@
const ClassInfo* ci = object->classInfo(vm);
if (ci->typedArrayStorageType == Adaptor::typeValue) {
- // The super fast case: we can just memcpy since we're the same type.
+ // The super fast case: we can just memmove since we're the same type.
JSGenericTypedArrayView* other = jsCast<JSGenericTypedArrayView*>(object);
length = std::min(length, other->length());
Modified: trunk/Source/_javascript_Core/runtime/JSObject.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/JSObject.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/JSObject.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1178,10 +1178,10 @@
Butterfly* newButterfly = Butterfly::createUninitialized(
vm, this, 0, propertyCapacity, true, ArrayStorage::sizeFor(neededLength));
- memcpy(
+ fastCopy(
newButterfly->propertyStorage() - propertySize,
m_butterfly->propertyStorage() - propertySize,
- propertySize * sizeof(EncodedJSValue));
+ propertySize);
ArrayStorage* newStorage = newButterfly->arrayStorage();
newStorage->setVectorLength(neededLength);
@@ -3580,7 +3580,7 @@
void* currentBase = oldButterfly->base(0, outOfLineCapacityAfter);
void* newBase = newButterfly->base(0, outOfLineCapacityAfter);
- memcpy(newBase, currentBase, Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes));
+ fastCopyBytes(newBase, currentBase, Butterfly::totalSize(0, outOfLineCapacityAfter, hasIndexingHeader, indexingPayloadSizeInBytes));
setButterfly(vm, newButterfly);
}
Modified: trunk/Source/_javascript_Core/runtime/PropertyTable.cpp (228305 => 228306)
--- trunk/Source/_javascript_Core/runtime/PropertyTable.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/_javascript_Core/runtime/PropertyTable.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -74,7 +74,7 @@
{
ASSERT(isPowerOf2(m_indexSize));
- memcpy(m_index, other.m_index, dataSize());
+ fastCopyBytes(m_index, other.m_index, dataSize());
iterator end = this->end();
for (iterator iter = begin(); iter != end; ++iter)
Modified: trunk/Source/WTF/ChangeLog (228305 => 228306)
--- trunk/Source/WTF/ChangeLog 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/ChangeLog 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,3 +1,141 @@
+2018-02-08 Filip Pizlo <[email protected]>
+
+ Experiment with alternative implementation of memcpy/memset
+ https://bugs.webkit.org/show_bug.cgi?id=182563
+
+ Reviewed by Michael Saboff and Mark Lam.
+
+ Adds a faster x86_64-specific implementation of memcpy and memset. These versions go by
+ different names than memcpy/memset and have a different API:
+
+ WTF::fastCopy<T>(T* dst, T* src, size_t N): copies N values of type T from src to dst.
+ WTF::fastZeroFill(T* dst, size_T N): writes N * sizeof(T) zeroes to dst.
+
+ There are also *Bytes variants that take void* for dst and src and size_t numBytes. Those are
+ most appropriate in places where the code is already computing bytes.
+
+ These will just call memcpy/memset on platforms where the optimized versions are not supported.
+
+ These new functions are not known to the compiler to be memcpy/memset. This has the effect that
+ the compiler will not try to replace them with anything else. This could be good or bad:
+
+ - It's *good* if the size is *not known* at compile time. In that case, by my benchmarks, these
+ versions are faster than either the memcpy/memset call or whatever else the compiler could
+ emit. This is because of a combination of inlining and the algorithm itself (see below).
+
+ - It's *bad* if the size is *known* at compile time. In that case, the compiler could
+ potentially emit a fully unrolled memcpy/memset. That might not happen if the size is large
+ (even if it's known), but in this patch I avoid replacing any memcpy/memset calls when the
+ size is a constant. In particular, this totally avoids the call overhead -- if the size is
+ small, then the compiler will emit a nice inlined copy or set. If the size is large, then the
+ most optimal thing to do is emit the shortest piece of code possible, and that's a call to
+ memcpy/memset.
+
+ It's unfortunate that you have to choose between them on your own. One way to avoid that might
+ have been to override the memcpy/memset symbols, so that the compiler can still do its
+ reasoning. But that's not quite right, since then we would lose inlining in the unknonw-size
+ case. Also, it's possible that for some unknown-size cases, the compiler could choose to emit
+ something on its own because it might think that some property of aliasing or alignment could
+ help it. I think it's a bit better to use our own copy/set implementations even in those cases.
+ Another way that I tried avoiding this is to detect inside fastCopy/fastZeroFill if the size is
+ constant. But there is no good way to do that in C++. There is a builtin for doing that inside a
+ macro, but that feels janky, so I didn't want to do it in this patch.
+
+ The reason why these new fastCopy/fastZeroFill functions are faster is that:
+
+ - They can be inlined. There is no function call. Only a few registers get clobbered. So, the
+ impact on the quality of the code surrounding the memcpy/memset is smaller.
+
+ - They use type information to select the implementation. For sizes that are multiples of 2, 4,
+ or 8, the resulting code performs dramatically better on small arrays than memcpy because it
+ uses fewer cycles. The difference is greatest for 2 and 4 byte types, since memcpy usually
+ handles small arrays by tiering from a 8-byte word copy loop to a byte copy loop. So, for 2
+ or 4 byte arrays, we use an algorithm that tiers from 8-byte word down to a 2-byte or 4-byte
+ (depending on type) copy loop. So, for example, when copying a 16-bit string that has 1, 2, or
+ 3 characters, this means doing 1, 2, or 3 word copies rather than 2, 4, or 6 byte copies. For
+ 8-byte types, the resulting savings are mainly that there is no check to see if a tier-down to
+ the byte-copy loop is needed -- so really that means reducing code size. 1-byte types don't
+ get this inherent advantage over memcpy/memset, but they still benefit from all of the other
+ advantages of these functions. Of course, this advantage isn't inherent to our approach. The
+ compiler could also notice that the arguments to memcpy/memset have some alignment properties.
+ It could do it even more generally than we do - for example a copy over bytes where the size
+ is a multiple of 4 can use the 4-byte word algorithm. But based on my tests, the compiler does
+ not do this (even though it does other things, like turn a memset call with a zero value
+ argument into a bzero call).
+
+ - They use a very nicely written word copy/set loop for small arrays. I spent a lot of time
+ getting the assembly just right. When we use memcpy/memset, sometimes we would optimize the
+ call by having a fast path word copy loop for small sizes. That's not necessary with this
+ implementation, since the assembly copy loop gets inlined.
+
+ - They use `rep movs` or `rep stos` for copies of 200 bytes or more. This decision benchmarks
+ poorly on every synthetic memcpy/memset benchmark I have built, and so unsurprisingly, it's
+ not what system memcpy/memset does. Most system memcpy/memset implementations end up doing
+ some SSE for medium-sized copies,. However, I previously found that this decision is bad for
+ one of the memset calls in GC (see clearArray() and friends in ArrayConventions.h|cpp) - I was
+ able to make the overhead of that call virtually disappear by doing `rep stos` more
+ aggressively. The theory behind this change is that it's not just the GC that prefers smaller
+ `rep` threshold and no SSE. I am betting that `rep`ing more is better when the heap gets
+ chaotic and the data being copied is used in interesting ways -- hence, synthetic
+ memcpy/memset benchmarks think it's bad (they don't do enough chaotic memory accesses) while
+ it's good for real-world uses. Also, when I previously worked on JVMs, I had found that the
+ best memcpy/memset heuristics when dealing with GC'd objects in a crazy heap were different
+ than any memcpy/memset in any system library.
+
+ This appears to be a 0.9% speed-up on PLT. I'm not sure if it's more because of the inlining or
+ the `rep`. I think it's both. I'll leave figuring out the exact tuning for future patches.
+
+ * wtf/BitVector.cpp:
+ (WTF::BitVector::setSlow):
+ (WTF::BitVector::clearAll):
+ (WTF::BitVector::resizeOutOfLine):
+ * wtf/BitVector.h:
+ (WTF::BitVector::wordCount):
+ (WTF::BitVector::OutOfLineBits::numWords const):
+ * wtf/ConcurrentBuffer.h:
+ (WTF::ConcurrentBuffer::growExact):
+ * wtf/FastBitVector.h:
+ (WTF::FastBitVectorWordOwner::operator=):
+ (WTF::FastBitVectorWordOwner::clearAll):
+ (WTF::FastBitVectorWordOwner::set):
+ * wtf/FastCopy.h: Added.
+ (WTF::fastCopy):
+ (WTF::fastCopyBytes):
+ * wtf/FastMalloc.cpp:
+ (WTF::fastZeroedMalloc):
+ (WTF::fastStrDup):
+ (WTF::tryFastZeroedMalloc):
+ * wtf/FastZeroFill.h: Added.
+ (WTF::fastZeroFill):
+ (WTF::fastZeroFillBytes):
+ * wtf/MD5.cpp:
+ * wtf/OSAllocator.h:
+ (WTF::OSAllocator::reallocateCommitted):
+ * wtf/StringPrintStream.cpp:
+ (WTF::StringPrintStream::increaseSize):
+ * wtf/Vector.h:
+ * wtf/persistence/PersistentDecoder.cpp:
+ (WTF::Persistence::Decoder::decodeFixedLengthData):
+ * wtf/persistence/PersistentEncoder.cpp:
+ (WTF::Persistence::Encoder::encodeFixedLengthData):
+ * wtf/text/CString.cpp:
+ (WTF::CString::init):
+ (WTF::CString::copyBufferIfNeeded):
+ * wtf/text/LineBreakIteratorPoolICU.h:
+ (WTF::LineBreakIteratorPool::makeLocaleWithBreakKeyword):
+ * wtf/text/StringBuilder.cpp:
+ (WTF::StringBuilder::allocateBuffer):
+ (WTF::StringBuilder::append):
+ * wtf/text/StringConcatenate.h:
+ * wtf/text/StringImpl.h:
+ (WTF::StringImpl::copyCharacters):
+ * wtf/text/icu/UTextProvider.cpp:
+ (WTF::uTextCloneImpl):
+ * wtf/text/icu/UTextProviderLatin1.cpp:
+ (WTF::uTextLatin1Clone):
+ (WTF::openLatin1UTextProvider):
+ * wtf/threads/Signals.cpp:
+
2018-02-06 Darin Adler <[email protected]>
Event improvements
Modified: trunk/Source/WTF/WTF.xcodeproj/project.pbxproj (228305 => 228306)
--- trunk/Source/WTF/WTF.xcodeproj/project.pbxproj 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/WTF.xcodeproj/project.pbxproj 2018-02-09 02:13:01 UTC (rev 228306)
@@ -206,6 +206,8 @@
0F5F3D681F3FEBA600B115A2 /* CagedUniquePtr.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CagedUniquePtr.h; sourceTree = "<group>"; };
0F60F32D1DFCBD1B00416D6C /* LockedPrintStream.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LockedPrintStream.cpp; sourceTree = "<group>"; };
0F60F32E1DFCBD1B00416D6C /* LockedPrintStream.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LockedPrintStream.h; sourceTree = "<group>"; };
+ 0F62A8A6202CCC14007B8623 /* FastCopy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FastCopy.h; sourceTree = "<group>"; };
+ 0F62A8A7202CCC15007B8623 /* FastZeroFill.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FastZeroFill.h; sourceTree = "<group>"; };
0F66B2801DC97BAB004A1D3F /* ClockType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ClockType.cpp; sourceTree = "<group>"; };
0F66B2811DC97BAB004A1D3F /* ClockType.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ClockType.h; sourceTree = "<group>"; };
0F66B2821DC97BAB004A1D3F /* MonotonicTime.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MonotonicTime.cpp; sourceTree = "<group>"; };
@@ -864,9 +866,11 @@
A8A4729F151A825A004123FF /* ExportMacros.h */,
0F7C5FB51D885CF20044F5E2 /* FastBitVector.cpp */,
0FD81AC4154FB22E00983E72 /* FastBitVector.h */,
+ 0F62A8A6202CCC14007B8623 /* FastCopy.h */,
A8A472A1151A825A004123FF /* FastMalloc.cpp */,
A8A472A2151A825A004123FF /* FastMalloc.h */,
0F79C7C31E73511800EB34D1 /* FastTLS.h */,
+ 0F62A8A7202CCC15007B8623 /* FastZeroFill.h */,
B38FD7BC168953E80065C969 /* FeatureDefines.h */,
0F9D335B165DBA73005AD387 /* FilePrintStream.cpp */,
0F9D335C165DBA73005AD387 /* FilePrintStream.h */,
Modified: trunk/Source/WTF/wtf/BitVector.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/BitVector.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/BitVector.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2011 Apple Inc. All rights reserved.
+ * Copyright (C) 2011-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -29,7 +29,9 @@
#include <algorithm>
#include <string.h>
#include <wtf/Assertions.h>
+#include <wtf/FastCopy.h>
#include <wtf/FastMalloc.h>
+#include <wtf/FastZeroFill.h>
#include <wtf/StdLibExtras.h>
namespace WTF {
@@ -41,7 +43,7 @@
newBitsOrPointer = other.m_bitsOrPointer;
else {
OutOfLineBits* newOutOfLineBits = OutOfLineBits::create(other.size());
- memcpy(newOutOfLineBits->bits(), other.bits(), byteCount(other.size()));
+ fastCopy(newOutOfLineBits->bits(), other.bits(), wordCount(other.size()));
newBitsOrPointer = bitwise_cast<uintptr_t>(newOutOfLineBits) >> 1;
}
if (!isInline() && !isEmptyOrDeletedValue())
@@ -69,7 +71,7 @@
if (isInline())
m_bitsOrPointer = makeInlineBits(0);
else
- memset(outOfLineBits()->bits(), 0, byteCount(size()));
+ fastZeroFill(outOfLineBits()->bits(), wordCount(size()));
}
BitVector::OutOfLineBits* BitVector::OutOfLineBits::create(size_t numBits)
@@ -93,14 +95,14 @@
if (isInline()) {
// Make sure that all of the bits are zero in case we do a no-op resize.
*newOutOfLineBits->bits() = m_bitsOrPointer & ~(static_cast<uintptr_t>(1) << maxInlineBits());
- memset(newOutOfLineBits->bits() + 1, 0, (newNumWords - 1) * sizeof(void*));
+ fastZeroFill(newOutOfLineBits->bits() + 1, newNumWords - 1);
} else {
if (numBits > size()) {
size_t oldNumWords = outOfLineBits()->numWords();
- memcpy(newOutOfLineBits->bits(), outOfLineBits()->bits(), oldNumWords * sizeof(void*));
- memset(newOutOfLineBits->bits() + oldNumWords, 0, (newNumWords - oldNumWords) * sizeof(void*));
+ fastCopy(newOutOfLineBits->bits(), outOfLineBits()->bits(), oldNumWords);
+ fastZeroFill(newOutOfLineBits->bits() + oldNumWords, newNumWords - oldNumWords);
} else
- memcpy(newOutOfLineBits->bits(), outOfLineBits()->bits(), newOutOfLineBits->numWords() * sizeof(void*));
+ fastCopy(newOutOfLineBits->bits(), outOfLineBits()->bits(), newOutOfLineBits->numWords());
OutOfLineBits::destroy(outOfLineBits());
}
m_bitsOrPointer = bitwise_cast<uintptr_t>(newOutOfLineBits) >> 1;
Modified: trunk/Source/WTF/wtf/BitVector.h (228305 => 228306)
--- trunk/Source/WTF/wtf/BitVector.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/BitVector.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -354,6 +354,11 @@
return (bitCount + 7) >> 3;
}
+ static size_t wordCount(uintptr_t bits)
+ {
+ return (bits + bitsInPointer() - 1) / bitsInPointer();
+ }
+
static uintptr_t makeInlineBits(uintptr_t bits)
{
ASSERT(!(bits & (static_cast<uintptr_t>(1) << maxInlineBits())));
@@ -418,7 +423,7 @@
class OutOfLineBits {
public:
size_t numBits() const { return m_numBits; }
- size_t numWords() const { return (m_numBits + bitsInPointer() - 1) / bitsInPointer(); }
+ size_t numWords() const { return wordCount(m_numBits); }
uintptr_t* bits() { return bitwise_cast<uintptr_t*>(this + 1); }
const uintptr_t* bits() const { return bitwise_cast<const uintptr_t*>(this + 1); }
Modified: trunk/Source/WTF/wtf/CMakeLists.txt (228305 => 228306)
--- trunk/Source/WTF/wtf/CMakeLists.txt 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/CMakeLists.txt 2018-02-09 02:13:01 UTC (rev 228306)
@@ -59,8 +59,10 @@
Expected.h
ExportMacros.h
FastBitVector.h
+ FastCopy.h
FastMalloc.h
FastTLS.h
+ FastZeroFill.h
FeatureDefines.h
FilePrintStream.h
FlipBytes.h
Modified: trunk/Source/WTF/wtf/ConcurrentBuffer.h (228305 => 228306)
--- trunk/Source/WTF/wtf/ConcurrentBuffer.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/ConcurrentBuffer.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -26,6 +26,7 @@
#pragma once
#include <wtf/Atomics.h>
+#include <wtf/FastCopy.h>
#include <wtf/FastMalloc.h>
#include <wtf/HashFunctions.h>
#include <wtf/Lock.h>
@@ -65,7 +66,7 @@
Array* newArray = createArray(newSize);
// This allows us to do ConcurrentBuffer<std::unique_ptr<>>.
if (array)
- memcpy(newArray->data, array->data, sizeof(T) * array->size);
+ fastCopy(newArray->data, array->data, array->size);
for (size_t i = array ? array->size : 0; i < newSize; ++i)
new (newArray->data + i) T();
WTF::storeStoreFence();
Modified: trunk/Source/WTF/wtf/FastBitVector.h (228305 => 228306)
--- trunk/Source/WTF/wtf/FastBitVector.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/FastBitVector.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2012, 2013, 2016 Apple Inc. All rights reserved.
+ * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -27,7 +27,9 @@
#include <string.h>
#include <wtf/Atomics.h>
+#include <wtf/FastCopy.h>
#include <wtf/FastMalloc.h>
+#include <wtf/FastZeroFill.h>
#include <wtf/PrintStream.h>
#include <wtf/StdLibExtras.h>
@@ -95,7 +97,7 @@
if (arrayLength() != other.arrayLength())
setEqualsSlow(other);
else {
- memcpy(m_words, other.m_words, arrayLength() * sizeof(uint32_t));
+ fastCopy(m_words, other.m_words, arrayLength());
m_numBits = other.m_numBits;
}
return *this;
@@ -115,13 +117,13 @@
void clearAll()
{
- memset(m_words, 0, arrayLength() * sizeof(uint32_t));
+ fastZeroFill(m_words, arrayLength());
}
void set(const FastBitVectorWordOwner& other)
{
ASSERT_WITH_SECURITY_IMPLICATION(m_numBits == other.m_numBits);
- memcpy(m_words, other.m_words, arrayLength() * sizeof(uint32_t));
+ fastCopy(m_words, other.m_words, arrayLength());
}
size_t numBits() const
Added: trunk/Source/WTF/wtf/FastCopy.h (0 => 228306)
--- trunk/Source/WTF/wtf/FastCopy.h (rev 0)
+++ trunk/Source/WTF/wtf/FastCopy.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2018 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <wtf/StdLibExtras.h>
+
+namespace WTF {
+
+template<typename T>
+void fastCopy(T* dst, const T* src, size_t length)
+{
+#if CPU(X86_64) && COMPILER(GCC_OR_CLANG)
+ uint64_t tmp = 0;
+ size_t count = length * sizeof(T);
+ if (!(sizeof(T) % sizeof(uint64_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $3, %%rcx\n\t"
+ "rep movsq\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint32_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $2, %%rcx\n\t"
+ "rep movsl\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $4, %%rcx\n\t" // FIXME: This isn't really a loop. https://bugs.webkit.org/show_bug.cgi?id=182617
+ "4:\n\t"
+ "movl (%%rsi, %%rcx), %%eax\n\t"
+ "movl %%eax, (%%rdi, %%rcx)\n\t"
+ "subq $4, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint16_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $1, %%rcx\n\t"
+ "rep movsw\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $6, %%rcx\n\t"
+ "4:\n\t"
+ "movw (%%rsi, %%rcx), %%ax\n\t"
+ "movw %%ax, (%%rdi, %%rcx)\n\t"
+ "subq $2, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "rep movsb\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $7, %%rcx\n\t"
+ "4:\n\t"
+ "movb (%%rsi, %%rcx), %%al\n\t"
+ "movb %%al, (%%rdi, %%rcx)\n\t"
+ "subq $1, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+#else
+ memcpy(dst, src, length * sizeof(T));
+#endif
+}
+
+inline void fastCopyBytes(void* dst, const void* src, size_t bytes)
+{
+ fastCopy(static_cast<char*>(dst), static_cast<const char*>(src), bytes);
+}
+
+} // namespace WTF
+
+using WTF::fastCopy;
+using WTF::fastCopyBytes;
Modified: trunk/Source/WTF/wtf/FastMalloc.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/FastMalloc.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/FastMalloc.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2005, 2007, Google Inc. All rights reserved.
- * Copyright (C) 2005-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2005-2018 Apple Inc. All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -31,6 +31,8 @@
#include <limits>
#include <string.h>
#include <wtf/DataLog.h>
+#include <wtf/FastCopy.h>
+#include <wtf/FastZeroFill.h>
#if OS(WINDOWS)
#include <windows.h>
@@ -78,7 +80,7 @@
void* fastZeroedMalloc(size_t n)
{
void* result = fastMalloc(n);
- memset(result, 0, n);
+ fastZeroFillBytes(result, n);
return result;
}
@@ -86,7 +88,7 @@
{
size_t len = strlen(src) + 1;
char* dup = static_cast<char*>(fastMalloc(len));
- memcpy(dup, src, len);
+ fastCopy(dup, src, len);
return dup;
}
@@ -95,7 +97,7 @@
void* result;
if (!tryFastMalloc(n).getValue(result))
return 0;
- memset(result, 0, n);
+ fastZeroFillBytes(result, n);
return result;
}
Added: trunk/Source/WTF/wtf/FastZeroFill.h (0 => 228306)
--- trunk/Source/WTF/wtf/FastZeroFill.h (rev 0)
+++ trunk/Source/WTF/wtf/FastZeroFill.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2018 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <wtf/StdLibExtras.h>
+
+namespace WTF {
+
+template<typename T>
+void fastZeroFill(T* dst, size_t length)
+{
+#if CPU(X86_64) && COMPILER(GCC_OR_CLANG)
+ uint64_t zero = 0;
+ size_t count = length * sizeof(T);
+ if (!(sizeof(T) % sizeof(uint64_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $3, %%rcx\n\t"
+ "rep stosq\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint32_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $2, %%rcx\n\t"
+ "rep stosl\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $4, %%rcx\n\t" // FIXME: This isn't really a loop. https://bugs.webkit.org/show_bug.cgi?id=182617
+ "4:\n\t"
+ "movl %%eax, (%%rdi, %%rcx)\n\t"
+ "subq $4, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint16_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $1, %%rcx\n\t"
+ "rep stosw\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $6, %%rcx\n\t"
+ "4:\n\t"
+ "movw %%ax, (%%rdi, %%rcx)\n\t"
+ "subq $2, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "rep stosb\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $7, %%rcx\n\t"
+ "4:\n\t"
+ "movb %%al, (%%rdi, %%rcx)\n\t"
+ "sub $1, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+#else
+ memset(dst, 0, length * sizeof(T));
+#endif
+}
+
+inline void fastZeroFillBytes(void* dst, size_t bytes)
+{
+ fastZeroFill(static_cast<char*>(dst), bytes);
+}
+
+} // namespace WTF
+
+using WTF::fastZeroFill;
+using WTF::fastZeroFillBytes;
Modified: trunk/Source/WTF/wtf/OSAllocator.h (228305 => 228306)
--- trunk/Source/WTF/wtf/OSAllocator.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/OSAllocator.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2010 Apple Inc. All rights reserved.
+ * Copyright (C) 2010-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -27,6 +27,7 @@
#define OSAllocator_h
#include <algorithm>
+#include <wtf/FastCopy.h>
#include <wtf/VMTags.h>
namespace WTF {
@@ -90,7 +91,7 @@
inline T* OSAllocator::reallocateCommitted(T* oldBase, size_t oldSize, size_t newSize, Usage usage, bool writable, bool executable)
{
void* newBase = reserveAndCommit(newSize, usage, writable, executable);
- memcpy(newBase, oldBase, std::min(oldSize, newSize));
+ fastCopyBytes(newBase, oldBase, std::min(oldSize, newSize));
decommitAndRelease(oldBase, oldSize);
return static_cast<T*>(newBase);
}
Modified: trunk/Source/WTF/wtf/StringPrintStream.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/StringPrintStream.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/StringPrintStream.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -28,6 +28,7 @@
#include <stdarg.h>
#include <stdio.h>
+#include <wtf/FastCopy.h>
#include <wtf/FastMalloc.h>
namespace WTF {
@@ -119,7 +120,7 @@
// fastRealloc will just do malloc+free anyway. Also, this simplifies the code since
// we can't realloc the inline buffer.
char* newBuffer = static_cast<char*>(fastMalloc(m_size));
- memcpy(newBuffer, m_buffer, m_next + 1);
+ fastCopy(newBuffer, m_buffer, m_next + 1);
if (m_buffer != m_inlineBuffer)
fastFree(m_buffer);
m_buffer = newBuffer;
Modified: trunk/Source/WTF/wtf/Vector.h (228305 => 228306)
--- trunk/Source/WTF/wtf/Vector.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/Vector.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -27,7 +27,9 @@
#include <type_traits>
#include <utility>
#include <wtf/CheckedArithmetic.h>
+#include <wtf/FastCopy.h>
#include <wtf/FastMalloc.h>
+#include <wtf/FastZeroFill.h>
#include <wtf/Forward.h>
#include <wtf/MallocPtr.h>
#include <wtf/MathExtras.h>
@@ -86,7 +88,7 @@
{
static void initialize(T* begin, T* end)
{
- memset(begin, 0, reinterpret_cast<char*>(end) - reinterpret_cast<char*>(begin));
+ fastZeroFill(begin, end - begin);
}
};
@@ -126,7 +128,7 @@
{
static void move(const T* src, const T* srcEnd, T* dst)
{
- memcpy(dst, src, reinterpret_cast<const char*>(srcEnd) - reinterpret_cast<const char*>(src));
+ fastCopy(dst, src, srcEnd - src);
}
static void moveOverlapping(const T* src, const T* srcEnd, T* dst)
{
Modified: trunk/Source/WTF/wtf/persistence/PersistentDecoder.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/persistence/PersistentDecoder.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/persistence/PersistentDecoder.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -52,7 +52,7 @@
if (!bufferIsLargeEnoughToContain(size))
return false;
- memcpy(data, m_bufferPosition, size);
+ fastCopy(data, m_bufferPosition, size);
m_bufferPosition += size;
Encoder::updateChecksumForData(m_sha1, data, size);
Modified: trunk/Source/WTF/wtf/persistence/PersistentEncoder.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/persistence/PersistentEncoder.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/persistence/PersistentEncoder.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -58,7 +58,7 @@
updateChecksumForData(m_sha1, data, size);
uint8_t* buffer = grow(size);
- memcpy(buffer, data, size);
+ fastCopy(buffer, data, size);
}
template<typename Type>
Modified: trunk/Source/WTF/wtf/text/CString.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/text/CString.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/CString.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2003-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2003-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -28,6 +28,7 @@
#include "CString.h"
#include <string.h>
+#include <wtf/FastCopy.h>
#include <wtf/text/StringHasher.h>
#include <wtf/text/StringMalloc.h>
@@ -66,7 +67,7 @@
ASSERT(str);
m_buffer = CStringBuffer::createUninitialized(length);
- memcpy(m_buffer->mutableData(), str, length);
+ fastCopy(m_buffer->mutableData(), str, length);
m_buffer->mutableData()[length] = '\0';
}
@@ -96,7 +97,7 @@
RefPtr<CStringBuffer> buffer = WTFMove(m_buffer);
size_t length = buffer->length();
m_buffer = CStringBuffer::createUninitialized(length);
- memcpy(m_buffer->mutableData(), buffer->data(), length + 1);
+ fastCopy(m_buffer->mutableData(), buffer->data(), length + 1);
}
bool CString::isSafeToSendToAnotherThread() const
Modified: trunk/Source/WTF/wtf/text/LineBreakIteratorPoolICU.h (228305 => 228306)
--- trunk/Source/WTF/wtf/text/LineBreakIteratorPoolICU.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/LineBreakIteratorPoolICU.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2011 Apple Inc. All Rights Reserved.
+ * Copyright (C) 2011-2018 Apple Inc. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -26,6 +26,8 @@
#pragma once
#include <unicode/uloc.h>
+#include <wtf/FastCopy.h>
+#include <wtf/FastZeroFill.h>
#include <wtf/HashMap.h>
#include <wtf/NeverDestroyed.h>
#include <wtf/ThreadSpecific.h>
@@ -51,7 +53,7 @@
if (!utf8Locale.length())
return locale;
Vector<char> scratchBuffer(utf8Locale.length() + 11, 0);
- memcpy(scratchBuffer.data(), utf8Locale.data(), utf8Locale.length());
+ fastCopy(scratchBuffer.data(), utf8Locale.data(), utf8Locale.length());
const char* keywordValue = nullptr;
switch (mode) {
@@ -75,7 +77,7 @@
return AtomicString::fromUTF8(scratchBuffer.data(), lengthNeeded);
if (status == U_BUFFER_OVERFLOW_ERROR) {
scratchBuffer.grow(lengthNeeded + 1);
- memset(scratchBuffer.data() + utf8Locale.length(), 0, scratchBuffer.size() - utf8Locale.length());
+ fastZeroFill(scratchBuffer.data() + utf8Locale.length(), scratchBuffer.size() - utf8Locale.length());
status = U_ZERO_ERROR;
int32_t lengthNeeded2 = uloc_setKeywordValue("lb", keywordValue, scratchBuffer.data(), scratchBuffer.size(), &status);
if (!U_SUCCESS(status) || lengthNeeded != lengthNeeded2)
Modified: trunk/Source/WTF/wtf/text/StringBuilder.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/text/StringBuilder.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/StringBuilder.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -99,7 +99,7 @@
ASSERT(m_is8Bit);
// Copy the existing data into a new buffer, set result to point to the end of the existing data.
auto buffer = StringImpl::createUninitialized(requiredLength, m_bufferCharacters8);
- memcpy(m_bufferCharacters8, currentCharacters, static_cast<size_t>(m_length) * sizeof(LChar)); // This can't overflow.
+ fastCopy(m_bufferCharacters8, currentCharacters, m_length);
// Update the builder state.
m_buffer = WTFMove(buffer);
@@ -114,7 +114,7 @@
ASSERT(!m_is8Bit);
// Copy the existing data into a new buffer, set result to point to the end of the existing data.
auto buffer = StringImpl::createUninitialized(requiredLength, m_bufferCharacters16);
- memcpy(m_bufferCharacters16, currentCharacters, static_cast<size_t>(m_length) * sizeof(UChar)); // This can't overflow.
+ fastCopy(m_bufferCharacters16, currentCharacters, m_length);
// Update the builder state.
m_buffer = WTFMove(buffer);
@@ -276,10 +276,10 @@
allocateBufferUpConvert(m_string.isNull() ? 0 : m_string.characters8(), expandedCapacity(capacity(), requiredLength));
}
- memcpy(m_bufferCharacters16 + m_length, characters, static_cast<size_t>(length) * sizeof(UChar));
+ fastCopy(m_bufferCharacters16 + m_length, characters, length);
m_length = requiredLength;
} else
- memcpy(appendUninitialized<UChar>(length), characters, static_cast<size_t>(length) * sizeof(UChar));
+ fastCopy(appendUninitialized<UChar>(length), characters, length);
ASSERT(m_buffer->length() >= m_length);
}
@@ -291,13 +291,7 @@
if (m_is8Bit) {
LChar* dest = appendUninitialized<LChar>(length);
- if (length > 8)
- memcpy(dest, characters, static_cast<size_t>(length) * sizeof(LChar));
- else {
- const LChar* end = characters + length;
- while (characters < end)
- *(dest++) = *(characters++);
- }
+ fastCopy(dest, characters, length);
} else {
UChar* dest = appendUninitialized<UChar>(length);
const LChar* end = characters + length;
Modified: trunk/Source/WTF/wtf/text/StringConcatenate.h (228305 => 228306)
--- trunk/Source/WTF/wtf/text/StringConcatenate.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/StringConcatenate.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -27,6 +27,7 @@
#define StringConcatenate_h
#include <string.h>
+#include <wtf/FastCopy.h>
#ifndef AtomicString_h
#include <wtf/text/AtomicString.h>
@@ -157,7 +158,7 @@
void writeTo(UChar* destination) const
{
- memcpy(destination, m_characters, m_length * sizeof(UChar));
+ fastCopy(destination, m_characters, m_length);
}
String toString() const { return String(m_characters, m_length); }
Modified: trunk/Source/WTF/wtf/text/StringImpl.h (228305 => 228306)
--- trunk/Source/WTF/wtf/text/StringImpl.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/StringImpl.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1066,7 +1066,7 @@
*destination = *source;
return;
}
- memcpy(destination, source, numCharacters * sizeof(CharacterType));
+ fastCopy(destination, source, numCharacters);
}
ALWAYS_INLINE void StringImpl::copyCharacters(UChar* destination, const LChar* source, unsigned numCharacters)
Modified: trunk/Source/WTF/wtf/text/icu/UTextProvider.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/text/icu/UTextProvider.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/icu/UTextProvider.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -28,6 +28,7 @@
#include <algorithm>
#include <string.h>
+#include <wtf/FastCopy.h>
namespace WTF {
@@ -55,10 +56,10 @@
void* extraNew = destination->pExtra;
int32_t flags = destination->flags;
int sizeToCopy = std::min(source->sizeOfStruct, destination->sizeOfStruct);
- memcpy(destination, source, sizeToCopy);
+ fastCopyBytes(destination, source, sizeToCopy);
destination->pExtra = extraNew;
destination->flags = flags;
- memcpy(destination->pExtra, source->pExtra, extraSize);
+ fastCopyBytes(destination->pExtra, source->pExtra, extraSize);
fixPointer(source, destination, destination->context);
fixPointer(source, destination, destination->p);
fixPointer(source, destination, destination->q);
Modified: trunk/Source/WTF/wtf/text/icu/UTextProviderLatin1.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/text/icu/UTextProviderLatin1.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/text/icu/UTextProviderLatin1.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -27,6 +27,7 @@
#include "UTextProviderLatin1.h"
#include "UTextProvider.h"
+#include <wtf/FastZeroFill.h>
#include <wtf/text/StringImpl.h>
namespace WTF {
@@ -82,7 +83,7 @@
result->a = source->a;
result->pFuncs = &uTextLatin1Funcs;
result->chunkContents = (UChar*)result->pExtra;
- memset(const_cast<UChar*>(result->chunkContents), 0, sizeof(UChar) * UTextWithBufferInlineCapacity);
+ fastZeroFill(const_cast<UChar*>(result->chunkContents), UTextWithBufferInlineCapacity);
return result;
}
@@ -228,7 +229,7 @@
text->a = length;
text->pFuncs = &uTextLatin1Funcs;
text->chunkContents = (UChar*)text->pExtra;
- memset(const_cast<UChar*>(text->chunkContents), 0, sizeof(UChar) * UTextWithBufferInlineCapacity);
+ fastZeroFill(const_cast<UChar*>(text->chunkContents), UTextWithBufferInlineCapacity);
return text;
}
Modified: trunk/Source/WTF/wtf/threads/Signals.cpp (228305 => 228306)
--- trunk/Source/WTF/wtf/threads/Signals.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/WTF/wtf/threads/Signals.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -172,7 +172,7 @@
Signal signal = fromMachException(exceptionType);
RELEASE_ASSERT(signal != Signal::Unknown);
- memcpy(outState, inState, inStateCount * sizeof(inState[0]));
+ fastCopy(outState, inState, inStateCount);
*outStateCount = inStateCount;
#if CPU(X86_64)
Modified: trunk/Source/bmalloc/ChangeLog (228305 => 228306)
--- trunk/Source/bmalloc/ChangeLog 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/ChangeLog 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,3 +1,28 @@
+2018-02-08 Filip Pizlo <[email protected]>
+
+ Experiment with alternative implementation of memcpy/memset
+ https://bugs.webkit.org/show_bug.cgi?id=182563
+
+ Reviewed by Michael Saboff and Mark Lam.
+
+ Add a faster x86_64-specific implementation of memcpy and memset. Ideally, this would just be
+ implemented in WTF, but we have to copy it into bmalloc since bmalloc sits below WTF on the
+ stack.
+
+ * bmalloc/Algorithm.h:
+ (bmalloc::fastCopy):
+ (bmalloc::fastZeroFill):
+ * bmalloc/Allocator.cpp:
+ (bmalloc::Allocator::reallocate):
+ * bmalloc/Bits.h:
+ (bmalloc::BitsWordOwner::operator=):
+ (bmalloc::BitsWordOwner::clearAll):
+ (bmalloc::BitsWordOwner::set):
+ * bmalloc/IsoPageInlines.h:
+ (bmalloc::IsoPage<Config>::IsoPage):
+ * bmalloc/Vector.h:
+ (bmalloc::Vector<T>::reallocateBuffer):
+
2018-02-05 JF Bastien <[email protected]>
Gigacage: enable only for WebContent process and token executables
Modified: trunk/Source/bmalloc/bmalloc/Algorithm.h (228305 => 228306)
--- trunk/Source/bmalloc/bmalloc/Algorithm.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/bmalloc/Algorithm.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -180,6 +180,221 @@
return false;
}
+template<typename T>
+void fastCopy(T* dst, T* src, size_t length)
+{
+#if BCPU(X86_64)
+ uint64_t tmp = 0;
+ size_t count = length * sizeof(T);
+ if (!(sizeof(T) % sizeof(uint64_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $3, %%rcx\n\t"
+ "rep movsq\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint32_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $2, %%rcx\n\t"
+ "rep movsl\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $4, %%rcx\n\t" // FIXME: This isn't really a loop. https://bugs.webkit.org/show_bug.cgi?id=182617
+ "4:\n\t"
+ "movl (%%rsi, %%rcx), %%eax\n\t"
+ "movl %%eax, (%%rdi, %%rcx)\n\t"
+ "subq $4, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint16_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $1, %%rcx\n\t"
+ "rep movsw\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $6, %%rcx\n\t"
+ "4:\n\t"
+ "movw (%%rsi, %%rcx), %%ax\n\t"
+ "movw %%ax, (%%rdi, %%rcx)\n\t"
+ "subq $2, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+ return;
+ }
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "rep movsb\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq (%%rsi, %%rcx), %%rax\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $7, %%rcx\n\t"
+ "4:\n\t"
+ "movb (%%rsi, %%rcx), %%al\n\t"
+ "movb %%al, (%%rdi, %%rcx)\n\t"
+ "subq $1, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+S"(src), "+c"(count), "+a"(tmp)
+ :
+ : "memory"
+ );
+#else
+ memcpy(dst, src, length * sizeof(T));
+#endif
+}
+
+template<typename T>
+void fastZeroFill(T* dst, size_t length)
+{
+#if BCPU(X86_64)
+ uint64_t zero = 0;
+ size_t count = length * sizeof(T);
+ if (!(sizeof(T) % sizeof(uint64_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $3, %%rcx\n\t"
+ "rep stosq\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint32_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $2, %%rcx\n\t"
+ "rep stosl\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $4, %%rcx\n\t" // FIXME: This isn't really a loop. https://bugs.webkit.org/show_bug.cgi?id=182617
+ "4:\n\t"
+ "movl %%eax, (%%rdi, %%rcx)\n\t"
+ "subq $4, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ if (!(sizeof(T) % sizeof(uint16_t))) {
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "shrq $1, %%rcx\n\t"
+ "rep stosw\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $6, %%rcx\n\t"
+ "4:\n\t"
+ "movw %%ax, (%%rdi, %%rcx)\n\t"
+ "subq $2, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+ return;
+ }
+ asm volatile (
+ "cmpq $200, %%rcx\n\t"
+ "jb 1f\n\t"
+ "rep stosb\n\t"
+ "jmp 2f\n\t"
+ "3:\n\t"
+ "movq %%rax, (%%rdi, %%rcx)\n\t"
+ "1:\n\t"
+ "subq $8, %%rcx\n\t"
+ "jae 3b\n\t"
+ "cmpq $-8, %%rcx\n\t"
+ "je 2f\n\t"
+ "addq $7, %%rcx\n\t"
+ "4:\n\t"
+ "movb %%al, (%%rdi, %%rcx)\n\t"
+ "sub $1, %%rcx\n\t"
+ "jae 4b\n\t"
+ "2:\n\t"
+ : "+D"(dst), "+c"(count)
+ : "a"(zero)
+ : "memory"
+ );
+#else
+ memset(dst, 0, length * sizeof(T));
+#endif
+}
+
} // namespace bmalloc
#endif // Algorithm_h
Modified: trunk/Source/bmalloc/bmalloc/Allocator.cpp (228305 => 228306)
--- trunk/Source/bmalloc/bmalloc/Allocator.cpp 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/bmalloc/Allocator.cpp 2018-02-09 02:13:01 UTC (rev 228306)
@@ -125,7 +125,7 @@
void* result = allocate(newSize);
size_t copySize = std::min(oldSize, newSize);
- memcpy(result, object, copySize);
+ fastCopy(static_cast<char*>(result), static_cast<char*>(object), copySize);
m_deallocator.deallocate(object);
return result;
}
Modified: trunk/Source/bmalloc/bmalloc/Bits.h (228305 => 228306)
--- trunk/Source/bmalloc/bmalloc/Bits.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/bmalloc/Bits.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -80,7 +80,7 @@
BitsWordOwner& operator=(const BitsWordOwner& other)
{
- memcpy(m_words, other.m_words, arrayLength() * sizeof(uint32_t));
+ fastCopy(m_words, other.m_words, arrayLength());
return *this;
}
@@ -91,12 +91,12 @@
void clearAll()
{
- memset(m_words, 0, arrayLength() * sizeof(uint32_t));
+ fastZeroFill(m_words, arrayLength());
}
void set(const BitsWordOwner& other)
{
- memcpy(m_words, other.m_words, arrayLength() * sizeof(uint32_t));
+ fastCopy(m_words, other.m_words, arrayLength());
}
size_t numBits() const
Modified: trunk/Source/bmalloc/bmalloc/IsoPageInlines.h (228305 => 228306)
--- trunk/Source/bmalloc/bmalloc/IsoPageInlines.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/bmalloc/IsoPageInlines.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -47,7 +47,7 @@
: m_directory(directory)
, m_index(index)
{
- memset(m_allocBits, 0, sizeof(m_allocBits));
+ fastZeroFill(m_allocBits, bitsArrayLength(numObjects));
}
template<typename Config>
Modified: trunk/Source/bmalloc/bmalloc/Vector.h (228305 => 228306)
--- trunk/Source/bmalloc/bmalloc/Vector.h 2018-02-09 01:58:23 UTC (rev 228305)
+++ trunk/Source/bmalloc/bmalloc/Vector.h 2018-02-09 02:13:01 UTC (rev 228306)
@@ -203,7 +203,7 @@
size_t vmSize = bmalloc::vmSize(newCapacity * sizeof(T));
T* newBuffer = vmSize ? static_cast<T*>(vmAllocate(vmSize)) : nullptr;
if (m_buffer) {
- std::memcpy(newBuffer, m_buffer, m_size * sizeof(T));
+ fastCopy(newBuffer, m_buffer, m_size);
vmDeallocate(m_buffer, bmalloc::vmSize(m_capacity * sizeof(T)));
}
