Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/7762#discussion_r35946576
--- Diff:
unsafe/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java ---
@@ -470,31 +482,52 @@ public void putNewKey(
dataPages.add(newPage);
pageCursor = 0;
currentDataPage = newPage;
+ dataPage = currentDataPage;
+ dataPageBaseObject = currentDataPage.getBaseObject();
+ dataPageInsertOffset = currentDataPage.getBaseOffset();
+ } else {
+ // There is enough space in the current data page.
+ dataPage = currentDataPage;
+ dataPageBaseObject = currentDataPage.getBaseObject();
+ dataPageInsertOffset = currentDataPage.getBaseOffset() +
pageCursor;
}
+ // --- Append the key and value data to the current data page
--------------------------------
+
+ long insertCursor = dataPageInsertOffset;
+
// Compute all of our offsets up-front:
- final Object pageBaseObject = currentDataPage.getBaseObject();
- final long pageBaseOffset = currentDataPage.getBaseOffset();
- final long keySizeOffsetInPage = pageBaseOffset + pageCursor;
- pageCursor += 8; // word used to store the key size
- final long keyDataOffsetInPage = pageBaseOffset + pageCursor;
- pageCursor += keyLengthBytes;
- final long valueSizeOffsetInPage = pageBaseOffset + pageCursor;
- pageCursor += 8; // word used to store the value size
- final long valueDataOffsetInPage = pageBaseOffset + pageCursor;
- pageCursor += valueLengthBytes;
+ final long keySizeOffsetInPage = insertCursor;
+ insertCursor += 8; // word used to store the key size
+ final long keyDataOffsetInPage = insertCursor;
+ insertCursor += keyLengthBytes;
+ final long valueSizeOffsetInPage = insertCursor;
+ insertCursor += 8; // word used to store the value size
+ final long valueDataOffsetInPage = insertCursor;
+ insertCursor += valueLengthBytes; // word used to store the value
size
// Copy the key
- PlatformDependent.UNSAFE.putLong(pageBaseObject,
keySizeOffsetInPage, keyLengthBytes);
+ PlatformDependent.UNSAFE.putLong(dataPageBaseObject,
keySizeOffsetInPage, keyLengthBytes);
PlatformDependent.copyMemory(
- keyBaseObject, keyBaseOffset, pageBaseObject, keyDataOffsetInPage,
keyLengthBytes);
+ keyBaseObject, keyBaseOffset, dataPageBaseObject,
keyDataOffsetInPage, keyLengthBytes);
// Copy the value
- PlatformDependent.UNSAFE.putLong(pageBaseObject,
valueSizeOffsetInPage, valueLengthBytes);
- PlatformDependent.copyMemory(
- valueBaseObject, valueBaseOffset, pageBaseObject,
valueDataOffsetInPage, valueLengthBytes);
+ PlatformDependent.UNSAFE.putLong(dataPageBaseObject,
valueSizeOffsetInPage, valueLengthBytes);
+ PlatformDependent.copyMemory(valueBaseObject, valueBaseOffset,
dataPageBaseObject,
+ valueDataOffsetInPage, valueLengthBytes);
+ // --- Update bookeeping data structures
-----------------------------------------------------
+
+ if (useOverflowPage) {
+ // Store the end-of-page marker at the end of the data page
+ PlatformDependent.UNSAFE.putLong(dataPageBaseObject, insertCursor,
END_OF_PAGE_MARKER);
+ } else {
+ pageCursor += requiredSize;
+ }
+
+ size++;
--- End diff --
i'd rename size -> numElements
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
