gatorsmile commented on a change in pull request #26229:
[SPARK-21492][SQL][Follow Up] Reimplement UnsafeExternalRowSorter in database
style iterator
URL: https://github.com/apache/spark/pull/26229#discussion_r340901926
##########
File path:
sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
##########
@@ -168,39 +167,40 @@ public void cleanupResources() {
sorter.cleanupResources();
}
- public Iterator<UnsafeRow> sort() throws IOException {
+ public Iterator<InternalRow> sort() throws IOException {
try {
final UnsafeSorterIterator sortedIterator = sorter.getSortedIterator();
if (!sortedIterator.hasNext()) {
// Since we won't ever call next() on an empty iterator, we need to
clean up resources
// here in order to prevent memory leaks.
cleanupResources();
}
- return new AbstractIterator<UnsafeRow>() {
+ return new RowIterator() {
private final int numFields = schema.length();
private UnsafeRow row = new UnsafeRow(numFields);
@Override
- public boolean hasNext() {
- return !isReleased && sortedIterator.hasNext();
- }
-
- @Override
- public UnsafeRow next() {
+ public boolean advanceNext() {
try {
- sortedIterator.loadNext();
- row.pointTo(
- sortedIterator.getBaseObject(),
- sortedIterator.getBaseOffset(),
- sortedIterator.getRecordLength());
- if (!hasNext()) {
- UnsafeRow copy = row.copy(); // so that we don't have dangling
pointers to freed page
- row = null; // so that we don't keep references to the base
object
- cleanupResources();
- return copy;
+ if (!isReleased && sortedIterator.hasNext()) {
+ sortedIterator.loadNext();
+ row.pointTo(
+ sortedIterator.getBaseObject(),
+ sortedIterator.getBaseOffset(),
+ sortedIterator.getRecordLength());
Review comment:
nit: indents
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
