peterxcli commented on issue #5462:
URL:
https://github.com/apache/datafusion-comet/issues/5462#issuecomment-5428888167
seems like there isnt sigificant benefit by doing this.
```java
package org.apache.spark.sql.comet.execution.shuffle;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.spark.unsafe.memory.MemoryBlock;
public final class SpillWriterMemoryUsageBenchmark {
private static final int ITERATIONS = 250_000;
private static volatile SpillWriter writer;
private static volatile long sink;
private static final class TestWriter extends SpillWriter {
TestWriter(int pages) throws ReflectiveOperationException {
allocatedPages = new LinkedList<>();
for (int i = 0; i < pages; i++) {
allocatedPages.add(new MemoryBlock(null, 0, 4096));
}
try {
Field counter =
SpillWriter.class.getDeclaredField("allocatedPageBytes");
counter.setAccessible(true);
((AtomicLong) counter.get(this)).set((long) pages * 4096);
} catch (NoSuchFieldException ignored) {
// Baseline implementation derives the value from allocatedPages.
}
}
@Override
protected void spill(int required) throws IOException {}
}
private static double sample() {
long sum = 0;
long start = System.nanoTime();
for (int i = 0; i < ITERATIONS; i++) {
sum += writer.getMemoryUsage() ^ i;
}
long elapsed = System.nanoTime() - start;
sink = sum;
return (double) elapsed / ITERATIONS;
}
public static void main(String[] args) throws Exception {
for (int pages : new int[] {1, 16, 256, 1024}) {
writer = new TestWriter(pages);
for (int i = 0; i < 5; i++) {
sample();
}
double[] samples = new double[9];
for (int i = 0; i < samples.length; i++) {
samples[i] = sample();
}
Arrays.sort(samples);
System.out.printf("pages=%d median_ns_per_sample=%.2f%n", pages,
samples[4]);
}
System.out.println("sink=" + sink);
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
