foosekis4j commented on code in PR #11211:
URL: https://github.com/apache/paimon/pull/11211#discussion_r4652663986
##########
pom.xml:
##########
@@ -303,6 +303,22 @@ under the License.
</dependencyManagement>
<profiles>
+ <profile>
+ <!-- GPU compaction (paimon-gpu) through TornadoVM, which needs
JDK 22 or later. -->
+ <id>gpu</id>
+ <modules>
+ <module>paimon-gpu</module>
+ </modules>
+ <properties>
+ <target.java.version>22</target.java.version>
Review Comment:
Thanks, fixed in efa2c2032a. The `gpu` profile now only adds the module;
`target.java.version=22` is set in `paimon-gpu/pom.xml`. With `-Pgpu`,
`paimon-api` compiles to class-file major 52 again (checked `CoreOptions` with
`javap`), and `paimon-gpu` stays at 66.
##########
paimon-gpu/src/main/java/org/apache/paimon/gpu/GpuMergeTreeCompactRewriter.java:
##########
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.gpu;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.compact.CompactResult;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryRowWriter;
+import org.apache.paimon.format.SimpleColStats;
+import org.apache.paimon.format.SimpleStatsExtractor;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataFilePathFactory;
+import org.apache.paimon.io.KeyValueFileWriterFactory;
+import org.apache.paimon.manifest.FileSource;
+import org.apache.paimon.mergetree.SortedRun;
+import org.apache.paimon.mergetree.compact.AbstractCompactRewriter;
+import org.apache.paimon.mergetree.compact.CompactRewriter;
+import org.apache.paimon.stats.SimpleStats;
+import org.apache.paimon.stats.SimpleStatsConverter;
+import org.apache.paimon.types.DataTypeRoot;
+import org.apache.paimon.utils.Pair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A deduplicating merge-tree compaction on the device. Each batch of sections
(key intervals that
+ * do not overlap) becomes one file; a compaction the device cannot take, or
one that fails there,
+ * is handed to Paimon's rewriter whole.
+ */
+final class GpuMergeTreeCompactRewriter extends AbstractCompactRewriter {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(GpuMergeTreeCompactRewriter.class);
+
+ private static final long MAX_SEQUENCE = Long.MAX_VALUE >> 1;
+
+ private final CompactRewriter fallback;
+ private final KeyValueFileWriterFactory writerFactory;
+ private final KvLayout layout;
+ private final DeviceCompactor device;
+ private final CoreOptions options;
+ private final long schemaId;
+ private final Optional<SimpleStatsExtractor> stats;
+ private final SimpleStatsConverter keyStats;
+ private final SimpleStatsConverter valueStats;
+ private final long maxBatchRows;
+ private final Stats counters;
+
+ GpuMergeTreeCompactRewriter(
+ CompactRewriter fallback,
+ KeyValueFileWriterFactory writerFactory,
+ KvLayout layout,
+ DeviceCompactor device,
+ CoreOptions options,
+ long schemaId,
+ Optional<SimpleStatsExtractor> stats,
+ long maxBatchRows,
+ Stats counters) {
+ this.fallback = fallback;
+ this.writerFactory = writerFactory;
+ this.layout = layout;
+ this.device = device;
+ this.options = options;
+ this.schemaId = schemaId;
+ this.stats = stats;
+ this.keyStats = new SimpleStatsConverter(layout.keyType);
+ this.valueStats = new SimpleStatsConverter(layout.valueType,
options.statsDenseStore());
+ this.maxBatchRows = maxBatchRows;
+ this.counters = counters;
+ }
+
+ @Override
+ public CompactResult rewrite(
+ int outputLevel, boolean dropDelete, List<List<SortedRun>>
sections) throws Exception {
+ List<DataFileMeta> before = extractFilesFromSections(sections);
+ String reason = unsupported(before, sections);
+ if (reason != null) {
+ LOG.debug("Compaction on the CPU: {}", reason);
+ counters.cpu(reason);
+ return fallback.rewrite(outputLevel, dropDelete, sections);
+ }
+ FileIO io = writerFactory.getFileIO();
+ DataFilePathFactory outputs = writerFactory.pathFactory(outputLevel);
+ List<DataFileMeta> after = new ArrayList<>();
+ List<Path> written = new ArrayList<>();
+ try {
+ for (List<List<SortedRun>> batch : batches(sections)) {
+ List<DataFileMeta> files = extractFilesFromSections(batch);
+ List<String> inputs = new ArrayList<>(files.size());
+ long rows = 0;
+ for (DataFileMeta file : files) {
+
inputs.add(local(writerFactory.pathFactory(file.level()).toPath(file)));
+ rows += file.rowCount();
+ }
+ long chars = layout.stringColumns.length == 0 ? 0 :
stringBytes(files);
+ Path output = outputs.newPath();
+ written.add(output);
+ long t0 = System.nanoTime();
+ DeviceCompactor.Summary summary =
+ device.compact(
+ inputs,
+ rows,
+ chars,
+ local(output),
+ dropDelete,
+ KvLayout.codec(options.fileCompression()));
+ counters.device(rows, summary.rows(), System.nanoTime() - t0);
+ if (summary.ties()) {
+ // Not a failure: which tied row Paimon keeps is its
readers' order.
+ for (Path path : written) {
+ io.deleteQuietly(path);
+ }
+ String tie = "rows of a key with equal sequence numbers
and kinds";
+ LOG.debug("Compaction on the CPU: {}", tie);
+ counters.cpu(tie);
+ return fallback.rewrite(outputLevel, dropDelete, sections);
Review Comment:
Fixed in efa2c2032a. The device path now only records why the CPU should
take the compaction: a tie breaks out of the batch loop, and a device failure
is caught. After the `try`, the written files are deleted and Paimon's rewriter
is called once, outside any device handler, so its own failure propagates
unchanged.
##########
paimon-gpu/src/main/java/org/apache/paimon/gpu/DeviceCompactor.java:
##########
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.gpu;
+
+import uk.ac.manchester.tornado.api.GridScheduler;
+import uk.ac.manchester.tornado.api.KernelContext;
+import uk.ac.manchester.tornado.api.TaskGraph;
+import uk.ac.manchester.tornado.api.TornadoExecutionPlan;
+import uk.ac.manchester.tornado.api.WorkerGrid1D;
+import uk.ac.manchester.tornado.api.enums.DataTransferMode;
+import uk.ac.manchester.tornado.api.types.arrays.ByteArray;
+import uk.ac.manchester.tornado.api.types.arrays.DoubleArray;
+import uk.ac.manchester.tornado.api.types.arrays.IntArray;
+import uk.ac.manchester.tornado.api.types.arrays.LongArray;
+import uk.ac.manchester.tornado.cudf.Cudf;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Runs one batch of a compaction on the device: reads the batch's files,
orders the rows by key and
+ * sequence number, keeps the last row of each key, and writes the survivors
to one Parquet file.
+ *
+ * <p>One plan a capacity; a batch with more rows, or more string bytes, than
the plan holds builds
+ * a larger one. Batches run one at a time.
+ */
+final class DeviceCompactor implements AutoCloseable {
+
+ /** What a batch wrote. */
+ static final class Summary {
+ private final int rows;
+ private final long minSequence;
+ private final long maxSequence;
+ private final long retractions;
+ private final long[] minKey;
+ private final long[] maxKey;
+ private final boolean ties;
+
+ Summary(
+ int rows,
+ long minSequence,
+ long maxSequence,
+ long retractions,
+ long[] minKey,
+ long[] maxKey,
+ boolean ties) {
+ this.rows = rows;
+ this.ties = ties;
+ this.minSequence = minSequence;
+ this.maxSequence = maxSequence;
+ this.retractions = retractions;
+ this.minKey = minKey;
+ this.maxKey = maxKey;
+ }
+
+ int rows() {
+ return rows;
+ }
+
+ long minSequence() {
+ return minSequence;
+ }
+
+ long maxSequence() {
+ return maxSequence;
+ }
+
+ long retractions() {
+ return retractions;
+ }
+
+ long[] minKey() {
+ return minKey;
+ }
+
+ long[] maxKey() {
+ return maxKey;
+ }
+
+ /** Whether two rows of a key had the same sequence number and kind. */
+ boolean ties() {
+ return ties;
+ }
+ }
+
+ private static final int MIN_CAPACITY = 1 << 20;
+
+ private final KvLayout layout;
+ private Plan plan;
+
+ DeviceCompactor(KvLayout layout) {
+ this.layout = layout;
+ }
+
+ /**
+ * The capacity a batch of {@code rows} needs: a power of two, at least
{@code MIN_CAPACITY}.
+ */
+ static int capacityFor(long rows) {
+ long capacity = MIN_CAPACITY;
+ while (capacity < rows) {
+ capacity <<= 1;
+ }
+ if (capacity > (1 << 30)) {
+ throw new IllegalArgumentException("A batch of " + rows + " rows
is too large");
+ }
+ return (int) capacity;
+ }
+
+ synchronized Summary compact(
Review Comment:
Fixed in efa2c2032a. `DeviceCompactor` uses a `ReentrantLock` taken with
`lockInterruptibly()`, so a cancelled task waiting for its turn wakes up. Once
it holds the lock, it checks for interruption and for a `closed` flag that
`close()` sets, and it does both checks before building any plan. An
interrupted caller gets `InterruptedException`, and a closed compactor gives
`CancellationException`. The rewriter also checks interruption before each
batch. It rethrows `InterruptedException`/`CancellationException`, and any
failure raised while the thread is interrupted, after deleting its files and
without a CPU fallback. The factory hands out the default rewriter after
`close()`. `DeviceCompactorTest` covers the closed and interrupted cases
without a device.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]