This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new aed75edcd [core] Add TimeTravelUtils to resolve snapshot from scan
params (#4307)
aed75edcd is described below
commit aed75edcd2c2f40c0e08c88bb26ac032db6a3ec7
Author: xuzifu666 <[email protected]>
AuthorDate: Mon Oct 14 20:50:34 2024 +0800
[core] Add TimeTravelUtils to resolve snapshot from scan params (#4307)
---
.../table/source/snapshot/TimeTravelUtil.java | 110 +++++++++++++++++++++
.../table/source/snapshot/TimeTravelUtilsTest.java | 85 ++++++++++++++++
2 files changed, 195 insertions(+)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java
new file mode 100644
index 000000000..2ef0d5153
--- /dev/null
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source.snapshot;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.utils.SnapshotManager;
+import org.apache.paimon.utils.TagManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** The util class of resolve snapshot from scan params for time travel. */
+public class TimeTravelUtil {
+
+ private static String[] scanKeys = {
+ CoreOptions.SCAN_SNAPSHOT_ID.key(),
+ CoreOptions.SCAN_TAG_NAME.key(),
+ CoreOptions.SCAN_WATERMARK.key(),
+ CoreOptions.SCAN_TIMESTAMP_MILLIS.key()
+ };
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TimeTravelUtil.class);
+
+ public static Snapshot resolveSnapshotFromOptions(
+ CoreOptions options, SnapshotManager snapshotManager) {
+ List<String> scanHandleKey = new ArrayList<>(1);
+ for (String key : scanKeys) {
+ if (options.toConfiguration().containsKey(key)) {
+ scanHandleKey.add(key);
+ }
+ }
+
+ if (scanHandleKey.size() == 0) {
+ LOG.warn("Not set any time travel parameter.");
+ return null;
+ }
+
+ Preconditions.checkArgument(
+ scanHandleKey.size() == 1,
+ String.format(
+ "Only one of the following parameters may be set :
[%s, %s, %s, %s]",
+ CoreOptions.SCAN_SNAPSHOT_ID.key(),
+ CoreOptions.SCAN_TAG_NAME.key(),
+ CoreOptions.SCAN_WATERMARK.key(),
+ CoreOptions.SCAN_TIMESTAMP_MILLIS.key()));
+
+ String key = scanHandleKey.get(0);
+ Snapshot snapshot = null;
+ if (key.equals(CoreOptions.SCAN_SNAPSHOT_ID.key())) {
+ snapshot = resolveSnapshotBySnapshotId(snapshotManager, options);
+ } else if (key.equals(CoreOptions.SCAN_WATERMARK.key())) {
+ snapshot = resolveSnapshotByWatermark(snapshotManager, options);
+ } else if (key.equals(CoreOptions.SCAN_TIMESTAMP_MILLIS.key())) {
+ snapshot = resolveSnapshotByTimestamp(snapshotManager, options);
+ } else if (key.equals(CoreOptions.SCAN_TAG_NAME.key())) {
+ snapshot = resolveSnapshotByTagName(snapshotManager, options);
+ }
+ return snapshot;
+ }
+
+ private static Snapshot resolveSnapshotBySnapshotId(
+ SnapshotManager snapshotManager, CoreOptions options) {
+ Long snapshotId = options.scanSnapshotId();
+ if (snapshotId != null && snapshotManager.snapshotExists(snapshotId)) {
+ return snapshotManager.snapshot(snapshotId);
+ }
+ return null;
+ }
+
+ private static Snapshot resolveSnapshotByTimestamp(
+ SnapshotManager snapshotManager, CoreOptions options) {
+ Long timestamp = options.scanTimestampMills();
+ return snapshotManager.earlierOrEqualTimeMills(timestamp);
+ }
+
+ private static Snapshot resolveSnapshotByWatermark(
+ SnapshotManager snapshotManager, CoreOptions options) {
+ Long watermark = options.scanWatermark();
+ return snapshotManager.laterOrEqualWatermark(watermark);
+ }
+
+ private static Snapshot resolveSnapshotByTagName(
+ SnapshotManager snapshotManager, CoreOptions options) {
+ String tagName = options.scanTagName();
+ TagManager tagManager =
+ new TagManager(snapshotManager.fileIO(),
snapshotManager.tablePath());
+ return tagManager.taggedSnapshot(tagName);
+ }
+}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java
new file mode 100644
index 000000000..4d5ea873f
--- /dev/null
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source.snapshot;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.table.sink.StreamTableCommit;
+import org.apache.paimon.table.sink.StreamTableWrite;
+import org.apache.paimon.utils.SnapshotManager;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/** Tests for {@link TimeTravelUtil}. */
+public class TimeTravelUtilsTest extends ScannerTestBase {
+
+ @Test
+ public void testResolveSnapshotFromOptions() throws Exception {
+ SnapshotManager snapshotManager = table.snapshotManager();
+ StreamTableWrite write = table.newWrite(commitUser);
+ StreamTableCommit commit = table.newCommit(commitUser);
+
+ write.write(rowData(1, 10, 100L));
+ commit.commit(0, write.prepareCommit(true, 0));
+
+ long ts = System.currentTimeMillis();
+
+ write.write(rowData(2, 30, 101L));
+ commit.commit(1, write.prepareCommit(true, 1));
+
+ write.write(rowData(3, 50, 500L));
+ commit.commit(2, write.prepareCommit(true, 2));
+
+ HashMap<String, String> optMap = new HashMap<>(4);
+ optMap.put("scan.snapshot-id", "2");
+ CoreOptions options = CoreOptions.fromMap(optMap);
+ Snapshot snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options,
snapshotManager);
+ assertNotNull(snapshot);
+ assertTrue(snapshot.id() == 2);
+
+ optMap.clear();
+ optMap.put("scan.timestamp-millis", ts + "");
+ options = CoreOptions.fromMap(optMap);
+ snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options,
snapshotManager);
+ assertTrue(snapshot.id() == 1);
+
+ table.createTag("tag3", 3);
+ optMap.clear();
+ optMap.put("scan.tag-name", "tag3");
+ options = CoreOptions.fromMap(optMap);
+ snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options,
snapshotManager);
+ assertTrue(snapshot.id() == 3);
+
+ // if contain more scan.xxx config would throw out
+ optMap.put("scan.snapshot-id", "2");
+ CoreOptions options1 = CoreOptions.fromMap(optMap);
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> TimeTravelUtil.resolveSnapshotFromOptions(options1,
snapshotManager),
+ "scan.snapshot-id scan.tag-name scan.watermark and
scan.timestamp-millis can contains only one");
+ write.close();
+ commit.close();
+ }
+}
