imbajin commented on code in PR #3399: URL: https://github.com/apache/hugegraph/pull/3399#discussion_r4505250748
########## hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map<String, Object> options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map<String, Long> ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List<Path> roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { Review Comment: Addressed in follow-up #3473 (head 47fe808fa): parsing preserves trailing-delimiter behavior, with a startup regression test. The original PR head remains unchanged, so this thread remains open here. ########## hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java: ########## @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map<String, Object> options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map<String, Long> ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List<Path> roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { + if (value.isBlank()) { + throw new IOException("Empty Store data path"); + } + Path root = Path.of(value).toAbsolutePath().normalize(); + if (offline && !Files.isDirectory(root)) { + throw new IOException("Expected an existing stopped Store data directory: " + root); + } + roots.add(root); + } + try (DatabaseLocks locks = new DatabaseLocks()) { + Path metadata = roots.get(0).resolve("hgstore-metadata"); + if (Files.exists(metadata.resolve("CURRENT"))) { + locks.add(metadata); + } + Set<Path> databases = new LinkedHashSet<>(); + Set<Path> snapshots = new LinkedHashSet<>(); + Set<Integer> discovered = new HashSet<>(); + for (Path root : roots) { + discover(root, null, databases, snapshots, discovered); + } + if (Files.exists(metadata.resolve("CURRENT"))) { + try (Database database = new Database(metadata, config); + RocksIterator iterator = database.db.newIterator()) { + byte[] prefix = MetadataKeyHelper.getPartitionStorePrefix(); + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + Metapb.PartitionStore location = Metapb.PartitionStore.parseFrom(iterator.value()); + // Normal moved roots are remapped by PartitionManager.resetPartitionPath(). + if (!discovered.contains(location.getPartitionId())) { + discover(Path.of(location.getStoreLocation()), location.getPartitionId(), + databases, snapshots, new HashSet<>()); + } + } + iterator.status(); + } + } + for (Path database : databases) { + requireCurrent(database); + locks.add(database); Review Comment: A follow-up implementation is available in #3473 (head 47fe808fa). Startup creates a missing empty LOCK and acquires the normal lock before inspecting the database; offline checking remains read-only and refuses missing locks. The new tests generate real RocksDB checkpoints and verify both clean and legacy data. This original PR head remains unchanged, so I am leaving this thread open here. ########## hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh: ########## @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Launcher-selection fixtures; native database inspection is tested separately. +set -Eeuo pipefail +SOURCE_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/../static/bin" && pwd)" +REPO_ROOT="$(cd "$SOURCE_BIN/../../../../../.." && pwd)" +SHARED_BIN="$REPO_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin" +TEST_ROOT=$(mktemp -d) +TEST_ROOT=$(cd "$TEST_ROOT" && pwd -P) +trap 'rm -rf "$TEST_ROOT"' EXIT +TOP="$TEST_ROOT/checker distribution" +mkdir -p "$TOP/bin" "$TOP/lib" "$TOP/conf" "$TEST_ROOT/java/bin" "$TEST_ROOT/fake-bin" +cp "$SOURCE_BIN/hugegraph-store-check.sh" "$SHARED_BIN/preload-topling.sh" "$TOP/bin/" +printf '%s\n' '# module options' '--add-opens=java.base/java.lang=ALL-UNNAMED' > "$TOP/bin/jvm-module.options" +touch "$TOP/lib/hg-store-node-test.jar" +cat > "$TEST_ROOT/java/bin/java" <<'JAVA' +#!/bin/bash +printf '%s\0' "$@" > "$CAPTURE.args" +printf '%s\n' "${TOPLING_RUNTIME_CLASSPATH:-}" > "$CAPTURE.runtime" +JAVA +cat > "$TEST_ROOT/fake-bin/uname" <<'UNAME' +#!/bin/bash +case "$1" in -m) echo x86_64 ;; *) echo Linux ;; esac +UNAME +chmod +x "$TEST_ROOT/java/bin/java" "$TEST_ROOT/fake-bin/uname" + +run_checker() { + env -u LD_PRELOAD -u TOPLING_ACTIVE_NATIVE -u TOPLING_RUNTIME_CLASSPATH \ + -u TOPLINGDB_EASY_MIGRATE_CONF -u TOPLING_AUTO_MIGRATE_CONF \ + JAVA_HOME="$TEST_ROOT/java" PATH="$TEST_ROOT/fake-bin:$PATH" \ + TOPLINGDB_ROCKSDB_PROVIDER="$1" CAPTURE="$TEST_ROOT/capture" \ + bash "$TOP/bin/hugegraph-store-check.sh" --check \ + --config "$TOP/conf" --data-path "$TOP/data with spaces" > "$TEST_ROOT/output" 2>&1 +} + +for provider in rocksdb topling; do + expected="$TOP/lib/hg-store-node-test.jar" + if [[ "$provider" == topling ]]; then + mkdir -p "$TOP/topling" + touch "$TOP/topling/rocksdbjni.jar" "$TOP/topling/librocksdbjni-linux64.so" + (cd "$TOP/topling" && sha256sum rocksdbjni.jar librocksdbjni-linux64.so > easy-migrate.sha256) + printf '%s\n' 'http: {auto_start_http: false}' > "$TOP/conf/toplingdb.yaml" + expected="$TOP/topling/rocksdbjni.jar:$expected" + fi + run_checker "$provider" || { cat "$TEST_ROOT/output"; exit 1; } + mapfile -d '' -t args < "$TEST_ROOT/capture.args" Review Comment: Addressed in follow-up #3473 (head 47fe808fa): the launcher fixture uses a Bash-compatible read loop in place of mapfile, and the fixture passed locally. The original PR head remains unchanged, so this thread remains open here. ########## hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map<String, Object> options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map<String, Long> ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List<Path> roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { + if (value.isBlank()) { + throw new IOException("Empty Store data path"); + } + Path root = Path.of(value).toAbsolutePath().normalize(); + if (offline && !Files.isDirectory(root)) { + throw new IOException("Expected an existing stopped Store data directory: " + root); + } + roots.add(root); + } + try (DatabaseLocks locks = new DatabaseLocks()) { + Path metadata = roots.get(0).resolve("hgstore-metadata"); + boolean metadataExists = Files.exists(metadata); + if (metadataExists) { + requireCurrent(metadata); + locks.add(metadata); + } + Set<Path> databases = new LinkedHashSet<>(); + Set<Path> snapshots = new LinkedHashSet<>(); + Set<Integer> discovered = new HashSet<>(); + for (Path root : roots) { + discover(root, null, databases, snapshots, discovered); + } + if (metadataExists) { + try (Database database = new Database(metadata, config); + RocksIterator iterator = database.db.newIterator()) { + byte[] prefix = MetadataKeyHelper.getPartitionStorePrefix(); + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + Metapb.PartitionStore location = Metapb.PartitionStore.parseFrom(iterator.value()); + // Normal moved roots are remapped by PartitionManager.resetPartitionPath(). + if (!discovered.contains(location.getPartitionId())) { + discover(Path.of(location.getStoreLocation()), location.getPartitionId(), + databases, snapshots, new HashSet<>()); + } + } + iterator.status(); + } + } + for (Path database : databases) { + requireCurrent(database); + locks.add(database); + checkDatabase(database, config); Review Comment: Still open: follow-up #3473 documents that representative large-Store startup-cost measurements have not been run. No performance conclusion is claimed. ########## hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommand.java: ########## @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.apache.hugegraph.util.RocksDBRuntime; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.FileSystemResource; + +/** Read-only command in the Store executable JAR. Does not start Spring, PD or a Raft node. */ +public final class GraphIdCheckCommand { + + private static final Set<String> VALUES = Set.of("--config", "--data-path"); + + private GraphIdCheckCommand() { + } + + public static void main(String[] args) { + System.exit(run(args)); + } + + static int run(String... args) { + try { + if (args.length == 1 && "--help".equals(args[0])) { + help(); + return 0; + } + if (args.length == 0) { + throw new IllegalArgumentException( + "Use --check to inspect a stopped Store, or --help"); + } + Map<String, String> arguments = new HashMap<>(); + for (int i = 0; i < args.length; i++) { + String name = args[i]; + String value = "true"; + if (VALUES.contains(name)) { + if (++i == args.length || args[i].startsWith("--") || args[i].isBlank()) { + throw new IllegalArgumentException("Missing value for " + name); + } + value = args[i]; + } else if (!"--check".equals(name)) { + throw new IllegalArgumentException("Unknown option " + name); + } + if (arguments.putIfAbsent(name, value) != null) { + throw new IllegalArgumentException("Repeated option " + name); + } + } + if (!arguments.containsKey("--check")) { + throw new IllegalArgumentException("Use --check to inspect a stopped Store, or --help"); + } + Properties properties = configuration(Path.of(arguments.getOrDefault("--config", "conf"))); + String data = arguments.getOrDefault("--data-path", System.getProperty("app.data-path", + properties.getProperty("app.data-path"))); + if (data == null || data.isBlank()) { + throw new IllegalArgumentException("Missing app.data-path; supply the effective --data-path"); + } + if (data.contains("${")) { + throw new IllegalArgumentException("Unresolved app.data-path; supply the effective --data-path"); + } + Map<String, Object> options = new HashMap<>(); + properties.forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + System.getProperties().forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + RocksDBRuntime.verify(String.valueOf(options.getOrDefault("rocksdb.provider", "rocksdb"))); + LegacyGraphIdChecker.checkOffline(data, LegacyGraphIdChecker.configuration(options)); + System.out.println("No legacy GraphID prefix or invalid GraphID mapping found in persisted " + + "databases and latest snapshots: " + data); Review Comment: Addressed in follow-up #3473 (head 47fe808fa): the command reports database and snapshot counts and explicitly warns if no business database or snapshot was inspected, including metadata-only roots. Empty-root and metadata-only command tests pass. The original PR head is unchanged, so this thread remains open here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
