imbajin commented on code in PR #3399: URL: https://github.com/apache/hugegraph/pull/3399#discussion_r4400775365
########## hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommand.java: ########## @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.apache.hugegraph.util.RocksDBRuntime; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.FileSystemResource; + +/** Read-only command in the Store executable JAR. Does not start Spring, PD or a Raft node. */ +public final class GraphIdCheckCommand { + + private static final Set<String> VALUES = Set.of("--config", "--data-path"); + + private GraphIdCheckCommand() { + } + + public static void main(String[] args) { + System.exit(run(args)); + } + + static int run(String... args) { + try { + if (args.length == 1 && "--help".equals(args[0])) { + help(); + return 0; + } + if (args.length == 0) { + throw new IllegalArgumentException( + "Use --check to inspect a stopped Store, or --help"); + } + Map<String, String> arguments = new HashMap<>(); + for (int i = 0; i < args.length; i++) { + String name = args[i]; + String value = "true"; + if (VALUES.contains(name)) { + if (++i == args.length || args[i].startsWith("--") || args[i].isBlank()) { + throw new IllegalArgumentException("Missing value for " + name); + } + value = args[i]; + } else if (!"--check".equals(name)) { + throw new IllegalArgumentException("Unknown option " + name); + } + if (arguments.putIfAbsent(name, value) != null) { + throw new IllegalArgumentException("Repeated option " + name); + } + } + if (!arguments.containsKey("--check")) { + throw new IllegalArgumentException("Use --check to inspect a stopped Store, or --help"); + } + Properties properties = configuration(Path.of(arguments.getOrDefault("--config", "conf"))); + String data = arguments.getOrDefault("--data-path", System.getProperty("app.data-path", + properties.getProperty("app.data-path"))); + if (data == null || data.isBlank()) { + throw new IllegalArgumentException("Missing app.data-path; supply the effective --data-path"); + } + if (data.contains("${")) { + throw new IllegalArgumentException("Unresolved app.data-path; supply the effective --data-path"); + } + Map<String, Object> options = new HashMap<>(); + properties.forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + System.getProperties().forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + RocksDBRuntime.verify(String.valueOf(options.getOrDefault("rocksdb.provider", "rocksdb"))); + LegacyGraphIdChecker.checkOffline(data, LegacyGraphIdChecker.configuration(options)); + System.out.println("No legacy GraphID prefix or invalid GraphID mapping found in persisted " + + "databases and latest snapshots: " + data); Review Comment: **Uncertain / Minor: report zero inspection coverage explicitly.** An existing empty directory, or an existing parent directory supplied by mistake, passes `checkOffline()` without inspecting any database or snapshot and then reaches this success message/exit 0. The explicit-empty-root test currently expects that result, so I cannot establish that accepting an empty Store is unintended. For an upgrade preflight, however, zero coverage is indistinguishable from a populated Store checked clean. Could the command print the database/snapshot counts and distinguish a zero-inspection result (or document that it is intentional)? The behavior is verified; whether zero coverage should fail remains an open contract question. ########## hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map<String, Object> options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map<String, Long> ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List<Path> roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { Review Comment: **Minor: preserve the existing `app.data-path` delimiter semantics.** For a previously accepted value such as `/a,/b,`, `split(",", -1)` produces a final empty item and this branch throws `Empty Store data path`. `checkStore()` wraps it as `StartupPreflightException`, so an otherwise healthy Store now refuses startup. The existing `StoreMetadata` constructor and `PartitionManager` still use `split(",")`, which discards trailing empty items. Please preserve their trailing-delimiter behavior, or explicitly document/migrate this stricter startup requirement, and add a regression test for a trailing comma. ########## hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh: ########## @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Launcher-selection fixtures; native database inspection is tested separately. +set -Eeuo pipefail +SOURCE_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/../static/bin" && pwd)" +REPO_ROOT="$(cd "$SOURCE_BIN/../../../../../.." && pwd)" +SHARED_BIN="$REPO_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin" +TEST_ROOT=$(mktemp -d) +TEST_ROOT=$(cd "$TEST_ROOT" && pwd -P) +trap 'rm -rf "$TEST_ROOT"' EXIT +TOP="$TEST_ROOT/checker distribution" +mkdir -p "$TOP/bin" "$TOP/lib" "$TOP/conf" "$TEST_ROOT/java/bin" "$TEST_ROOT/fake-bin" +cp "$SOURCE_BIN/hugegraph-store-check.sh" "$SHARED_BIN/preload-topling.sh" "$TOP/bin/" +printf '%s\n' '# module options' '--add-opens=java.base/java.lang=ALL-UNNAMED' > "$TOP/bin/jvm-module.options" +touch "$TOP/lib/hg-store-node-test.jar" +cat > "$TEST_ROOT/java/bin/java" <<'JAVA' +#!/bin/bash +printf '%s\0' "$@" > "$CAPTURE.args" +printf '%s\n' "${TOPLING_RUNTIME_CLASSPATH:-}" > "$CAPTURE.runtime" +JAVA +cat > "$TEST_ROOT/fake-bin/uname" <<'UNAME' +#!/bin/bash +case "$1" in -m) echo x86_64 ;; *) echo Linux ;; esac +UNAME +chmod +x "$TEST_ROOT/java/bin/java" "$TEST_ROOT/fake-bin/uname" + +run_checker() { + env -u LD_PRELOAD -u TOPLING_ACTIVE_NATIVE -u TOPLING_RUNTIME_CLASSPATH \ + -u TOPLINGDB_EASY_MIGRATE_CONF -u TOPLING_AUTO_MIGRATE_CONF \ + JAVA_HOME="$TEST_ROOT/java" PATH="$TEST_ROOT/fake-bin:$PATH" \ + TOPLINGDB_ROCKSDB_PROVIDER="$1" CAPTURE="$TEST_ROOT/capture" \ + bash "$TOP/bin/hugegraph-store-check.sh" --check \ + --config "$TOP/conf" --data-path "$TOP/data with spaces" > "$TEST_ROOT/output" 2>&1 +} + +for provider in rocksdb topling; do + expected="$TOP/lib/hg-store-node-test.jar" + if [[ "$provider" == topling ]]; then + mkdir -p "$TOP/topling" + touch "$TOP/topling/rocksdbjni.jar" "$TOP/topling/librocksdbjni-linux64.so" + (cd "$TOP/topling" && sha256sum rocksdbjni.jar librocksdbjni-linux64.so > easy-migrate.sha256) + printf '%s\n' 'http: {auto_start_http: false}' > "$TOP/conf/toplingdb.yaml" + expected="$TOP/topling/rocksdbjni.jar:$expected" + fi + run_checker "$provider" || { cat "$TEST_ROOT/output"; exit 1; } + mapfile -d '' -t args < "$TEST_ROOT/capture.args" Review Comment: **Note / unscored: this test fixture needs a newer Bash.** `mapfile` is unavailable in the Bash 3.2 bundled with macOS, so this fixture cannot run there as-is. Linux CI and the shipped checker launcher are unaffected. Please document the fixture's Bash requirement or use a portable reader if local macOS execution is intended. ########## hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map<String, Object> options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map<String, Long> ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List<Path> roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { + if (value.isBlank()) { + throw new IOException("Empty Store data path"); + } + Path root = Path.of(value).toAbsolutePath().normalize(); + if (offline && !Files.isDirectory(root)) { + throw new IOException("Expected an existing stopped Store data directory: " + root); + } + roots.add(root); + } + try (DatabaseLocks locks = new DatabaseLocks()) { + Path metadata = roots.get(0).resolve("hgstore-metadata"); + boolean metadataExists = Files.exists(metadata); + if (metadataExists) { + requireCurrent(metadata); + locks.add(metadata); + } + Set<Path> databases = new LinkedHashSet<>(); + Set<Path> snapshots = new LinkedHashSet<>(); + Set<Integer> discovered = new HashSet<>(); + for (Path root : roots) { + discover(root, null, databases, snapshots, discovered); + } + if (metadataExists) { + try (Database database = new Database(metadata, config); + RocksIterator iterator = database.db.newIterator()) { + byte[] prefix = MetadataKeyHelper.getPartitionStorePrefix(); + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + Metapb.PartitionStore location = Metapb.PartitionStore.parseFrom(iterator.value()); + // Normal moved roots are remapped by PartitionManager.resetPartitionPath(). + if (!discovered.contains(location.getPartitionId())) { + discover(Path.of(location.getStoreLocation()), location.getPartitionId(), + databases, snapshots, new HashSet<>()); + } + } + iterator.status(); + } + } + for (Path database : databases) { + requireCurrent(database); + locks.add(database); + checkDatabase(database, config); Review Comment: **Uncertain / unscored: startup cost at large partition counts.** This startup preflight opens each discovered database and snapshot for inspection. The code establishes that work is added to every boot, but no representative large-Store startup measurement was reported, so I cannot conclude that the cost is material. A measurement across realistic partition/version/snapshot counts would clarify the operational trade-off; this is not a confirmed performance defect or an additional scored finding. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
