nvharikrishna commented on code in PR #212: URL: https://github.com/apache/cassandra-sidecar/pull/212#discussion_r2111575134
########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); + dirIndexMap.put(instanceMetadata.hintsDir(), LIVE_MIGRATION_HINTS_DIR_PATH + "/0"); + dirIndexMap.put(instanceMetadata.commitlogDir(), LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"); + if (instanceMetadata.savedCachesDir() != null) + { + dirIndexMap.put(instanceMetadata.savedCachesDir(), LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"); + } + if (instanceMetadata.cdcDir() != null) + { + dirIndexMap.put(instanceMetadata.cdcDir(), LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirIndexMap.put(instanceMetadata.localSystemDataFileDir(), LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0"); + } + for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + { + dirIndexMap.put(instanceMetadata.dataDirs().get(i), LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i); + } + return dirIndexMap; + } + + /** + * Returns map of directory and set of placeholders that represent the directory. + */ + public static Map<String, Set<String>> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + { + Map<String, Set<String>> placeholderMap = new Hashtable<>(); Review Comment: HashMap allows null key and values. placeholderMap should not have either null key or null value. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); Review Comment: HashMap allows null key and values. dirIndexMap should not have either null key or null value. ########## server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtilTest.java: ########## @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.mockito.Mockito; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.localPath; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.entry; +import static org.mockito.Mockito.when; + +class LiveMigrationInstanceMetadataUtilTest +{ + private static final String DATA_DIR = "data"; + private static final String HINTS_DIR = "hints"; + private static final String COMMITLOG_DIR = "commitlog"; + private static final String SAVED_CACHES_DIR = "saved_caches"; + private static final String CDC_RAW_DIR = "cdc_raw"; + private static final String LOCAL_SYSTEM_DIR = "local_system_data"; + private static final String STAGING_DIR = "sstable-staging"; + + private static final String FILE_NAME = "file1.db"; + + @TempDir + Path tempDir; + + @Test + public void testDirsToCopy() + { + String cassandraHomeDir = tempDir.resolve("testDirsToCopy").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + List<String> dataDirs = new ArrayList<>(2); + String dataDir2 = DATA_DIR + "2"; + dataDirs.add(cassandraHomeDir + "/" + DATA_DIR); + dataDirs.add(cassandraHomeDir + "/" + dataDir2); + when(instanceMetadata.dataDirs()).thenReturn(dataDirs); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + + assertThat(dirsToCopy.size()).isEqualTo(7); + assertThat(dirsToCopy).contains(cassandraHomeDir + "/" + DATA_DIR) + .contains(cassandraHomeDir + "/" + dataDir2) + .contains(cassandraHomeDir + "/" + HINTS_DIR) + .contains(cassandraHomeDir + "/" + COMMITLOG_DIR) + .contains(cassandraHomeDir + "/" + SAVED_CACHES_DIR) + .contains(cassandraHomeDir + "/" + CDC_RAW_DIR) + .contains(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR); + } + + @Test + public void testDirsToCopyFewDirsNotConfigured() + { + String cassandraHomeDir = tempDir.resolve("testDirsToCopyFewDirsNotConfigured").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + when(instanceMetadata.cdcDir()).thenReturn(null); + when(instanceMetadata.localSystemDataFileDir()).thenReturn(null); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + + assertThat(dirsToCopy).hasSize(4); + + assertThat(dirsToCopy).doesNotContain(cassandraHomeDir + "/" + CDC_RAW_DIR) + .doesNotContain(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR); + } + + @Test + public void testDirPathPrefixMap() + { + String cassandraHomeDir = tempDir.resolve("testDirPathPrefixMap").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + List<String> dataDirs = new ArrayList<>(2); + String dataDir2 = DATA_DIR + "2"; + dataDirs.add(cassandraHomeDir + "/" + DATA_DIR); + dataDirs.add(cassandraHomeDir + "/" + dataDir2); + when(instanceMetadata.dataDirs()).thenReturn(dataDirs); + + Map<String, String> dirPathPrefixMap = LiveMigrationInstanceMetadataUtil.dirPathPrefixMap(instanceMetadata); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + assertThat(dirPathPrefixMap).hasSize(dirsToCopy.size()); + assertThat(dirPathPrefixMap.keySet()).containsAll(dirsToCopy); + assertThat(dirPathPrefixMap).contains( + entry(cassandraHomeDir + "/" + DATA_DIR, LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/0"), + entry(cassandraHomeDir + "/" + dataDir2, LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/1"), + entry(cassandraHomeDir + "/" + HINTS_DIR, LIVE_MIGRATION_HINTS_DIR_PATH + "/0"), + entry(cassandraHomeDir + "/" + COMMITLOG_DIR, LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"), + entry(cassandraHomeDir + "/" + SAVED_CACHES_DIR, LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"), + entry(cassandraHomeDir + "/" + CDC_RAW_DIR, LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"), + entry(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR, LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0") + ); + } + + @Test + public void testDirPathPrefixMapFewDirsNotConfigured() + { + String cassandraHomeDir = tempDir.resolve("testDirsToCopyFewDirsNotConfigured").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + when(instanceMetadata.cdcDir()).thenReturn(null); + when(instanceMetadata.localSystemDataFileDir()).thenReturn(null); + + Map<String, String> dirPathPrefixMap = LiveMigrationInstanceMetadataUtil.dirPathPrefixMap(instanceMetadata); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + assertThat(dirPathPrefixMap).hasSize(dirsToCopy.size()); + assertThat(dirPathPrefixMap.keySet()).containsAll(dirsToCopy); + assertThat(dirPathPrefixMap).doesNotContainKey(cassandraHomeDir + "/" + CDC_RAW_DIR) + .doesNotContainKey(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR); + } + + @Test + public void testDirPlaceholderMap() + { + String cassandraHomeDir = tempDir.resolve("testDirPlaceholderMap").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + List<String> dataDirs = new ArrayList<>(2); + String dataDir2 = DATA_DIR + "2"; + dataDirs.add(cassandraHomeDir + "/" + DATA_DIR); + dataDirs.add(cassandraHomeDir + "/" + dataDir2); + when(instanceMetadata.dataDirs()).thenReturn(dataDirs); + + Map<String, Set<String>> dirPathPrefixMap = LiveMigrationInstanceMetadataUtil.dirPlaceHoldersMap(instanceMetadata); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + assertThat(dirPathPrefixMap).hasSize(dirsToCopy.size()); + assertThat(dirPathPrefixMap.keySet()).containsAll(dirsToCopy); + assertThat(dirPathPrefixMap).contains( + + entry(cassandraHomeDir + "/" + DATA_DIR, new HashSet<>() + {{ + add(DATA_FILE_DIR_PLACEHOLDER); + add(DATA_FILE_DIR_PLACEHOLDER + "_" + 0); + }}), + entry(cassandraHomeDir + "/" + dataDir2, new HashSet<>() + {{ + add(DATA_FILE_DIR_PLACEHOLDER); + add(DATA_FILE_DIR_PLACEHOLDER + "_" + 1); + }}), + entry(cassandraHomeDir + "/" + HINTS_DIR, Collections.singleton(HINTS_DIR_PLACEHOLDER)), + entry(cassandraHomeDir + "/" + COMMITLOG_DIR, Collections.singleton(COMMITLOG_DIR_PLACEHOLDER)), + entry(cassandraHomeDir + "/" + SAVED_CACHES_DIR, Collections.singleton(SAVED_CACHES_DIR_PLACEHOLDER)), + entry(cassandraHomeDir + "/" + CDC_RAW_DIR, Collections.singleton(CDC_RAW_DIR_PLACEHOLDER)), + entry(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR, Collections.singleton(LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER)) + + ); + } + + @Test + public void testDirPlaceholderMapFewDirsNotConfigured() + { + String cassandraHomeDir = tempDir.resolve("testDirPlaceholderMapFewDirsNotConfigured").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + when(instanceMetadata.cdcDir()).thenReturn(null); + when(instanceMetadata.localSystemDataFileDir()).thenReturn(null); + + Map<String, Set<String>> dirPathPrefixMap = LiveMigrationInstanceMetadataUtil.dirPlaceHoldersMap(instanceMetadata); + + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + assertThat(dirPathPrefixMap).hasSize(dirsToCopy.size()); + assertThat(dirPathPrefixMap.keySet()).containsAll(dirsToCopy); + + assertThat(dirPathPrefixMap).doesNotContainKey(cassandraHomeDir + "/" + CDC_RAW_DIR) + .doesNotContainKey(cassandraHomeDir + "/" + LOCAL_SYSTEM_DIR); + } + + + @Test + public void testLocalPath() + { + String cassandraHomeDir = tempDir.resolve("testLocalPath").toString(); + InstanceMetadata instanceMetadata = getInstanceMetadata(cassandraHomeDir); + + validateLocalPath(instanceMetadata.dataDirs().get(0) + "/" + FILE_NAME, Review Comment: I am defending it in the file stream API (not part of this PR), but make sense to handle it as part of this utility too. Updated it. Added test cases too. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationPlaceholderUtil.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jetbrains.annotations.Nullable; + +/** + * Utility class for replacing placeholder in Live Migration exclusion paths. + * <p> + * A placeholder looks like this: "${PLACEHOLDER}/*\/*\/*-Data.db". + */ +public class LiveMigrationPlaceholderUtil +{ + + public static final String DATA_FILE_DIR_PLACEHOLDER = "DATA_FILE_DIR"; + public static final String COMMITLOG_DIR_PLACEHOLDER = "COMMITLOG_DIR"; + public static final String HINTS_DIR_PLACEHOLDER = "HINTS_DIR"; + public static final String SAVED_CACHES_DIR_PLACEHOLDER = "SAVED_CACHES_DIR"; + public static final String CDC_RAW_DIR_PLACEHOLDER = "CDC_RAW_DIR"; + public static final String LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER = "LOCAL_SYSTEM_DATA_FILE_DIR"; + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{(\\w+)}.*"); + + /** + * Replaces a placeholder with a value specified in given input. There can be more than one placeholder + * pointing to singe value. + * + * @param input input string having placeholder to replace + * @param placeholders set of placeholders to search for + * @param value value with which placeholder will be replaced + * @return input unchanged if no placeholder is found and input with placeholder replaced if placeholder is in given + * placeholders, otherwise null. + */ + @Nullable + public static String replacePlaceholder(String input, Set<String> placeholders, String value) + { + String placeholder = getPlaceholder(input); + + if (placeholder == null) + { + return input; + } + if (placeholders.contains(placeholder)) + { + return input.replace("${" + placeholder + "}", value); + } + + return null; + } + + public static boolean hasAnyPlaceholder(String input) + { + String placeholder = getPlaceholder(input); + return placeholder != null; + } + + public static boolean hasPlaceholder(String input, Set<String> knownPlaceHolders) Review Comment: Done. Also renamed the method to "hasAnyPlaceholder" ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/MigrationFileVisitor.java: ########## @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.jetbrains.annotations.NotNull; + +/** + * File visitor to walk through given directory. + */ +public class MigrationFileVisitor extends SimpleFileVisitor<Path> +{ + private static final Logger LOGGER = LoggerFactory.getLogger(MigrationFileVisitor.class); + private final String homeDir; + private final List<Path> validFiles; + private final Set<PathMatcher> filesToExclude; + private final Set<PathMatcher> directoriesToExclude; + + public MigrationFileVisitor(@NotNull final String homeDir, + @NotNull final Set<PathMatcher> filesToExcludeMatchers, + @NotNull final Set<PathMatcher> directoriesToExcludeMatchers) + { + this.homeDir = homeDir; + this.validFiles = new ArrayList<>(); + this.filesToExclude = filesToExcludeMatchers; + this.directoriesToExclude = directoriesToExcludeMatchers; + } + + private boolean shouldExcludeDir(Path dir) + { + Objects.requireNonNull(dir); + return directoriesToExclude.stream().anyMatch(matcher -> matcher.matches(dir)); + } + + private boolean shouldExcludeFile(Path file) + { + Objects.requireNonNull(file); + return filesToExclude.stream().anyMatch(matcher -> matcher.matches(file)); + } + + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + { + if (shouldExcludeDir(dir)) + { + return FileVisitResult.SKIP_SUBTREE; + } + + if (!dir.toAbsolutePath().toString().equals(homeDir)) + { + validFiles.add(dir); + } + return FileVisitResult.CONTINUE; + } + + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); + dirIndexMap.put(instanceMetadata.hintsDir(), LIVE_MIGRATION_HINTS_DIR_PATH + "/0"); + dirIndexMap.put(instanceMetadata.commitlogDir(), LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"); + if (instanceMetadata.savedCachesDir() != null) + { + dirIndexMap.put(instanceMetadata.savedCachesDir(), LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"); + } + if (instanceMetadata.cdcDir() != null) + { + dirIndexMap.put(instanceMetadata.cdcDir(), LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirIndexMap.put(instanceMetadata.localSystemDataFileDir(), LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0"); + } + for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + { + dirIndexMap.put(instanceMetadata.dataDirs().get(i), LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i); + } + return dirIndexMap; + } + + /** + * Returns map of directory and set of placeholders that represent the directory. + */ + public static Map<String, Set<String>> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + { + Map<String, Set<String>> placeholderMap = new Hashtable<>(); + + placeholderMap.put(instanceMetadata.hintsDir(), Collections.singleton(HINTS_DIR_PLACEHOLDER)); + placeholderMap.put(instanceMetadata.commitlogDir(), Collections.singleton(COMMITLOG_DIR_PLACEHOLDER)); + if (instanceMetadata.savedCachesDir() != null) + { + placeholderMap.put(instanceMetadata.savedCachesDir(), Collections.singleton(SAVED_CACHES_DIR_PLACEHOLDER)); + } + + if (instanceMetadata.cdcDir() != null) + { + placeholderMap.put(instanceMetadata.cdcDir(), Collections.singleton(CDC_RAW_DIR_PLACEHOLDER)); + } + + if (instanceMetadata.localSystemDataFileDir() != null) + { + placeholderMap.put(instanceMetadata.localSystemDataFileDir(), + Collections.singleton(LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER)); + } + + List<String> dataDirs = instanceMetadata.dataDirs(); + for (int i = 0; i < dataDirs.size(); i++) + { + String dir = dataDirs.get(i); + Set<String> placeholders = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(DATA_FILE_DIR_PLACEHOLDER, DATA_FILE_DIR_PLACEHOLDER + "_" + i))); Review Comment: Thanks! Just realised that JDK 8 support is dropped for server. Done. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); + dirIndexMap.put(instanceMetadata.hintsDir(), LIVE_MIGRATION_HINTS_DIR_PATH + "/0"); + dirIndexMap.put(instanceMetadata.commitlogDir(), LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"); + if (instanceMetadata.savedCachesDir() != null) + { + dirIndexMap.put(instanceMetadata.savedCachesDir(), LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"); + } + if (instanceMetadata.cdcDir() != null) + { + dirIndexMap.put(instanceMetadata.cdcDir(), LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirIndexMap.put(instanceMetadata.localSystemDataFileDir(), LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0"); + } + for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + { + dirIndexMap.put(instanceMetadata.dataDirs().get(i), LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i); + } + return dirIndexMap; + } + + /** + * Returns map of directory and set of placeholders that represent the directory. + */ + public static Map<String, Set<String>> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + { + Map<String, Set<String>> placeholderMap = new Hashtable<>(); + + placeholderMap.put(instanceMetadata.hintsDir(), Collections.singleton(HINTS_DIR_PLACEHOLDER)); + placeholderMap.put(instanceMetadata.commitlogDir(), Collections.singleton(COMMITLOG_DIR_PLACEHOLDER)); + if (instanceMetadata.savedCachesDir() != null) + { + placeholderMap.put(instanceMetadata.savedCachesDir(), Collections.singleton(SAVED_CACHES_DIR_PLACEHOLDER)); + } + + if (instanceMetadata.cdcDir() != null) + { + placeholderMap.put(instanceMetadata.cdcDir(), Collections.singleton(CDC_RAW_DIR_PLACEHOLDER)); + } + + if (instanceMetadata.localSystemDataFileDir() != null) + { + placeholderMap.put(instanceMetadata.localSystemDataFileDir(), + Collections.singleton(LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER)); + } + + List<String> dataDirs = instanceMetadata.dataDirs(); + for (int i = 0; i < dataDirs.size(); i++) + { + String dir = dataDirs.get(i); + Set<String> placeholders = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(DATA_FILE_DIR_PLACEHOLDER, DATA_FILE_DIR_PLACEHOLDER + "_" + i))); + placeholderMap.put(dir, placeholders); + } + + return placeholderMap; + } + + /** + * Converts given live migration file download URL to local path. + * + * @param fileUrl Live migration file download URL + * @param metadata Cassandra instance metadata + * @return local path for given live migration file download URL + */ + public static String localPath(@NotNull String fileUrl, + @NotNull InstanceMetadata metadata) + { + + Map<String, String> urlToLocalDirMap = migrationUrlLocalDirMap(metadata); + for (Map.Entry<String, String> entry : urlToLocalDirMap.entrySet()) + { + if (fileUrl.startsWith(entry.getKey())) + { + assert entry.getValue() != null : "No local path found for url " + fileUrl; Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.config.yaml; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; + +/** + * Configuration required for live migrating the Cassandra instances. + */ +public class LiveMigrationConfigurationImpl implements LiveMigrationConfiguration +{ + + private final Set<String> filesToExclude; + private final Set<String> directoriesToExclude; + private final Map<String, String> migrationMap; + + public LiveMigrationConfigurationImpl() + { + this(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()); + } + + @JsonCreator + public LiveMigrationConfigurationImpl(@JsonProperty("files_to_exclude") Set<String> filesToExclude, + @JsonProperty("dirs_to_exclude") Set<String> directoriesToExclude, + @JsonProperty("migration_map") Map<String, String> migrationMap) + { + this.filesToExclude = filesToExclude; + this.directoriesToExclude = directoriesToExclude; + this.migrationMap = migrationMap; + } + + @JsonProperty("files_to_exclude") + public Set<String> filesToExclude() + { + return filesToExclude; + } + + @JsonProperty("dirs_to_exclude") + public Set<String> directoriesToExclude() + { + return directoriesToExclude; + } + + @JsonProperty("migration_map") + public Map<String, String> migrationMap() + { + return migrationMap; Review Comment: Done while initializing member variable itself. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationPlaceholderUtil.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jetbrains.annotations.Nullable; + +/** + * Utility class for replacing placeholder in Live Migration exclusion paths. + * <p> + * A placeholder looks like this: "${PLACEHOLDER}/*\/*\/*-Data.db". + */ +public class LiveMigrationPlaceholderUtil +{ + + public static final String DATA_FILE_DIR_PLACEHOLDER = "DATA_FILE_DIR"; + public static final String COMMITLOG_DIR_PLACEHOLDER = "COMMITLOG_DIR"; + public static final String HINTS_DIR_PLACEHOLDER = "HINTS_DIR"; + public static final String SAVED_CACHES_DIR_PLACEHOLDER = "SAVED_CACHES_DIR"; + public static final String CDC_RAW_DIR_PLACEHOLDER = "CDC_RAW_DIR"; + public static final String LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER = "LOCAL_SYSTEM_DATA_FILE_DIR"; + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{(\\w+)}.*"); + + /** + * Replaces a placeholder with a value specified in given input. There can be more than one placeholder + * pointing to singe value. + * + * @param input input string having placeholder to replace + * @param placeholders set of placeholders to search for + * @param value value with which placeholder will be replaced + * @return input unchanged if no placeholder is found and input with placeholder replaced if placeholder is in given + * placeholders, otherwise null. + */ + @Nullable + public static String replacePlaceholder(String input, Set<String> placeholders, String value) + { + String placeholder = getPlaceholder(input); + + if (placeholder == null) + { + return input; + } + if (placeholders.contains(placeholder)) + { + return input.replace("${" + placeholder + "}", value); + } + + return null; + } + + public static boolean hasAnyPlaceholder(String input) Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/MigrationFileVisitor.java: ########## @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.jetbrains.annotations.NotNull; + +/** + * File visitor to walk through given directory. + */ +public class MigrationFileVisitor extends SimpleFileVisitor<Path> +{ + private static final Logger LOGGER = LoggerFactory.getLogger(MigrationFileVisitor.class); + private final String homeDir; + private final List<Path> validFiles; + private final Set<PathMatcher> filesToExclude; + private final Set<PathMatcher> directoriesToExclude; + + public MigrationFileVisitor(@NotNull final String homeDir, + @NotNull final Set<PathMatcher> filesToExcludeMatchers, + @NotNull final Set<PathMatcher> directoriesToExcludeMatchers) + { + this.homeDir = homeDir; + this.validFiles = new ArrayList<>(); + this.filesToExclude = filesToExcludeMatchers; + this.directoriesToExclude = directoriesToExcludeMatchers; + } + + private boolean shouldExcludeDir(Path dir) + { + Objects.requireNonNull(dir); + return directoriesToExclude.stream().anyMatch(matcher -> matcher.matches(dir)); + } + + private boolean shouldExcludeFile(Path file) + { + Objects.requireNonNull(file); + return filesToExclude.stream().anyMatch(matcher -> matcher.matches(file)); + } + + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + { + if (shouldExcludeDir(dir)) + { + return FileVisitResult.SKIP_SUBTREE; + } + + if (!dir.toAbsolutePath().toString().equals(homeDir)) + { + validFiles.add(dir); Review Comment: visitFile is only for files, want to add directory, so adding it in `preVisitDirectory`. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImpl.java: ########## @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; + +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.hasAnyPlaceholder; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.hasPlaceholder; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.replacePlaceholder; + +/** + * Helper class to get the list of files to use during Live Migration. + */ +public class CassandraInstanceFilesImpl implements CassandraInstanceFiles +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(CassandraInstanceFilesImpl.class); + + private final InstanceMetadata instanceMetadata; + + private final LiveMigrationConfiguration configuration; + + public CassandraInstanceFilesImpl(InstanceMetadata instanceMetadata, + LiveMigrationConfiguration configuration) + { + this.instanceMetadata = instanceMetadata; + this.configuration = configuration; + } + + @Override + public List<InstanceFileInfo> getFiles() throws IOException + { + return getFiles(configuration.filesToExclude(), configuration.directoriesToExclude()); + } + + private List<InstanceFileInfo> getFiles(Set<String> filesToExclude, + Set<String> dirsToExclude) throws IOException + { + List<DirVisitor> dirVisitors = getDirVisitorList(filesToExclude, dirsToExclude); + List<InstanceFileInfo> instanceFileInfos = new ArrayList<>(); + + for (DirVisitor dirVisitor : dirVisitors) + { + instanceFileInfos.addAll(dirVisitor.getFiles()); + } + + LOGGER.info("{} number of files are getting listed", instanceFileInfos.size()); + return instanceFileInfos; + } + + public List<DirVisitor> getDirVisitorList(Set<String> filesToExclude, + Set<String> dirsToExclude) + { + List<DirVisitor> dataFilesToVisit = new ArrayList<>(); + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + Map<String, String> dirPathPrefix = LiveMigrationInstanceMetadataUtil.dirPathPrefixMap(instanceMetadata); + Map<String, Set<String>> dirPlaceholderMap = LiveMigrationInstanceMetadataUtil.dirPlaceHoldersMap(instanceMetadata); + + for (String dir : dirsToCopy) + { + getDirToVisit(dir, + dirPlaceholderMap.get(dir), + dirPathPrefix.get(dir), + filesToExclude, + dirsToExclude) + .ifPresent(dataFilesToVisit::add); + } + + return dataFilesToVisit; + } + + Optional<DirVisitor> getDirToVisit(String homeDir, Set<String> placeholders, String pathPrefix, + Set<String> filesToExclude, Set<String> dirsToExclude) + { + if (null == homeDir) + { + return Optional.empty(); + } + Path homeDirPath = Paths.get(homeDir); + if (!Files.exists(homeDirPath)) + { + return Optional.empty(); + } + + Objects.requireNonNull(placeholders); + Objects.requireNonNull(pathPrefix); Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImpl.java: ########## @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; + +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.hasAnyPlaceholder; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.hasPlaceholder; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.replacePlaceholder; + +/** + * Helper class to get the list of files to use during Live Migration. + */ +public class CassandraInstanceFilesImpl implements CassandraInstanceFiles +{ + + private static final Logger LOGGER = LoggerFactory.getLogger(CassandraInstanceFilesImpl.class); + + private final InstanceMetadata instanceMetadata; + + private final LiveMigrationConfiguration configuration; + + public CassandraInstanceFilesImpl(InstanceMetadata instanceMetadata, + LiveMigrationConfiguration configuration) + { + this.instanceMetadata = instanceMetadata; + this.configuration = configuration; + } + + @Override + public List<InstanceFileInfo> getFiles() throws IOException + { + return getFiles(configuration.filesToExclude(), configuration.directoriesToExclude()); + } + + private List<InstanceFileInfo> getFiles(Set<String> filesToExclude, + Set<String> dirsToExclude) throws IOException + { + List<DirVisitor> dirVisitors = getDirVisitorList(filesToExclude, dirsToExclude); + List<InstanceFileInfo> instanceFileInfos = new ArrayList<>(); + + for (DirVisitor dirVisitor : dirVisitors) + { + instanceFileInfos.addAll(dirVisitor.getFiles()); + } + + LOGGER.info("{} number of files are getting listed", instanceFileInfos.size()); + return instanceFileInfos; + } + + public List<DirVisitor> getDirVisitorList(Set<String> filesToExclude, + Set<String> dirsToExclude) + { + List<DirVisitor> dataFilesToVisit = new ArrayList<>(); + List<String> dirsToCopy = LiveMigrationInstanceMetadataUtil.dirsToCopy(instanceMetadata); + Map<String, String> dirPathPrefix = LiveMigrationInstanceMetadataUtil.dirPathPrefixMap(instanceMetadata); + Map<String, Set<String>> dirPlaceholderMap = LiveMigrationInstanceMetadataUtil.dirPlaceHoldersMap(instanceMetadata); + + for (String dir : dirsToCopy) + { + getDirToVisit(dir, + dirPlaceholderMap.get(dir), + dirPathPrefix.get(dir), + filesToExclude, + dirsToExclude) + .ifPresent(dataFilesToVisit::add); + } + + return dataFilesToVisit; + } + + Optional<DirVisitor> getDirToVisit(String homeDir, Set<String> placeholders, String pathPrefix, + Set<String> filesToExclude, Set<String> dirsToExclude) + { + if (null == homeDir) + { + return Optional.empty(); + } + Path homeDirPath = Paths.get(homeDir); + if (!Files.exists(homeDirPath)) + { + return Optional.empty(); Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/DirVisitor.java: ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.jetbrains.annotations.NotNull; + +/** + * Explores single directory of a Cassandra Instance. + */ +public class DirVisitor +{ + private static final Logger LOGGER = LoggerFactory.getLogger(DirVisitor.class); + private final String homeDir; + private final Path homeDirPath; + private final String pathPrefix; + private final MigrationFileVisitor fileVisitor; + + public DirVisitor(String homeDir, + String pathPrefix, + Set<PathMatcher> fileExclusionMatchers, + Set<PathMatcher> dirExclusionMatchers) + { + this.homeDir = homeDir; + this.homeDirPath = Paths.get(homeDir); + this.pathPrefix = pathPrefix; + this.fileVisitor = new MigrationFileVisitor(homeDir, fileExclusionMatchers, dirExclusionMatchers); + } + + /** + * Returns list of files and directories (urls) that can be downloaded by destination which is trying + * to clone current instance. + * + * @return List of {@link InstanceFileInfo} for each file and directory considered for Live Migration. + * @throws IOException - when cannot access files of a data home directory + */ + public List<InstanceFileInfo> getFiles() throws IOException + { + Files.walkFileTree(homeDirPath, fileVisitor); + List<Path> validFiles = fileVisitor.getValidFilePaths(); + final List<InstanceFileInfo> infos = new ArrayList<>(); + for (final Path path : validFiles) + { + infos.add(toInstanceFileInfo(path)); + } + LOGGER.debug("Found {} instance files for homeDir {}", infos.size(), homeDir); + return infos; + } + + private InstanceFileInfo toInstanceFileInfo(@NotNull Path path) throws IOException + { + long lastModifiedTime = Files.getLastModifiedTime(path).toMillis(); + String fileUrl = getInstanceFileUrl(path); + InstanceFileInfo.FileType fileType = Files.isDirectory(path) ? Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/DirVisitor.java: ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.jetbrains.annotations.NotNull; + +/** + * Explores single directory of a Cassandra Instance. + */ +public class DirVisitor +{ + private static final Logger LOGGER = LoggerFactory.getLogger(DirVisitor.class); + private final String homeDir; + private final Path homeDirPath; + private final String pathPrefix; + private final MigrationFileVisitor fileVisitor; + + public DirVisitor(String homeDir, + String pathPrefix, + Set<PathMatcher> fileExclusionMatchers, + Set<PathMatcher> dirExclusionMatchers) + { + this.homeDir = homeDir; + this.homeDirPath = Paths.get(homeDir); + this.pathPrefix = pathPrefix; + this.fileVisitor = new MigrationFileVisitor(homeDir, fileExclusionMatchers, dirExclusionMatchers); + } + + /** + * Returns list of files and directories (urls) that can be downloaded by destination which is trying + * to clone current instance. + * + * @return List of {@link InstanceFileInfo} for each file and directory considered for Live Migration. + * @throws IOException - when cannot access files of a data home directory + */ + public List<InstanceFileInfo> getFiles() throws IOException + { + Files.walkFileTree(homeDirPath, fileVisitor); + List<Path> validFiles = fileVisitor.getValidFilePaths(); + final List<InstanceFileInfo> infos = new ArrayList<>(); + for (final Path path : validFiles) + { + infos.add(toInstanceFileInfo(path)); + } + LOGGER.debug("Found {} instance files for homeDir {}", infos.size(), homeDir); + return infos; + } + + private InstanceFileInfo toInstanceFileInfo(@NotNull Path path) throws IOException + { + long lastModifiedTime = Files.getLastModifiedTime(path).toMillis(); + String fileUrl = getInstanceFileUrl(path); + InstanceFileInfo.FileType fileType = Files.isDirectory(path) ? + InstanceFileInfo.FileType.DIRECTORY + : InstanceFileInfo.FileType.FILE; + // 'size' doesn't have any significance for directories. Hence, setting it to -1 explicitly. + long size = Files.isDirectory(path) ? -1 : Files.size(path); + + return new InstanceFileInfo(fileUrl, size, fileType, lastModifiedTime); + } + + /** + * Constructs live migration URL path for given file. + * + * @param path file for which URL needs to be constructed. + * @return URL path for given file + */ + private String getInstanceFileUrl(@NotNull Path path) Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationPlaceholderUtil.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jetbrains.annotations.Nullable; + +/** + * Utility class for replacing placeholder in Live Migration exclusion paths. + * <p> + * A placeholder looks like this: "${PLACEHOLDER}/*\/*\/*-Data.db". + */ +public class LiveMigrationPlaceholderUtil +{ + + public static final String DATA_FILE_DIR_PLACEHOLDER = "DATA_FILE_DIR"; + public static final String COMMITLOG_DIR_PLACEHOLDER = "COMMITLOG_DIR"; + public static final String HINTS_DIR_PLACEHOLDER = "HINTS_DIR"; + public static final String SAVED_CACHES_DIR_PLACEHOLDER = "SAVED_CACHES_DIR"; + public static final String CDC_RAW_DIR_PLACEHOLDER = "CDC_RAW_DIR"; + public static final String LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER = "LOCAL_SYSTEM_DATA_FILE_DIR"; + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{(\\w+)}.*"); + + /** + * Replaces a placeholder with a value specified in given input. There can be more than one placeholder + * pointing to singe value. + * + * @param input input string having placeholder to replace + * @param placeholders set of placeholders to search for + * @param value value with which placeholder will be replaced + * @return input unchanged if no placeholder is found and input with placeholder replaced if placeholder is in given + * placeholders, otherwise null. + */ + @Nullable + public static String replacePlaceholder(String input, Set<String> placeholders, String value) + { + String placeholder = getPlaceholder(input); + + if (placeholder == null) + { + return input; + } + if (placeholders.contains(placeholder)) + { + return input.replace("${" + placeholder + "}", value); + } + + return null; + } + + public static boolean hasAnyPlaceholder(String input) + { + String placeholder = getPlaceholder(input); + return placeholder != null; + } + + public static boolean hasPlaceholder(String input, Set<String> knownPlaceHolders) + { + String placeholder = getPlaceholder(input); + return placeholder != null && knownPlaceHolders.contains(placeholder); + } + + private static @Nullable String getPlaceholder(String input) + { + Matcher matcher = PLACEHOLDER_PATTERN.matcher(input); + if (matcher.find()) + { + return matcher.group(1); Review Comment: input can have only one placeholder -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For additional commands, e-mail: pr-h...@cassandra.apache.org
