nvharikrishna commented on code in PR #212: URL: https://github.com/apache/cassandra-sidecar/pull/212#discussion_r2116686926
########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); Review Comment: Ran into null keys while unit testing (as some of the directories are optional). Changed it from HashMap to HashTable to catch null keys early. Null checks are already added to optional values. Changed it to HashMap and wrapped it with unmodifiable before returning. ########## client-common/src/main/java/org/apache/cassandra/sidecar/common/ApiEndpointsV1.java: ########## @@ -141,6 +141,27 @@ public final class ApiEndpointsV1 public static final String STREAM_STATS_ROUTE = API_V1 + CASSANDRA + "/stats/streams"; public static final String TABLE_STATS_ROUTE = API_V1 + CASSANDRA + PER_KEYSPACE + PER_TABLE + "/stats"; + + // Live Migration APIs + public static final String LIVE_MIGRATION_API_PREFIX = API_V1 + "/live-migration"; + + public static final String LIVE_MIGRATION_LIST_FILES_PATH = "/files-list"; Review Comment: done ########## client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFilesListResponse.java: ########## @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.cassandra.sidecar.common.ApiEndpointsV1; +import org.jetbrains.annotations.NotNull; + +/** + * Class representing response for {@link ApiEndpointsV1#LIVE_MIGRATION_LIST_FILES_API}. + * Holds list of file urls (with their individual sizes) + total size of files + * <p> + * Sample format: + * <pre> + * { + * "totalSize": 123456789, + * "files" : [ + * { + * "fileUrl": "/api/v1/live-migration/files/{type_of_directory}/{directory_index}/{relative_file_path}", + * "size": 123456789, + * "fileType": "FILE", + * "lastModifiedTime": 1707193655406 + * }, + * + * { + * "fileUrl": "/api/v1/live-migration/files/{type_of_directory}/{directory_index}/{relative_file_path}", + * "size": -1, + * "fileType": "DIRECTORY", + * "lastModifiedTime": 1707798915999 + * } + * ] + * } + * </pre> + */ +public class InstanceFilesListResponse +{ + private final List<InstanceFileInfo> files; Review Comment: Initializing with unmodifiable list in the constructor. ########## client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFilesListResponse.java: ########## @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.cassandra.sidecar.common.ApiEndpointsV1; +import org.jetbrains.annotations.NotNull; + +/** + * Class representing response for {@link ApiEndpointsV1#LIVE_MIGRATION_LIST_FILES_API}. + * Holds list of file urls (with their individual sizes) + total size of files + * <p> + * Sample format: + * <pre> + * { + * "totalSize": 123456789, + * "files" : [ + * { + * "fileUrl": "/api/v1/live-migration/files/{type_of_directory}/{directory_index}/{relative_file_path}", + * "size": 123456789, + * "fileType": "FILE", + * "lastModifiedTime": 1707193655406 + * }, + * + * { + * "fileUrl": "/api/v1/live-migration/files/{type_of_directory}/{directory_index}/{relative_file_path}", + * "size": -1, + * "fileType": "DIRECTORY", + * "lastModifiedTime": 1707798915999 + * } + * ] + * } + * </pre> + */ +public class InstanceFilesListResponse +{ + private final List<InstanceFileInfo> files; + private final long totalSize; + + public InstanceFilesListResponse(@NotNull final List<InstanceFileInfo> files) + { + this.files = files; + long sum = 0L; + for (InstanceFileInfo file : files) + { + if (file.fileType.equals(InstanceFileInfo.FileType.FILE)) + { + long size = file.size; + sum += size; + } + } + totalSize = sum; + } + + @JsonCreator + public InstanceFilesListResponse(@JsonProperty("files") final List<InstanceFileInfo> files, + @JsonProperty("totalSize") final long totalSize) Review Comment: done ########## client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java: ########## @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Class for holding information of a downloadable file during Live Migration. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class InstanceFileInfo +{ + public final String fileUrl; + // size is only relevant when the fileType is 'FILE' + public final long size; Review Comment: done ########## conf/sidecar.yaml: ########## @@ -362,3 +362,10 @@ s3_client: # uri: # username: # password: + +live_migration: + special_files_to_exclude: [] + special_dirs_to_exclude: + - ${DATA_FILE_DIR}/*/*/snapshots # Excludes snapshot directories to copy to destination + migration_map: + localhost1: localhost4 Review Comment: done, also comment this entry as it enables migration. ########## server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java: ########## @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.config; + +import java.util.Map; +import java.util.Set; + +/** + * Configuration for Live Migration feature. + */ +public interface LiveMigrationConfiguration +{ + + /** + * Files to be excluded from Live Migration. + * @return set of file exclusion patterns. + */ + Set<String> filesToExclude(); + + /** + * Directories to be excluded from Live Migration. + * @return set of directory exclusion patterns. + */ + Set<String> directoriesToExclude(); + + /** + * Map of source and destination Cassandra instances to migrate. + * + * @return Map of strings where key is the source instance name and value is the destination instance name. Review Comment: yes, updated it ########## server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java: ########## @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.config; + +import java.util.Map; +import java.util.Set; + +/** + * Configuration for Live Migration feature. + */ +public interface LiveMigrationConfiguration +{ + + /** + * Files to be excluded from Live Migration. + * @return set of file exclusion patterns. + */ + Set<String> filesToExclude(); + + /** + * Directories to be excluded from Live Migration. + * @return set of directory exclusion patterns. + */ + Set<String> directoriesToExclude(); + + /** + * Map of source and destination Cassandra instances to migrate. + * + * @return Map of strings where key is the source instance name and value is the destination instance name. + */ + Map<String, String> migrationMap(); +} Review Comment: This configuration cannot be per task. Per task - could mean that one source host can be specified in one request and another source in the next request. The map should be determined by the admin/operator but not the caller. Accepting remote host as part of the migration request becomes security risk. Cassandra won't be running when the sidecar is started at the destination. So it cannot fetch sidecar internal keyspace for the config. Does sidecar running in one host talks to Cassandra running in other hosts? I would say this configuration is per cluster as it tells which sidecars are going to allow these endpoints/migration and which sidecars are not going to allow. > If the live migration configuration is expressed in the form of yaml config, it would require rolling restart of sidecar to pick up the configurations. Agree that it would be painful, but I couldn't see any other way at the moment. Probably, sidecar supporting hot reloading of configuration would help. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java: ########## @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.HINTS_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.SAVED_CACHES_DIR_PLACEHOLDER; + +/** + * Utility class for having all {@link InstanceMetadata} related helper functions related to + * Live Migration in one place. + */ +@SuppressWarnings("ConstantValue") +public class LiveMigrationInstanceMetadataUtil +{ + + /** + * List of directories Live Migration considers to transfer between source and destination. + */ + public static List<String> dirsToCopy(InstanceMetadata instanceMetadata) + { + List<String> dirsToCopy = new ArrayList<>(); + dirsToCopy.add(instanceMetadata.hintsDir()); + dirsToCopy.add(instanceMetadata.commitlogDir()); + if (instanceMetadata.savedCachesDir() != null) + { + dirsToCopy.add(instanceMetadata.savedCachesDir()); + } + if (instanceMetadata.cdcDir() != null) + { + dirsToCopy.add(instanceMetadata.cdcDir()); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); + } + dirsToCopy.addAll(instanceMetadata.dataDirs()); + return dirsToCopy; + } + + /** + * Returns map of directory that can be copied and the index to use while constructing + * url for file transfer. + */ + public static Map<String, String> dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map<String, String> dirIndexMap = new Hashtable<>(); + dirIndexMap.put(instanceMetadata.hintsDir(), LIVE_MIGRATION_HINTS_DIR_PATH + "/0"); + dirIndexMap.put(instanceMetadata.commitlogDir(), LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"); + if (instanceMetadata.savedCachesDir() != null) + { + dirIndexMap.put(instanceMetadata.savedCachesDir(), LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"); + } + if (instanceMetadata.cdcDir() != null) + { + dirIndexMap.put(instanceMetadata.cdcDir(), LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"); + } + if (instanceMetadata.localSystemDataFileDir() != null) + { + dirIndexMap.put(instanceMetadata.localSystemDataFileDir(), LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0"); + } + for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + { + dirIndexMap.put(instanceMetadata.dataDirs().get(i), LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i); + } + return dirIndexMap; + } + + /** + * Returns map of directory and set of placeholders that represent the directory. + */ + public static Map<String, Set<String>> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + { + Map<String, Set<String>> placeholderMap = new Hashtable<>(); Review Comment: Updated it. ########## client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java: ########## @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Class for holding information of a downloadable file during Live Migration. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class InstanceFileInfo +{ + public final String fileUrl; + // size is only relevant when the fileType is 'FILE' + public final long size; + public final FileType fileType; + + // The latest timestamp at which the file/dir was modified represented in milliseconds. + public final long lastModifiedTime; + + public InstanceFileInfo(@JsonProperty("fileUrl") final String fileUrl, + @JsonProperty("size") final long size, + @JsonProperty("fileType") final FileType fileType, + @JsonProperty("lastModifiedTime") final long lastModifiedTime) Review Comment: done ########## conf/sidecar.yaml: ########## @@ -362,3 +362,10 @@ s3_client: # uri: # username: # password: + +live_migration: + special_files_to_exclude: [] + special_dirs_to_exclude: Review Comment: Thanks! it is a miss from my side to update this config. ########## server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationMap.java: ########## @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Map; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +/** + * Map of source to destination C* instances to do the Live Migration. + */ +public interface LiveMigrationMap +{ + /** + * Tells whether given instance is configured as source or not. + * + * @param instanceMeta Cassandra instance metadata + * @return true if given instance is configured as source for live migration. + */ + default boolean isSource(@NotNull final InstanceMetadata instanceMeta) Review Comment: done ########## server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java: ########## @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.config.yaml; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; + +/** + * Configuration required for live migrating the Cassandra instances. + */ +public class LiveMigrationConfigurationImpl implements LiveMigrationConfiguration +{ + + private final Set<String> filesToExclude; + private final Set<String> directoriesToExclude; + private final Map<String, String> migrationMap; + + public LiveMigrationConfigurationImpl() + { + this(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap()); + } + + @JsonCreator + public LiveMigrationConfigurationImpl(@JsonProperty("files_to_exclude") Set<String> filesToExclude, + @JsonProperty("dirs_to_exclude") Set<String> directoriesToExclude, + @JsonProperty("migration_map") Map<String, String> migrationMap) Review Comment: Good catch, thanks! added unit test. ########## server/src/main/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFiles.java: ########## @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.util.List; + +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; + +/** + * Lists files that a Cassandra instance has. + */ +public interface CassandraInstanceFiles +{ + /** + * Lists files belong to a Cassandra instance after excluding default set files to exclude + * {@link LiveMigrationConfiguration#filesToExclude()} and default set of directories to exclude + * {@link LiveMigrationConfiguration#directoriesToExclude()}. + * + * @return list of files a Cassandra instance has after excluding default files and folders. + * @throws IOException when failed to read any file or folder. + */ + List<InstanceFileInfo> getFiles() throws IOException; Review Comment: Tested with 70K files few months back, don't recollect the numbers, will test it again. At least I recollect that I did not find abnormal gc pauses with 8gb heap (could be subjective to heap environment/size). As far as I know JDK11 doesn't support iterative file walking. List documentation says that `If this list contains more than Integer.MAX_VALUE elements, returns Integer.MAX_VALUE.`, I guess it itself acts as a limiting factor. ########## server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationMap.java: ########## @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Map; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.jetbrains.annotations.NotNull; + +/** + * Map of source to destination C* instances to do the Live Migration. + */ +public interface LiveMigrationMap +{ + /** + * Tells whether given instance is configured as source or not. + * + * @param instanceMeta Cassandra instance metadata + * @return true if given instance is configured as source for live migration. + */ + default boolean isSource(@NotNull final InstanceMetadata instanceMeta) + { + Map<String, String> map = getMigrationMap(); + return map != null && map.containsKey(String.valueOf(instanceMeta.host())); + } + + /** + * Tells whether given instance is configured as destination or not. + * + * @param instanceMeta Cassandra instance metadata + * @return true if given instance is configured as destination for live migration. + */ + default boolean isDestination(@NotNull final InstanceMetadata instanceMeta) + { + Map<String, String> map = getMigrationMap(); + return map != null && map.containsValue(String.valueOf(instanceMeta.host())); + } + + /** + * Tells whether given instance is configured as either source or destination. + * + * @param instanceMeta Cassandra instance metadata + * @return true if given instance is either source or destination + */ + default boolean isAny(@NotNull final InstanceMetadata instanceMeta) + { + return isSource(instanceMeta) || isDestination(instanceMeta); + } + + /** + * Returns map of source and destination + * + * @return map of source and destination + */ + Map<String, String> getMigrationMap(); Review Comment: LiveMigrationConfigurationImpl default constructor is using empty sets and maps. By the way I see sidecar configuration is constructed, I don't think it will be null. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For additional commands, e-mail: pr-h...@cassandra.apache.org
