frankgh commented on code in PR #3598: URL: https://github.com/apache/cassandra/pull/3598#discussion_r2051825716
########## src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java: ########## @@ -84,7 +85,7 @@ private SystemDistributedKeyspace() * * // TODO: TCM - how do we evolve these tables? */ - public static final long GENERATION = 6; + public static final long GENERATION = 7; Review Comment: can we add a description for gen 7 above? ########## doc/modules/cassandra/pages/managing/operating/metrics.adoc: ########## @@ -1081,6 +1088,67 @@ partitions processed per logged batch partitions processed per unlogged batch |=== +== Automated Repair Metrics + +Metrics specifc to automated repair. + +Reported name format: + +*Metric Name*:: +`org.apache.cassandra.metrics.AutoRepair.<MetricName>` +*JMX MBean*:: +`org.apache.cassandra.metrics:type=AutoRepair name=<MetricName> repairType=<RepairType>` + +[cols=",,",options="header",] +|=== +|Name |Type |Description +|RepairsInProgress |Gauge<Integer> |Repair is in progress +on the node + +|NodeRepairTimeInSec |Gauge<Integer> |Time taken to repair +the node in seconds + +|ClusterRepairTimeInSec |Gauge<Integer> |Time taken to repair +the entire Cassandra cluster in seconds + +|LongestUnrepairedSec |Gauge<Integer> |Time since the last repair +ran on the node in seconds + +|RepairStartLagSec|Gauge<Integer> |If a repair has not run within min_repair_interval, how long past this value since +repairs last completed. Useful for determining if repairs are behind schedule. + +|SucceededTokenRangesCount |Gauge<Integer> |Number of token ranges successfully repaired on the node + +|FailedTokenRangesCount |Gauge<Integer> |Number of token ranges failed to repair on the node + +|SkippedTokenRangesCount |Gauge<Integer> |Number of token ranges skipped +on the node + +|SkippedTablesCount |Gauge<Integer> |Number of tables skipped +on the node + +|TotalMVTablesConsideredForRepair |Gauge<Integer> |Number of materialized +views considered on the node + +|TotalDisabledRepairTables |Gauge<Integer> |Number of tables on which +the automated repair has been disabled on the node + +|RepairTurnMyTurn |Counter |Represents the node's turn to repair + +|RepairTurnMyTurnDueToPriority |Counter |Represents the node's turn to repair +due to priority set in the automated repair + +|RepairDelayedByReplica |Counter |Represents occurrences of a node's turn being +delayed because a replica was currently taking its turn. Only revelent if Review Comment: relevant maybe? ```suggestion delayed because a replica was currently taking its turn. Only relevant if ``` ########## src/java/org/apache/cassandra/service/ActiveRepairService.java: ########## @@ -1086,6 +1108,16 @@ public void setRepairPendingCompactionRejectThreshold(int value) DatabaseDescriptor.setRepairPendingCompactionRejectThreshold(value); } + public double getIncrementalRepairDiskHeadroomRejectRatio() + { + return DatabaseDescriptor.getIncrementalRepairDiskHeadroomRejectRatio(); + } + + public void setIncrementalRepairDiskHeadroomRejectRatio(double value) + { + DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(value); Review Comment: can we validate for only positive numbers to be set here between the values of 0-1? ########## src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java: ########## @@ -0,0 +1,932 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.tcm.compatibility.TokenRingUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; +import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; +import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; + +/** + * The default implementation of {@link IAutoRepairTokenRangeSplitter} that attempts to: + * <ol> + * <li>Create smaller, consistent repair times</li> + * <li>Minimize the impact on hosts</li> + * <li>Reduce overstreaming</li> + * <li>Reduce number of repairs</li> + * </ol> + * <p> + * To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions + * within a range and splits it accordingly to bound the size of the token ranges used for repair assignments. + * </p> + * <p> + * Refer to + * <a href="https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#repair-token-range-splitter">Auto Repair documentation for this implementation</a> + * for a more thorough breakdown of this implementation. + * </p> + * <p> + * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration + * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following + * configuration parameters are offered. Review Comment: did we intend to list the configuration parameters below in the javadocs? ########## src/java/org/apache/cassandra/schema/AutoRepairParams.java: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.StringUtils; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.utils.LocalizeString; + +import static java.lang.String.format; +import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; + +/** + * AutoRepair table parameters - used to define the auto-repair configuration for a table. + */ +public final class AutoRepairParams +{ + public enum Option + { + FULL_ENABLED, + INCREMENTAL_ENABLED, + PREVIEW_REPAIRED_ENABLED, + PRIORITY; + + @Override + public String toString() + { + return toLowerCaseLocalized(name()); + } + } + + private final ImmutableMap<String, String> options; + + public static final Map<String, String> DEFAULT_OPTIONS = ImmutableMap.of( + LocalizeString.toLowerCaseLocalized(Option.FULL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.INCREMENTAL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.PREVIEW_REPAIRED_ENABLED.name()), Boolean.toString(true), + Option.PRIORITY.toString(), "0" + ); + + AutoRepairParams(Map<String, String> options) + { + this.options = ImmutableMap.copyOf(options); + } + + public static final AutoRepairParams DEFAULT = + new AutoRepairParams(DEFAULT_OPTIONS); + + public static AutoRepairParams create(Map<String, String> options) + { + Map<String, String> optionsMap = new TreeMap<>(); + for (Map.Entry<String, String> entry : DEFAULT_OPTIONS.entrySet()) + { + optionsMap.put(entry.getKey(), entry.getValue()); + } + if (options != null) + { + for (Map.Entry<String, String> entry : options.entrySet()) + { + if (Arrays.stream(Option.values()).noneMatch(option -> option.toString().equalsIgnoreCase(entry.getKey()))) + { + throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); + } + optionsMap.put(entry.getKey(), entry.getValue()); + } + } + return new AutoRepairParams(optionsMap); + } + + public boolean repairEnabled(AutoRepairConfig.RepairType type) + { + String option = LocalizeString.toLowerCaseLocalized(type.toString()) + "_enabled"; + String enabled = options.get(option); + return enabled == null + ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(option)) + : Boolean.parseBoolean(enabled); + } + + public int priority() + { + String priority = options.get(Option.PRIORITY.toString()); + return priority == null + ? Integer.parseInt(DEFAULT_OPTIONS.get(Option.PRIORITY.toString())) + : Integer.parseInt(priority); Review Comment: NIT: ```suggestion String priority = options.getOrDefault(Option.PRIORITY.toString(), DEFAULT_OPTIONS.get(Option.PRIORITY.toString())); return Integer.parseInt(priority); ``` ########## src/java/org/apache/cassandra/schema/AutoRepairParams.java: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.StringUtils; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.utils.LocalizeString; + +import static java.lang.String.format; +import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; + +/** + * AutoRepair table parameters - used to define the auto-repair configuration for a table. + */ +public final class AutoRepairParams +{ + public enum Option + { + FULL_ENABLED, + INCREMENTAL_ENABLED, + PREVIEW_REPAIRED_ENABLED, + PRIORITY; + + @Override + public String toString() + { + return toLowerCaseLocalized(name()); + } + } + + private final ImmutableMap<String, String> options; + + public static final Map<String, String> DEFAULT_OPTIONS = ImmutableMap.of( + LocalizeString.toLowerCaseLocalized(Option.FULL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.INCREMENTAL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.PREVIEW_REPAIRED_ENABLED.name()), Boolean.toString(true), + Option.PRIORITY.toString(), "0" + ); + + AutoRepairParams(Map<String, String> options) + { + this.options = ImmutableMap.copyOf(options); + } + + public static final AutoRepairParams DEFAULT = + new AutoRepairParams(DEFAULT_OPTIONS); + + public static AutoRepairParams create(Map<String, String> options) + { + Map<String, String> optionsMap = new TreeMap<>(); + for (Map.Entry<String, String> entry : DEFAULT_OPTIONS.entrySet()) + { + optionsMap.put(entry.getKey(), entry.getValue()); + } + if (options != null) + { + for (Map.Entry<String, String> entry : options.entrySet()) + { + if (Arrays.stream(Option.values()).noneMatch(option -> option.toString().equalsIgnoreCase(entry.getKey()))) + { + throw new ConfigurationException(format("Unknown property '%s'", entry.getKey())); + } + optionsMap.put(entry.getKey(), entry.getValue()); + } + } + return new AutoRepairParams(optionsMap); + } + + public boolean repairEnabled(AutoRepairConfig.RepairType type) + { + String option = LocalizeString.toLowerCaseLocalized(type.toString()) + "_enabled"; + String enabled = options.get(option); + return enabled == null + ? Boolean.parseBoolean(DEFAULT_OPTIONS.get(option)) + : Boolean.parseBoolean(enabled); Review Comment: NIT: ```suggestion String enabled = options.getOrDefault(option, DEFAULT_OPTIONS.get(option)); return Boolean.parseBoolean(enabled); ``` ########## src/java/org/apache/cassandra/schema/AutoRepairParams.java: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang3.StringUtils; + +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.repair.autorepair.AutoRepairConfig; +import org.apache.cassandra.utils.LocalizeString; + +import static java.lang.String.format; +import static org.apache.cassandra.utils.LocalizeString.toLowerCaseLocalized; + +/** + * AutoRepair table parameters - used to define the auto-repair configuration for a table. + */ +public final class AutoRepairParams +{ + public enum Option + { + FULL_ENABLED, + INCREMENTAL_ENABLED, + PREVIEW_REPAIRED_ENABLED, + PRIORITY; + + @Override + public String toString() + { + return toLowerCaseLocalized(name()); + } + } + + private final ImmutableMap<String, String> options; + + public static final Map<String, String> DEFAULT_OPTIONS = ImmutableMap.of( + LocalizeString.toLowerCaseLocalized(Option.FULL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.INCREMENTAL_ENABLED.name()), Boolean.toString(true), + LocalizeString.toLowerCaseLocalized(Option.PREVIEW_REPAIRED_ENABLED.name()), Boolean.toString(true), + Option.PRIORITY.toString(), "0" + ); + + AutoRepairParams(Map<String, String> options) + { + this.options = ImmutableMap.copyOf(options); + } + + public static final AutoRepairParams DEFAULT = + new AutoRepairParams(DEFAULT_OPTIONS); + + public static AutoRepairParams create(Map<String, String> options) + { + Map<String, String> optionsMap = new TreeMap<>(); + for (Map.Entry<String, String> entry : DEFAULT_OPTIONS.entrySet()) + { + optionsMap.put(entry.getKey(), entry.getValue()); + } Review Comment: NIT, we can initialize this in the constructor directly: ```suggestion Map<String, String> optionsMap = new TreeMap<>(DEFAULT_OPTIONS); ``` ########## src/java/org/apache/cassandra/repair/autorepair/RepairTokenRangeSplitter.java: ########## @@ -0,0 +1,932 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair.autorepair; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.tcm.compatibility.TokenRingUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; +import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; +import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.AutoRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.repair.autorepair.AutoRepairUtils.split; + +/** + * The default implementation of {@link IAutoRepairTokenRangeSplitter} that attempts to: + * <ol> + * <li>Create smaller, consistent repair times</li> + * <li>Minimize the impact on hosts</li> + * <li>Reduce overstreaming</li> + * <li>Reduce number of repairs</li> + * </ol> + * <p> + * To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions + * within a range and splits it accordingly to bound the size of the token ranges used for repair assignments. + * </p> + * <p> + * Refer to + * <a href="https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#repair-token-range-splitter">Auto Repair documentation for this implementation</a> + * for a more thorough breakdown of this implementation. + * </p> + * <p> + * While this splitter has a lot of tuning parameters, the expectation is that the established default configuration + * shall be sensible for all {@link org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType}'s. The following + * configuration parameters are offered. + * </p> + */ +public class RepairTokenRangeSplitter implements IAutoRepairTokenRangeSplitter +{ + private static final Logger logger = LoggerFactory.getLogger(RepairTokenRangeSplitter.class); + + // Default max bytes to 100TiB, which is much more readable than Long.MAX_VALUE + private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(100_000, DataStorageSpec.DataStorageUnit.GIBIBYTES); Review Comment: I think this should be 102_400 to be equal to 100TiB ```suggestion private static final DataStorageSpec.LongBytesBound MAX_BYTES = new DataStorageSpec.LongBytesBound(102_400, DataStorageSpec.DataStorageUnit.GIBIBYTES); ``` ########## doc/modules/cassandra/pages/managing/operating/metrics.adoc: ########## @@ -1081,6 +1088,67 @@ partitions processed per logged batch partitions processed per unlogged batch |=== +== Automated Repair Metrics + +Metrics specifc to automated repair. + +Reported name format: + +*Metric Name*:: +`org.apache.cassandra.metrics.AutoRepair.<MetricName>` +*JMX MBean*:: +`org.apache.cassandra.metrics:type=AutoRepair name=<MetricName> repairType=<RepairType>` + +[cols=",,",options="header",] +|=== +|Name |Type |Description +|RepairsInProgress |Gauge<Integer> |Repair is in progress +on the node + +|NodeRepairTimeInSec |Gauge<Integer> |Time taken to repair +the node in seconds + +|ClusterRepairTimeInSec |Gauge<Integer> |Time taken to repair +the entire Cassandra cluster in seconds + +|LongestUnrepairedSec |Gauge<Integer> |Time since the last repair +ran on the node in seconds + +|RepairStartLagSec|Gauge<Integer> |If a repair has not run within min_repair_interval, how long past this value since +repairs last completed. Useful for determining if repairs are behind schedule. + +|SucceededTokenRangesCount |Gauge<Integer> |Number of token ranges successfully repaired on the node + +|FailedTokenRangesCount |Gauge<Integer> |Number of token ranges failed to repair on the node + +|SkippedTokenRangesCount |Gauge<Integer> |Number of token ranges skipped +on the node + +|SkippedTablesCount |Gauge<Integer> |Number of tables skipped +on the node + +|TotalMVTablesConsideredForRepair |Gauge<Integer> |Number of materialized +views considered on the node + +|TotalDisabledRepairTables |Gauge<Integer> |Number of tables on which +the automated repair has been disabled on the node + +|RepairTurnMyTurn |Counter |Represents the node's turn to repair + +|RepairTurnMyTurnDueToPriority |Counter |Represents the node's turn to repair +due to priority set in the automated repair + +|RepairDelayedByReplica |Counter |Represents occurrences of a node's turn being +delayed because a replica was currently taking its turn. Only revelent if +`allow_parallel_replica_repair` is false. + +|RepairDelayedBySchedule |Counter |Represents occurrences of a node's turn being +delayed because it was already being repaired in another schedule. Only relevent Review Comment: relevant maybe? ```suggestion delayed because it was already being repaired in another schedule. Only relevant ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: pr-unsubscr...@cassandra.apache.org For additional commands, e-mail: pr-h...@cassandra.apache.org