Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/326f3a7c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/326f3a7c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/326f3a7c Branch: refs/heads/trunk Commit: 326f3a7c7d8f20d0389d9cc036b0bb32d37462be Parents: 68c2d2e 809f3b3 Author: Marcus Eriksson <[email protected]> Authored: Mon Aug 28 15:40:11 2017 +0200 Committer: Marcus Eriksson <[email protected]> Committed: Mon Aug 28 15:41:04 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 19 ++++-- .../cassandra/io/sstable/SSTableLoader.java | 4 +- .../io/sstable/format/SSTableReader.java | 66 +++++++++++++------- .../unit/org/apache/cassandra/db/ScrubTest.java | 2 +- 5 files changed, 62 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 73e7db6,23d0b8e..5aecc9d --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -62,8 -60,10 +62,11 @@@ import org.apache.cassandra.exceptions. import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.index.internal.CassandraIndex; import org.apache.cassandra.index.transactions.UpdateTransaction; + import org.apache.cassandra.io.FSError; ++import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; + import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.*; @@@ -781,11 -767,24 +785,18 @@@ public class ColumnFamilyStore implemen { reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata); } - catch (IOException e) + catch (CorruptSSTableException ex) + { + FileUtils.handleCorruptSSTable(ex); + logger.error("Corrupt sstable {}; skipping table", entry, ex); + continue; + } + catch (FSError ex) { - SSTableReader.logOpenException(entry.getKey(), e); + FileUtils.handleFSError(ex); + logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex); continue; } - catch (IOException ex) - { - FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA))); - logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex); - continue; - } newSSTables.add(reader); } @@@ -1912,7 -1906,7 +1923,7 @@@ } } } -- catch (IOException | RuntimeException e) ++ catch (FSReadError | RuntimeException e) { // In case one of the snapshot sstables fails to open, // we must release the references to the ones we opened so far http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/SSTableLoader.java index dc56520,043f6fa..69e5c85 --- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java @@@ -25,7 -25,7 +25,8 @@@ import java.util.* import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.apache.cassandra.config.CFMetaData; ++import org.apache.cassandra.io.FSError; +import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Range; @@@ -138,8 -138,8 +139,9 @@@ public class SSTableLoader implements S // to conserve heap space when bulk loading sstable.releaseSummary(); } -- catch (IOException e) ++ catch (FSError e) { ++ // todo: should we really continue if we can't open all sstables? outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage())); } return false; http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index bb1eb0d,03af2bb..2801e36 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@@ -344,9 -352,9 +344,9 @@@ public abstract class SSTableReader ext return base; } -- public static SSTableReader open(Descriptor descriptor) throws IOException ++ public static SSTableReader open(Descriptor descriptor) { - CFMetaData metadata; + TableMetadataRef metadata; if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR)) { int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); @@@ -362,24 -374,24 +362,24 @@@ return open(descriptor, metadata); } - public static SSTableReader open(Descriptor desc, TableMetadataRef metadata) throws IOException - public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException ++ public static SSTableReader open(Descriptor desc, TableMetadataRef metadata) { return open(desc, componentsFor(desc), metadata); } - public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) throws IOException - public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException ++ public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) { return open(descriptor, components, metadata, true, true); } // use only for offline or "Standalone" operations -- public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs) throws IOException ++ public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs) { return open(descriptor, components, cfs.metadata, false, false); // do not track hotness } // use only for offline or "Standalone" operations - public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata) throws IOException - public static SSTableReader openNoValidation(Descriptor descriptor, CFMetaData metadata) throws IOException ++ public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata) { return open(descriptor, componentsFor(descriptor), metadata, false, false); // do not track hotness } @@@ -393,14 -405,14 +393,22 @@@ * @return opened SSTableReader * @throws IOException */ - public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) throws IOException - public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException ++ public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) { // Minimum components without which we can't do anything assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor; assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor; EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); -- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); ++ Map<MetadataType, MetadataComponent> sstableMetadata; ++ try ++ { ++ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); ++ } ++ catch (IOException e) ++ { ++ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS)); ++ } ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); @@@ -435,7 -447,7 +443,16 @@@ .withChunkCache(ChunkCache.instance)) { if (!sstable.loadSummary()) -- sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL); ++ { ++ try ++ { ++ sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL); ++ } ++ catch(IOException e) ++ { ++ throw new CorruptSSTableException(e, sstable.getFilename()); ++ } ++ } long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size()); @@@ -448,20 -460,29 +465,29 @@@ } public static SSTableReader open(Descriptor descriptor, - Set<Component> components, - CFMetaData metadata, - boolean validate, - boolean trackHotness) throws IOException + Set<Component> components, + TableMetadataRef metadata, + boolean validate, - boolean trackHotness) throws IOException ++ boolean trackHotness) { // Minimum components without which we can't do anything assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor; assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor; // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content - assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor; + assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor; EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); - Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); + + Map<MetadataType, MetadataComponent> sstableMetadata; + try + { + sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); + } + catch (IOException e) + { + throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS)); + } ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); @@@ -511,16 -537,16 +542,8 @@@ } } -- public static void logOpenException(Descriptor descriptor, IOException e) -- { -- if (e instanceof FileNotFoundException) -- logger.error("Missing sstable component in {}; skipped because of {}", descriptor, e.getMessage()); -- else -- logger.error("Corrupt sstable {}; skipped", descriptor, e); -- } -- public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries, - final CFMetaData metadata) + final TableMetadataRef metadata) { final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>(); @@@ -548,11 -574,12 +571,6 @@@ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex); return; } -- catch (IOException ex) -- { - FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA))); -- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex); -- return; -- } sstables.add(sstable); } }; http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
