This is an automated email from the ASF dual-hosted git repository.
frankgh pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-analytics.git
The following commit(s) were added to refs/heads/trunk by this push:
new 672d66a CASSANDRA-19199 Remove write option VALIDATE_SSTABLES to
enforce validation
672d66a is described below
commit 672d66a64a21e23c4d81c089b426360c2bb708b7
Author: jkonisa <[email protected]>
AuthorDate: Wed Dec 6 10:25:56 2023 -0800
CASSANDRA-19199 Remove write option VALIDATE_SSTABLES to enforce validation
Patch by Jyothsna Konisa; Reviewed by Yifan Cai, Francisco Guerrero for
CASSANDRA-19199
---
CHANGES.txt | 1 +
.../java/org/apache/cassandra/spark/bulkwriter/BulkSparkConf.java | 2 --
.../org/apache/cassandra/spark/bulkwriter/CassandraJobInfo.java | 6 ------
.../main/java/org/apache/cassandra/spark/bulkwriter/JobInfo.java | 2 --
.../apache/cassandra/spark/bulkwriter/SidecarDataTransferApi.java | 7 ++-----
.../java/org/apache/cassandra/spark/bulkwriter/WriterOptions.java | 1 -
.../apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java | 6 ------
7 files changed, 3 insertions(+), 22 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index f580f89..e8b431b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.0.0
+ * Remove write option VALIDATE_SSTABLES to enforce validation
(CASSANDRA-19199)
* Fix bulk writing when using identifiers that need quotes (CASSANDRA-19031)
* Fix bulk reading when using identifiers that need quotes (CASSANDRA-19024)
* Remove unused dead code (CASSANDRA-19148)
diff --git
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/BulkSparkConf.java
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/BulkSparkConf.java
index 7ea972e..87e6f2f 100644
---
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/BulkSparkConf.java
+++
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/BulkSparkConf.java
@@ -124,7 +124,6 @@ public class BulkSparkConf implements Serializable
protected final String ttl;
protected final String timestamp;
protected final SparkConf conf;
- public final boolean validateSSTables;
public final int commitThreadsPerInstance;
public boolean quoteIdentifiers;
protected final int effectiveSidecarPort;
@@ -141,7 +140,6 @@ public class BulkSparkConf implements Serializable
this.sidecarInstances = buildSidecarInstances(options,
effectiveSidecarPort);
this.keyspace = MapUtils.getOrThrow(options,
WriterOptions.KEYSPACE.name());
this.table = MapUtils.getOrThrow(options, WriterOptions.TABLE.name());
- this.validateSSTables = MapUtils.getBoolean(options,
WriterOptions.VALIDATE_SSTABLES.name(), true, "validate SSTables");
this.skipExtendedVerify = MapUtils.getBoolean(options,
WriterOptions.SKIP_EXTENDED_VERIFY.name(), true,
"skip extended
verification of SSTables by Cassandra");
this.consistencyLevel =
ConsistencyLevel.CL.valueOf(MapUtils.getOrDefault(options,
WriterOptions.BULK_WRITER_CL.name(), "EACH_QUORUM"));
diff --git
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/CassandraJobInfo.java
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/CassandraJobInfo.java
index 73104fa..e4d4bbe 100644
---
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/CassandraJobInfo.java
+++
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/CassandraJobInfo.java
@@ -76,12 +76,6 @@ public class CassandraJobInfo implements JobInfo
return conf.commitBatchSize;
}
- @Override
- public boolean validateSSTables()
- {
- return conf.validateSSTables;
- }
-
@Override
public boolean skipExtendedVerify()
{
diff --git
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/JobInfo.java
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/JobInfo.java
index 15185b3..69efb05 100644
---
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/JobInfo.java
+++
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/JobInfo.java
@@ -48,8 +48,6 @@ public interface JobInfo extends Serializable
TokenPartitioner getTokenPartitioner();
- boolean validateSSTables();
-
boolean skipExtendedVerify();
boolean quoteIdentifiers();
diff --git
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/SidecarDataTransferApi.java
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/SidecarDataTransferApi.java
index 8ce5619..545b010 100644
---
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/SidecarDataTransferApi.java
+++
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/SidecarDataTransferApi.java
@@ -105,11 +105,8 @@ public class SidecarDataTransferApi implements
DataTransferApi
String uploadId = getUploadId(uuids.get(0), job.getId().toString());
ImportSSTableRequest.ImportOptions importOptions = new
ImportSSTableRequest.ImportOptions();
- if (job.validateSSTables())
- {
- importOptions.verifySSTables(true)
- .extendedVerify(!job.skipExtendedVerify());
- }
+ // Always verify SSTables on import
+
importOptions.verifySSTables(true).extendedVerify(!job.skipExtendedVerify());
try
{
diff --git
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/WriterOptions.java
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/WriterOptions.java
index 95c7876..b8688e7 100644
---
a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/WriterOptions.java
+++
b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/bulkwriter/WriterOptions.java
@@ -30,7 +30,6 @@ public enum WriterOptions implements WriterOption
BATCH_SIZE,
COMMIT_THREADS_PER_INSTANCE,
COMMIT_BATCH_SIZE,
- VALIDATE_SSTABLES,
SKIP_EXTENDED_VERIFY,
WRITE_MODE,
KEYSTORE_PASSWORD,
diff --git
a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
index 8aee396..37c6d33 100644
---
a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
+++
b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
@@ -240,12 +240,6 @@ public class MockBulkWriterContext implements
BulkWriterContext, ClusterInfo, Jo
return 1;
}
- @Override
- public boolean validateSSTables()
- {
- return true;
- }
-
@Override
public boolean skipExtendedVerify()
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
