This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new 1de1ce5 Shard split supports encryption. (#103)
1de1ce5 is described below
commit 1de1ce559338ae111e5c7de8fc83b01b2b7c3cd1
Author: Bruno Roustant <[email protected]>
AuthorDate: Fri Apr 26 16:04:54 2024 +0200
Shard split supports encryption. (#103)
---
.../solr/encryption/EncryptionUpdateHandler.java | 31 +++--
.../encryption/EncryptionRequestHandlerTest.java | 30 ++---
.../apache/solr/encryption/EncryptionTestUtil.java | 18 ++-
.../EncryptionUpdateHandlerSplitTest.java | 136 +++++++++++++++++++++
.../resources/configs/collection1/solrconfig.xml | 2 +-
5 files changed, 183 insertions(+), 34 deletions(-)
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
index 8b86d5f..d9cee7a 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateHandler.java
@@ -20,6 +20,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.solr.core.SolrCore;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DirectUpdateHandler2;
+import org.apache.solr.update.SplitIndexCommand;
import org.apache.solr.update.UpdateHandler;
import java.io.IOException;
@@ -57,25 +58,37 @@ public class EncryptionUpdateHandler extends
DirectUpdateHandler2 {
if (!super.shouldCommit(cmd, writer)) {
return false;
}
+ cmd.commitData = transferCommitData(cmd.commitData);
+ return true;
+ }
+
+ private Map<String, String> transferCommitData(Map<String, String>
commitData) throws IOException {
// Two cases:
- // - If cmd.commitData is null, then transfer all the latest commit
transferable
+ // - If commitData is null, then transfer all the latest commit
transferable
// data to the current commit.
- // - If cmd.commitData is not null, nothing is transferred. It is the
caller
+ // - If commitData is not null, nothing is transferred. It is the caller
// responsibility to include all the required user data from the latest
commit.
// That way, the caller can remove some entries.
- if (cmd.commitData == null) {
+ if (commitData == null) {
Map<String, String> latestCommitData =
readLatestCommit(core).getUserData();
- Map<String, String> commitData = null;
+ Map<String, String> newCommitData = null;
for (Map.Entry<String, String> latestCommitEntry :
latestCommitData.entrySet()) {
if (latestCommitEntry.getKey().startsWith(TRANSFERABLE_COMMIT_DATA)) {
- if (commitData == null) {
- commitData = new HashMap<>();
+ if (newCommitData == null) {
+ newCommitData = new HashMap<>();
}
- commitData.put(latestCommitEntry.getKey(),
latestCommitEntry.getValue());
+ newCommitData.put(latestCommitEntry.getKey(),
latestCommitEntry.getValue());
}
}
- cmd.commitData = commitData;
+ return newCommitData;
}
- return true;
+ return commitData;
+ }
+
+ @Override
+ public void split(SplitIndexCommand cmd) throws IOException {
+ // Transfer the parent core commit data to the sub-shard cores.
+ cmd.commitData = transferCommitData(cmd.commitData);
+ super.split(cmd);
}
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
index f2515af..a943bae 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
@@ -34,7 +34,6 @@ import java.util.UUID;
import static
org.apache.solr.encryption.EncryptionDirectoryFactory.PROPERTY_INNER_ENCRYPTION_DIRECTORY_FACTORY;
import static org.apache.solr.encryption.EncryptionRequestHandler.NO_KEY_ID;
-import static org.apache.solr.encryption.EncryptionTestUtil.EncryptionStatus;
import static org.apache.solr.encryption.EncryptionUtil.getKeyIdFromCommit;
import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_2;
@@ -88,7 +87,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
@Test
public void testEncryptionFromNoKeysToOneKey_NoIndex() throws Exception {
// Send an encrypt request with a key id on an empty index.
- encryptAndExpectCompletion(KEY_ID_1);
+ testUtil.encryptAndExpectCompletion(KEY_ID_1);
// Index some documents to create a first segment.
testUtil.indexDocsAndCommit("weather broadcast");
@@ -104,10 +103,10 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
@Test
public void testEncryptionFromNoKeysToOneKeyToNoKeys_NoIndex() throws
Exception {
// Send an encrypt request with a key id on an empty index.
- encryptAndExpectCompletion(KEY_ID_1);
+ testUtil.encryptAndExpectCompletion(KEY_ID_1);
// Send another encrypt request with no key id, still on the empty index.
- encryptAndExpectCompletion(NO_KEY_ID);
+ testUtil.encryptAndExpectCompletion(NO_KEY_ID);
// Index some documents to create a first segment.
testUtil.indexDocsAndCommit("weather broadcast");
@@ -134,7 +133,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
forceClearText = false;
// Send an encrypt request with a key id.
- encryptAndWaitForCompletion(KEY_ID_1);
+ testUtil.encryptAndWaitForCompletion(KEY_ID_1);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -154,7 +153,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("foggy weather");
// Send an encrypt request with another key id.
- encryptAndWaitForCompletion(KEY_ID_2);
+ testUtil.encryptAndWaitForCompletion(KEY_ID_2);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -173,7 +172,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("foggy weather");
// Send an encrypt request with no key id.
- encryptAndWaitForCompletion(NO_KEY_ID);
+ testUtil.encryptAndWaitForCompletion(NO_KEY_ID);
// Verify that the segment is cleartext.
forceClearText = true;
@@ -185,7 +184,7 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("cloudy weather");
// Send an encrypt request with another key id.
- encryptAndWaitForCompletion(KEY_ID_2);
+ testUtil.encryptAndWaitForCompletion(KEY_ID_2);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -196,21 +195,6 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.assertQueryReturns("weather", 4);
}
- private void encryptAndExpectCompletion(String keyId) {
- encrypt(keyId, true);
- }
-
- private void encryptAndWaitForCompletion(String keyId) throws
InterruptedException {
- encrypt(keyId, false);
- testUtil.waitUntilEncryptionIsComplete(keyId);
- }
-
- private void encrypt(String keyId, boolean expectComplete) {
- EncryptionStatus encryptionStatus = testUtil.encrypt(keyId);
- assertTrue(encryptionStatus.isSuccess());
- assertEquals(expectComplete, encryptionStatus.isComplete());
- }
-
private static void clearMockValues() {
forceClearText = false;
soleKeyIdAllowed = null;
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
index 61d36c4..ebb9538 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
@@ -45,6 +45,7 @@ import static
org.apache.solr.encryption.EncryptionRequestHandler.PARAM_KEY_ID;
import static
org.apache.solr.encryption.EncryptionRequestHandler.STATE_COMPLETE;
import static org.apache.solr.encryption.EncryptionRequestHandler.STATUS;
import static
org.apache.solr.encryption.EncryptionRequestHandler.STATUS_SUCCESS;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -100,7 +101,7 @@ public class EncryptionTestUtil {
*/
public void assertQueryReturns(String query, int expectedNumResults) throws
Exception {
QueryResponse response = cloudSolrClient.query(collectionName, new
SolrQuery(query));
- Assert.assertEquals(expectedNumResults, response.getResults().size());
+ assertEquals(expectedNumResults, response.getResults().size());
}
public EncryptionStatus encrypt(String keyId) {
@@ -116,6 +117,21 @@ public class EncryptionTestUtil {
return encryptionStatus;
}
+ public void encryptAndExpectCompletion(String keyId) {
+ encryptAndCheck(keyId, true);
+ }
+
+ public void encryptAndWaitForCompletion(String keyId) throws
InterruptedException {
+ encryptAndCheck(keyId, false);
+ waitUntilEncryptionIsComplete(keyId);
+ }
+
+ private void encryptAndCheck(String keyId, boolean expectComplete) {
+ EncryptionStatus encryptionStatus = encrypt(keyId);
+ assertTrue(encryptionStatus.isSuccess());
+ assertEquals(expectComplete, encryptionStatus.isComplete());
+ }
+
public void waitUntilEncryptionIsComplete(String keyId) throws
InterruptedException {
RetryUtil.retryUntil("Timeout waiting for encryption completion",
50,
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateHandlerSplitTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateHandlerSplitTest.java
new file mode 100644
index 0000000..4e40031
--- /dev/null
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateHandlerSplitTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.encryption;
+
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.update.SolrIndexSplitter;
+import org.apache.solr.update.SplitIndexCommand;
+import org.apache.solr.update.UpdateHandler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.solr.encryption.CommitUtil.readLatestCommit;
+import static org.apache.solr.encryption.EncryptionUtil.getKeyIdFromCommit;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
+
+/**
+ * Tests {@link EncryptionUpdateHandler} commit data transfer during shard
split.
+ */
+public class EncryptionUpdateHandlerSplitTest extends SolrCloudTestCase {
+
+ private static final String COLLECTION_PREFIX =
EncryptionUpdateHandlerSplitTest.class.getSimpleName() + "-collection-";
+
+ private String collectionName;
+ private EncryptionTestUtil testUtil;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ System.setProperty("solr.updateHandler",
TestEncryptionUpdateHandler.class.getName());
+ EncryptionTestUtil.setInstallDirProperty();
+ cluster = new MiniSolrCloudCluster.Builder(2, createTempDir())
+ .addConfig("config", EncryptionTestUtil.getConfigPath("collection1"))
+ .configure();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ collectionName = COLLECTION_PREFIX + UUID.randomUUID();
+ CollectionAdminRequest
+ .createCollection(collectionName, 2, 1)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(collectionName, 2, 2);
+ testUtil = new EncryptionTestUtil(cluster.getSolrClient(), collectionName);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+
CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
+ super.tearDown();
+ }
+
+ @Test
+ public void testSplitTransfersCommitData() throws Exception {
+ // Index some documents to create a first encrypted segment.
+ testUtil.encryptAndExpectCompletion(KEY_ID_1);
+ testUtil.indexDocsAndCommit("weather broadcast");
+
+ SolrIndexSplitter.SplitMethod splitMethod = random().nextBoolean() ?
+ SolrIndexSplitter.SplitMethod.LINK :
SolrIndexSplitter.SplitMethod.REWRITE;
+ CollectionAdminRequest.SplitShard splitShard =
CollectionAdminRequest.splitShard(collectionName)
+ .setNumSubShards(2)
+ .setShardName("shard1")
+ .setSplitMethod(splitMethod.toLower());
+ splitShard.process(cluster.getSolrClient());
+ waitForState("Timed out waiting for sub-shards to be active",
+ collectionName,
+ SolrCloudTestCase.activeClusterShape(3, 4));
+
+ assertNotNull(TestEncryptionUpdateHandler.cmd);
+
+ if (TestEncryptionUpdateHandler.cmd.cores != null) {
+ // Check the sub-shard cores if we have them (LINK split method).
+ for (SolrCore subCore : TestEncryptionUpdateHandler.cmd.cores) {
+ assertEquals(KEY_ID_1, getKeyIdFromCommitData(subCore));
+ }
+ }
+ }
+
+ private static String getKeyIdFromCommitData(SolrCore core) throws
IOException {
+ Map<String, String> commitData = readLatestCommit(core).getUserData();
+ assertNotNull(commitData);
+ return getKeyIdFromCommit("0", commitData);
+ }
+
+ /**
+ * Captures the {@link SplitIndexCommand} for validation.
+ */
+ public static class TestEncryptionUpdateHandler extends
EncryptionUpdateHandler {
+
+ static volatile SplitIndexCommand cmd;
+
+ public TestEncryptionUpdateHandler(SolrCore core) {
+ super(core);
+ }
+
+ public TestEncryptionUpdateHandler(SolrCore core, UpdateHandler
updateHandler) {
+ super(core, updateHandler);
+ }
+
+ @Override
+ public void split(SplitIndexCommand cmd) throws IOException {
+ TestEncryptionUpdateHandler.cmd = cmd;
+ // Check the parent core commit data here while it is still active.
+ SolrCore parentCore = cmd.getReq().getSearcher().getCore();
+ assertEquals(KEY_ID_1, getKeyIdFromCommitData(parentCore));
+ super.split(cmd);
+ }
+ }
+}
diff --git a/encryption/src/test/resources/configs/collection1/solrconfig.xml
b/encryption/src/test/resources/configs/collection1/solrconfig.xml
index d9a7bb8..0062cf5 100644
--- a/encryption/src/test/resources/configs/collection1/solrconfig.xml
+++ b/encryption/src/test/resources/configs/collection1/solrconfig.xml
@@ -45,7 +45,7 @@
</directoryFactory>
<!-- EncryptionUpdateHandler transfers the encryption key ids from a commit
to the next. -->
- <updateHandler class="org.apache.solr.encryption.EncryptionUpdateHandler">
+ <updateHandler
class="${solr.updateHandler:org.apache.solr.encryption.EncryptionUpdateHandler}">
<!-- EncryptionUpdateLog encrypts transaction logs if the index is
encrypted. -->
<updateLog
class="${solr.updateLog:org.apache.solr.encryption.EncryptionUpdateLog}"/>
</updateHandler>
