This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/main by this push:
new 1951920 Encrypt handler calls EncryptionUpdateLog to reencrypt logs.
Rewrite the test. (#96)
1951920 is described below
commit 195192017369a8c4468f34af215f717b7617dbf8
Author: Bruno Roustant <[email protected]>
AuthorDate: Mon Dec 4 17:57:55 2023 +0100
Encrypt handler calls EncryptionUpdateLog to reencrypt logs. Rewrite the
test. (#96)
---
.../encryption/EncryptionDirectoryFactory.java | 2 +-
.../solr/encryption/EncryptionRequestHandler.java | 42 ++-
.../solr/encryption/EncryptionTransactionLog.java | 90 ++++--
.../solr/encryption/EncryptionUpdateLog.java | 94 ++++--
.../org/apache/solr/encryption/KeySupplier.java | 12 +-
.../encryption/crypto/CharStreamEncrypter.java | 35 +++
.../encryption/crypto/EncryptingIndexOutput.java | 1 -
.../org/apache/solr/update/TransactionLog.java | 2 +-
.../solr/encryption/EncryptionHeavyLoadTest.java | 28 +-
.../encryption/EncryptionRequestHandlerTest.java | 94 ++----
.../apache/solr/encryption/EncryptionTestUtil.java | 61 ++++
.../solr/encryption/EncryptionUpdateLogTest.java | 325 ++++++++++-----------
.../apache/solr/encryption/TestingKeySupplier.java | 3 +-
.../encryption/crypto/AesCtrEncrypterTest.java | 8 +-
.../encryption/crypto/CharStreamEncrypterTest.java | 8 +-
.../CryptoTestUtil.java} | 30 +-
.../crypto/DecryptingChannelInputStreamTest.java | 5 +-
.../crypto/DecryptingIndexInputTest.java | 5 +-
.../crypto/EncryptingIndexOutputTest.java | 5 +-
.../crypto/EncryptingOutputStreamTest.java | 5 +-
.../resources/configs/collection1/solrconfig.xml | 2 +-
21 files changed, 490 insertions(+), 367 deletions(-)
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
index ad51886..102005f 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionDirectoryFactory.java
@@ -78,7 +78,7 @@ public class EncryptionDirectoryFactory extends
MMapDirectoryFactory {
}
KeySupplier.Factory keySupplierFactory =
coreContainer.getResourceLoader().newInstance(keySupplierFactoryClass,
KeySupplier.Factory.class);
- keySupplierFactory.init(args);
+ keySupplierFactory.init(args, coreContainer);
try {
keySupplier = keySupplierFactory.create();
} catch (IOException e) {
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
index 9c3b099..77e6c4d 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionRequestHandler.java
@@ -29,6 +29,8 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.UpdateHandler;
+import org.apache.solr.update.UpdateLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -209,7 +211,8 @@ public class EncryptionRequestHandler extends
RequestHandlerBase {
if (isCommitActiveKeyId(keyId, segmentInfos)) {
log.debug("{} provided keyId={} is the current active key id",
ENCRYPTION_LOG_PREFIX, keyId);
if
(Boolean.parseBoolean(segmentInfos.getUserData().get(COMMIT_ENCRYPTION_PENDING)))
{
- encryptionComplete = areAllSegmentsEncryptedWithKeyId(keyId,
req.getCore(), segmentInfos);
+ encryptionComplete = areAllSegmentsEncryptedWithKeyId(keyId,
req.getCore(), segmentInfos)
+ && areAllLogsEncryptedWithKeyId(keyId, req.getCore(),
segmentInfos);
if (encryptionComplete) {
commitEncryptionComplete(keyId, segmentInfos, req);
}
@@ -328,13 +331,22 @@ public class EncryptionRequestHandler extends
RequestHandlerBase {
log.debug("{} submitting async encryption", ENCRYPTION_LOG_PREFIX);
executor.submit(() -> {
try {
- log.debug("{} running async encryption", ENCRYPTION_LOG_PREFIX);
+ EncryptionUpdateLog updateLog = getEncryptionUpdateLog(req.getCore());
+ if (updateLog != null) {
+ log.debug("{} running async update log encryption",
ENCRYPTION_LOG_PREFIX);
+ boolean logEncryptionComplete = updateLog.encryptLogs();
+ log.info("{} {} encrypted the update log in {}",
+ ENCRYPTION_LOG_PREFIX, logEncryptionComplete ?
"successfully" : "partially", elapsedTime(startTimeMs));
+ }
+
+ log.debug("{} running async index encryption", ENCRYPTION_LOG_PREFIX);
CommitUpdateCommand commitCmd = new CommitUpdateCommand(req, true);
// Trigger EncryptionMergePolicy.findForcedMerges() to re-encrypt
// each segment which is not encrypted with the latest active key id.
commitCmd.maxOptimizeSegments = Integer.MAX_VALUE;
req.getCore().getUpdateHandler().commit(commitCmd);
log.info("{} successfully encrypted the index in {}",
ENCRYPTION_LOG_PREFIX, elapsedTime(startTimeMs));
+
} catch (IOException e) {
log.error("{} exception while encrypting the index after {}",
ENCRYPTION_LOG_PREFIX, elapsedTime(startTimeMs), e);
} finally {
@@ -346,9 +358,9 @@ public class EncryptionRequestHandler extends
RequestHandlerBase {
});
}
- public static boolean areAllSegmentsEncryptedWithKeyId(@Nullable String
keyId,
- SolrCore core,
- SegmentInfos
segmentInfos) throws IOException {
+ private boolean areAllSegmentsEncryptedWithKeyId(@Nullable String keyId,
+ SolrCore core,
+ SegmentInfos segmentInfos)
throws IOException {
DirectoryFactory directoryFactory = core.getDirectoryFactory();
Directory indexDir = directoryFactory.get(core.getIndexDir(),
DirectoryFactory.DirContext.DEFAULT,
@@ -364,6 +376,26 @@ public class EncryptionRequestHandler extends
RequestHandlerBase {
}
}
+ private boolean areAllLogsEncryptedWithKeyId(String keyId, SolrCore core,
SegmentInfos segmentInfos)
+ throws IOException {
+ EncryptionUpdateLog updateLog = getEncryptionUpdateLog(core);
+ return updateLog == null || updateLog.areAllLogsEncryptedWithKeyId(keyId,
segmentInfos);
+ }
+
+ private EncryptionUpdateLog getEncryptionUpdateLog(SolrCore core) {
+ UpdateHandler updateHandler = core.getUpdateHandler();
+ if (updateHandler == null) {
+ return null;
+ }
+ if (!(updateHandler.getUpdateLog() instanceof EncryptionUpdateLog)) {
+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
+ UpdateLog.class.getSimpleName()
+ + " must be configured with an "
+ + EncryptionUpdateLog.class.getSimpleName());
+ }
+ return (EncryptionUpdateLog) updateHandler.getUpdateLog();
+ }
+
private boolean isCommitActiveKeyId(String keyId, SegmentInfos segmentInfos)
{
String keyRef = getActiveKeyRefFromCommit(segmentInfos.getUserData());
String activeKeyId = keyRef == null ? null : getKeyIdFromCommit(keyRef,
segmentInfos.getUserData());
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionTransactionLog.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionTransactionLog.java
index b9d842a..6b62329 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionTransactionLog.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionTransactionLog.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.encryption;
+import org.apache.solr.encryption.crypto.AesCtrEncrypterFactory;
import org.apache.solr.encryption.crypto.DecryptingChannelInputStream;
import org.apache.solr.encryption.crypto.EncryptingOutputStream;
import org.apache.solr.update.TransactionLog;
@@ -44,8 +45,8 @@ import static
org.apache.solr.encryption.crypto.AesCtrUtil.IV_LENGTH;
*/
public class EncryptionTransactionLog extends TransactionLog {
- static final int ENCRYPTION_KEY_HEADER_LENGTH = 2 * Integer.BYTES;
- private static final int ENCRYPTION_FULL_HEADER_LENGTH =
ENCRYPTION_KEY_HEADER_LENGTH + IV_LENGTH;
+ protected static final int ENCRYPTION_KEY_HEADER_LENGTH = 2 * Integer.BYTES;
+ protected static final int ENCRYPTION_FULL_HEADER_LENGTH =
ENCRYPTION_KEY_HEADER_LENGTH + IV_LENGTH;
/** Creates an {@link EncryptionTransactionLog}. */
public EncryptionTransactionLog(Path tlogFile,
@@ -55,18 +56,38 @@ public class EncryptionTransactionLog extends
TransactionLog {
this(tlogFile, globalStrings, openExisting, directorySupplier, new
IvHolder());
}
- private EncryptionTransactionLog(Path tlogFile,
+ protected EncryptionTransactionLog(Path tlogFile,
Collection<String> globalStrings,
boolean openExisting,
EncryptionDirectorySupplier
directorySupplier,
IvHolder ivHolder) {
- super(tlogFile,
+ this(tlogFile,
globalStrings,
openExisting,
new EncryptionOutputStreamOpener(directorySupplier, ivHolder),
new EncryptionChannelInputStreamOpener(directorySupplier, ivHolder));
}
+ protected EncryptionTransactionLog(Path tlogFile,
+ Collection<String> globalStrings,
+ boolean openExisting,
+ EncryptionOutputStreamOpener
outputStreamOpener,
+ EncryptionChannelInputStreamOpener
channelInputStreamOpener) {
+ super(tlogFile,
+ globalStrings,
+ openExisting,
+ outputStreamOpener,
+ channelInputStreamOpener);
+ }
+
+ public Path path() {
+ return tlog;
+ }
+
+ public int refCount() {
+ return refcount.get();
+ }
+
@Override
protected void setWrittenCount(long fileStartOffset) throws IOException {
if (os instanceof EncryptingOutputStream) {
@@ -135,23 +156,24 @@ public class EncryptionTransactionLog extends
TransactionLog {
}
/** Supplies and releases {@link EncryptionDirectory}. */
- interface EncryptionDirectorySupplier {
+ protected interface EncryptionDirectorySupplier {
EncryptionDirectory get();
void release(EncryptionDirectory directory) throws IOException;
}
- private static class IvHolder {
+ /** Holds the IV only during the constructor call. */
+ protected static class IvHolder {
private byte[] iv;
}
- private static class EncryptionOutputStreamOpener implements
OutputStreamOpener {
+ protected static class EncryptionOutputStreamOpener implements
OutputStreamOpener {
- private final EncryptionDirectorySupplier directorySupplier;
- private final IvHolder ivHolder;
+ protected final EncryptionDirectorySupplier directorySupplier;
+ protected final IvHolder ivHolder;
- EncryptionOutputStreamOpener(EncryptionDirectorySupplier
directorySupplier, IvHolder ivHolder) {
+ protected EncryptionOutputStreamOpener(EncryptionDirectorySupplier
directorySupplier, IvHolder ivHolder) {
this.directorySupplier = directorySupplier;
this.ivHolder = ivHolder;
}
@@ -169,11 +191,11 @@ public class EncryptionTransactionLog extends
TransactionLog {
// The output stream has to be wrapped to be encrypted with the key.
directory.shouldCheckEncryptionWhenReading = true;
writeEncryptionHeader(keyRef, outputStream);
- EncryptingOutputStream eos = new EncryptingOutputStream(outputStream,
- position,
- ivHolder.iv,
-
directory.getKeySecret(keyRef),
-
directory.getEncrypterFactory());
+ EncryptingOutputStream eos =
createEncryptingOutputStream(outputStream,
+ position,
+
ivHolder.iv,
+
directory.getKeySecret(keyRef),
+
directory.getEncrypterFactory());
ivHolder.iv = eos.getIv();
return eos;
}
@@ -183,15 +205,24 @@ public class EncryptionTransactionLog extends
TransactionLog {
directorySupplier.release(directory);
}
}
+
+ protected EncryptingOutputStream createEncryptingOutputStream(OutputStream
outputStream,
+ long
position,
+ byte[] iv,
+ byte[] key,
+
AesCtrEncrypterFactory factory)
+ throws IOException {
+ return new EncryptingOutputStream(outputStream, position, iv, key,
factory);
+ }
}
- private static class EncryptionChannelInputStreamOpener implements
ChannelInputStreamOpener {
+ protected static class EncryptionChannelInputStreamOpener implements
ChannelInputStreamOpener {
- private final EncryptionDirectorySupplier directorySupplier;
- private final IvHolder ivHolder;
- private final ByteBuffer readBuffer;
+ protected final EncryptionDirectorySupplier directorySupplier;
+ protected final IvHolder ivHolder;
+ protected final ByteBuffer readBuffer;
- EncryptionChannelInputStreamOpener(
+ protected EncryptionChannelInputStreamOpener(
EncryptionDirectorySupplier directorySupplier, IvHolder ivHolder) {
this.directorySupplier = directorySupplier;
this.ivHolder = ivHolder;
@@ -206,11 +237,11 @@ public class EncryptionTransactionLog extends
TransactionLog {
String keyRef = readEncryptionHeader(channel, readBuffer);
if (keyRef != null) {
// The IndexInput has to be wrapped to be decrypted with the key.
- DecryptingChannelInputStream dcis = new
DecryptingChannelInputStream(channel,
-
ENCRYPTION_KEY_HEADER_LENGTH,
-
position,
-
directory.getKeySecret(keyRef),
-
directory.getEncrypterFactory());
+ DecryptingChannelInputStream dcis =
createDecryptingChannelInputStream(channel,
+
ENCRYPTION_KEY_HEADER_LENGTH,
+
position,
+
directory.getKeySecret(keyRef),
+
directory.getEncrypterFactory());
ivHolder.iv = dcis.getIv();
return dcis;
}
@@ -221,5 +252,14 @@ public class EncryptionTransactionLog extends
TransactionLog {
ivHolder.iv = null;
return CHANNEL_INPUT_STREAM_OPENER.open(channel, position);
}
+
+ protected DecryptingChannelInputStream
createDecryptingChannelInputStream(FileChannel channel,
+
long offset,
+
long position,
+
byte[] key,
+
AesCtrEncrypterFactory factory)
+ throws IOException {
+ return new DecryptingChannelInputStream(channel, offset, position, key,
factory);
+ }
}
}
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateLog.java
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateLog.java
index de35a55..ab60c8a 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateLog.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/EncryptionUpdateLog.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.encryption;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
@@ -44,62 +45,115 @@ import java.util.Objects;
import static
org.apache.solr.encryption.EncryptionTransactionLog.ENCRYPTION_KEY_HEADER_LENGTH;
import static
org.apache.solr.encryption.EncryptionTransactionLog.readEncryptionHeader;
import static
org.apache.solr.encryption.EncryptionTransactionLog.writeEncryptionHeader;
+import static org.apache.solr.encryption.EncryptionUtil.ENCRYPTION_LOG_PREFIX;
import static
org.apache.solr.encryption.EncryptionUtil.getActiveKeyRefFromCommit;
+import static org.apache.solr.encryption.EncryptionUtil.getKeyIdFromCommit;
/**
* {@link UpdateLog} that creates {@link EncryptionUpdateLog} to encrypts logs
if the
* core index is marked for encryption.
* <p>
* The encryption is triggered when the {@link EncryptionRequestHandler} is
called. It
- * commits with some metadata that mark the index for encryption. From that
point, this
- * {@link EncryptionUpdateLog} will first re-encrypt old log files with the
active
- * encryption key id (nearly as fast as a file copy), and it will encrypt new
log files
- * with the same active key.
+ * commits with some metadata that mark the index for encryption. It also calls
+ * {@link #encryptLogs} to (re)encrypt old log files with the active
encryption key id
+ * (nearly as fast as a file copy). New log files created with {@link
#newTransactionLog}
+ * will be encrypted using the active key stored in the commit metadata.
*/
public class EncryptionUpdateLog extends UpdateLog {
private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final int REENCRYPTION_BUFFER_SIZE = 1024;
+ protected static final int REENCRYPTION_BUFFER_SIZE = 1024;
- private final DirectorySupplier directorySupplier = new DirectorySupplier();
+ protected final DirectorySupplier directorySupplier = new
DirectorySupplier();
+
+ protected boolean shouldEncryptOldLogs = true;
@Override
public void init(UpdateHandler updateHandler, SolrCore core) {
directorySupplier.init(core);
super.init(updateHandler, core);
+ try {
+ encryptLogs();
+ } catch (IOException e) {
+ log.error("{} exception while encrypting old transaction logs",
ENCRYPTION_LOG_PREFIX, e);
+ }
}
@Override
public TransactionLog newTransactionLog(Path tlogFile, Collection<String>
globalStrings, boolean openExisting) {
- if (openExisting) {
- assert Files.exists(tlogFile) : tlogFile + " does not exist";
+ return new EncryptionTransactionLog(tlogFile, globalStrings, openExisting,
directorySupplier);
+ }
+
+ /**
+ * (Re)encrypts all existing transaction logs if an encryption key is
defined in the {@link SolrCore}
+ * index commit metadata.
+ *
+ * @return {@code true} if all logs were successfully encrypted; {@code
false} if some logs were not
+ * encrypted because they are still in use, in this case the encryption will
be retried at the next commit.
+ */
+ public synchronized boolean encryptLogs() throws IOException {
+ boolean allLogsEncrypted = true;
+ for (TransactionLog log : logs) {
+ EncryptionTransactionLog encLog = (EncryptionTransactionLog) log;
+ if (encLog.refCount() <= 1) {
+ // The log is only owned by this update log. We can encrypt it.
+ encryptLog(encLog);
+ } else {
+ allLogsEncrypted = false;
+ }
+ }
+ shouldEncryptOldLogs = !allLogsEncrypted;
+ return allLogsEncrypted;
+ }
+
+ @Override
+ protected synchronized void addOldLog(TransactionLog oldLog, boolean
removeOld) {
+ super.addOldLog(oldLog, removeOld);
+ if (shouldEncryptOldLogs) {
try {
- reencryptOldLogFileIfRequired(tlogFile);
- } catch (Exception e) {
- // Absorb the exception and continue opening the transaction log.
- log.error("Failure to re-encrypt log file (non fatal) {}", tlogFile,
e);
+ encryptLogs();
+ } catch (IOException e) {
+ log.error("{} exception while encrypting old transaction logs",
ENCRYPTION_LOG_PREFIX, e);
}
}
- return new EncryptionTransactionLog(tlogFile, globalStrings, openExisting,
directorySupplier);
}
- protected void reencryptOldLogFileIfRequired(Path tlog) throws IOException {
- if (Files.size(tlog) > 0) {
+ /**
+ * Returns whether all logs are encrypted with the provided key id.
+ */
+ public synchronized boolean areAllLogsEncryptedWithKeyId(String keyId,
SegmentInfos segmentInfos) throws IOException {
+ if (!logs.isEmpty()) {
+ ByteBuffer readBuffer = ByteBuffer.allocate(4);
+ for (TransactionLog log : logs) {
+ try (FileChannel logChannel =
FileChannel.open(((EncryptionTransactionLog) log).path(),
StandardOpenOption.READ)) {
+ String logKeyRef = readEncryptionHeader(logChannel, readBuffer);
+ String logKeyId = logKeyRef == null ? null :
getKeyIdFromCommit(logKeyRef, segmentInfos.getUserData());
+ if (!Objects.equals(logKeyId, keyId)) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ protected void encryptLog(EncryptionTransactionLog log) throws IOException {
+ if (Files.size(log.path()) > 0) {
EncryptionDirectory directory = directorySupplier.get();
try {
directory.forceReadCommitUserData();
- try (FileChannel inputChannel = FileChannel.open(tlog,
StandardOpenOption.READ)) {
+ try (FileChannel inputChannel = FileChannel.open(log.path(),
StandardOpenOption.READ)) {
String inputKeyRef = readEncryptionHeader(inputChannel,
ByteBuffer.allocate(4));
String activeKeyRef =
getActiveKeyRefFromCommit(directory.getLatestCommitData().data);
if (!Objects.equals(inputKeyRef, activeKeyRef)) {
- Path newLog = tlog.resolveSibling(tlog.getFileName() + ".enc");
+ Path newLog = log.path().resolveSibling(log.path().getFileName() +
".enc");
try (OutputStream outputStream = new
FileOutputStream(newLog.toFile())) {
reencrypt(inputChannel, inputKeyRef, outputStream, activeKeyRef,
directory);
}
- Path backupLog = tlog.resolveSibling(tlog.getFileName() + ".bak");
- Files.move(tlog, backupLog);
- Files.move(newLog, tlog);
+ Path backupLog =
log.path().resolveSibling(log.path().getFileName() + ".bak");
+ Files.move(log.path(), backupLog);
+ Files.move(newLog, log.path());
Files.delete(backupLog);
}
}
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/KeySupplier.java
b/encryption/src/main/java/org/apache/solr/encryption/KeySupplier.java
index 6a533df..2bab61c 100644
--- a/encryption/src/main/java/org/apache/solr/encryption/KeySupplier.java
+++ b/encryption/src/main/java/org/apache/solr/encryption/KeySupplier.java
@@ -18,7 +18,7 @@ package org.apache.solr.encryption;
import org.apache.lucene.index.IndexFileNames;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.plugin.NamedListInitializedPlugin;
+import org.apache.solr.core.CoreContainer;
import javax.annotation.Nullable;
import java.io.Closeable;
@@ -70,10 +70,14 @@ public interface KeySupplier extends Closeable {
/**
* Creates {@link KeySupplier}.
*/
- interface Factory extends NamedListInitializedPlugin {
+ interface Factory {
- /** This factory may be configured with parameters defined in
solrconfig.xml. */
- void init(NamedList<?> args);
+ /**
+ * Initializes this factory.
+ *
+ * @param args non-null list of initialization parameters (may be empty).
+ */
+ void init(NamedList<?> args, CoreContainer coreContainer);
/** Creates a {@link KeySupplier}. */
KeySupplier create() throws IOException;
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/crypto/CharStreamEncrypter.java
b/encryption/src/main/java/org/apache/solr/encryption/crypto/CharStreamEncrypter.java
index d370a1e..15a8b7b 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/crypto/CharStreamEncrypter.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/crypto/CharStreamEncrypter.java
@@ -27,6 +27,7 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.StringReader;
+import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -50,6 +51,23 @@ public class CharStreamEncrypter {
this.factory = factory;
}
+ /**
+ * Encrypts an input string to base 64 characters compatible with JSON.
+ *
+ * @param key AES key, can either 16, 24 or 32 bytes.
+ * @return the encrypted base 64 chars.
+ */
+ public String encrypt(String input, byte[] key) {
+ StringBuilder output = new StringBuilder(input.length() * 2);
+ try {
+ encrypt(input, key, output);
+ } catch (IOException e) {
+ // Never happens when appending to a StringBuilder.
+ throw new UncheckedIOException(e);
+ }
+ return output.toString();
+ }
+
/**
* Encrypts an input string to base 64 characters compatible with JSON.
*
@@ -97,6 +115,23 @@ public class CharStreamEncrypter {
}
}
+ /**
+ * Decrypts an input string previously encrypted with {@link #encrypt}.
+ *
+ * @param key AES key, can either 16, 24 or 32 bytes.
+ * @return the decrypted chars.
+ */
+ public String decrypt(String input, byte[] key) {
+ StringBuilder output = new StringBuilder((int) (input.length() * 0.6f));
+ try {
+ decrypt(input, key, output);
+ } catch (IOException e) {
+ // Never happens when appending to a StringBuilder.
+ throw new UncheckedIOException(e);
+ }
+ return output.toString();
+ }
+
/**
* Decrypts an input string previously encrypted with {@link #encrypt}.
*
diff --git
a/encryption/src/main/java/org/apache/solr/encryption/crypto/EncryptingIndexOutput.java
b/encryption/src/main/java/org/apache/solr/encryption/crypto/EncryptingIndexOutput.java
index 7e18770..04a56f2 100644
---
a/encryption/src/main/java/org/apache/solr/encryption/crypto/EncryptingIndexOutput.java
+++
b/encryption/src/main/java/org/apache/solr/encryption/crypto/EncryptingIndexOutput.java
@@ -83,7 +83,6 @@ public class EncryptingIndexOutput extends IndexOutput {
throws IOException {
super("Encrypting " + indexOutput.toString(), indexOutput.getName());
this.indexOutput = indexOutput;
-
byte[] iv = generateRandomIv();
encrypter = factory.create(key, iv);
encrypter.init(0);
diff --git
a/encryption/src/main/java/org/apache/solr/update/TransactionLog.java
b/encryption/src/main/java/org/apache/solr/update/TransactionLog.java
index 309f783..5ced26c 100644
--- a/encryption/src/main/java/org/apache/solr/update/TransactionLog.java
+++ b/encryption/src/main/java/org/apache/solr/update/TransactionLog.java
@@ -74,7 +74,7 @@ public class TransactionLog implements Closeable {
public static final String END_MESSAGE = "SOLR_TLOG_END";
long id;
- Path tlog;
+ protected Path tlog;
protected FileChannel channel;
protected OutputStream os;
// all accesses to this stream should be synchronized on "this" (The
TransactionLog)
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionHeavyLoadTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionHeavyLoadTest.java
index aa6be0c..3459cf6 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionHeavyLoadTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionHeavyLoadTest.java
@@ -18,16 +18,12 @@ package org.apache.solr.encryption;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -44,9 +40,11 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
-import static org.apache.solr.encryption.EncryptionRequestHandler.*;
-import static
org.apache.solr.encryption.EncryptionRequestHandlerTest.EncryptionStatus;
-import static org.apache.solr.encryption.TestingKeySupplier.*;
+import static org.apache.solr.encryption.EncryptionRequestHandler.NO_KEY_ID;
+import static org.apache.solr.encryption.EncryptionTestUtil.EncryptionStatus;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_2;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_3;
/**
* Tests the encryption handler under heavy concurrent load test.
@@ -192,12 +190,12 @@ public class EncryptionHeavyLoadTest extends
SolrCloudTestCase {
private boolean encrypt(String keyId, boolean waitForCompletion) throws
Exception {
EncryptionStatus encryptionStatus = sendEncryptionRequests(keyId);
- if (!encryptionStatus.complete) {
+ if (!encryptionStatus.isComplete()) {
if (!waitForCompletion) {
return false;
}
print("waiting for encryption completion for keyId=" + keyId);
- while (!encryptionStatus.complete) {
+ while (!encryptionStatus.isComplete()) {
if (isTimeElapsed()) {
return false;
}
@@ -214,16 +212,8 @@ public class EncryptionHeavyLoadTest extends
SolrCloudTestCase {
}
private EncryptionStatus sendEncryptionRequests(String keyId) {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(PARAM_KEY_ID, keyId);
- GenericSolrRequest encryptRequest = new
GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/encrypt", params);
- EncryptionStatus encryptionStatus = new EncryptionStatus();
- testUtil.forAllReplicas(replica -> {
- NamedList<Object> response = testUtil.requestCore(encryptRequest,
replica);
- encryptionStatus.success &= response.get(STATUS).equals(STATUS_SUCCESS);
- encryptionStatus.complete &=
response.get(ENCRYPTION_STATE).equals(STATE_COMPLETE);
- });
- print("encrypt keyId=" + keyId + " => response success=" +
encryptionStatus.success + " complete=" + encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(keyId);
+ print("encrypt keyId=" + keyId + " => response success=" +
encryptionStatus.isSuccess() + " complete=" + encryptionStatus.isComplete());
return encryptionStatus;
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
index 05e3da1..5b70650 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionRequestHandlerTest.java
@@ -19,15 +19,10 @@ package org.apache.solr.encryption;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.RetryUtil;
import org.apache.solr.encryption.crypto.AesCtrEncrypterFactory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -36,10 +31,10 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import static
org.apache.solr.encryption.EncryptionDirectoryFactory.PROPERTY_INNER_ENCRYPTION_DIRECTORY_FACTORY;
-import static org.apache.solr.encryption.EncryptionRequestHandler.*;
+import static org.apache.solr.encryption.EncryptionRequestHandler.NO_KEY_ID;
+import static org.apache.solr.encryption.EncryptionTestUtil.EncryptionStatus;
import static org.apache.solr.encryption.EncryptionUtil.getKeyIdFromCommit;
import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_2;
@@ -93,9 +88,9 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
@Test
public void testEncryptionFromNoKeysToOneKey_NoIndex() throws Exception {
// Send an encrypt request with a key id on an empty index.
- EncryptionStatus encryptionStatus = encrypt(KEY_ID_1);
- assertTrue(encryptionStatus.success);
- assertTrue(encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(KEY_ID_1);
+ assertTrue(encryptionStatus.isSuccess());
+ assertTrue(encryptionStatus.isComplete());
// Index some documents to create a first segment.
testUtil.indexDocsAndCommit("weather broadcast");
@@ -111,14 +106,14 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
@Test
public void testEncryptionFromNoKeysToOneKeyToNoKeys_NoIndex() throws
Exception {
// Send an encrypt request with a key id on an empty index.
- EncryptionStatus encryptionStatus = encrypt(KEY_ID_1);
- assertTrue(encryptionStatus.success);
- assertTrue(encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(KEY_ID_1);
+ assertTrue(encryptionStatus.isSuccess());
+ assertTrue(encryptionStatus.isComplete());
// Send another encrypt request with no key id, still on the empty index.
- encryptionStatus = encrypt(NO_KEY_ID);
- assertTrue(encryptionStatus.success);
- assertTrue(encryptionStatus.complete);
+ encryptionStatus = testUtil.encrypt(NO_KEY_ID);
+ assertTrue(encryptionStatus.isSuccess());
+ assertTrue(encryptionStatus.isComplete());
// Index some documents to create a first segment.
testUtil.indexDocsAndCommit("weather broadcast");
@@ -145,11 +140,11 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
forceClearText = false;
// Send an encrypt request with a key id.
- EncryptionStatus encryptionStatus = encrypt(KEY_ID_1);
- assertTrue(encryptionStatus.success);
- assertFalse(encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(KEY_ID_1);
+ assertTrue(encryptionStatus.isSuccess());
+ assertFalse(encryptionStatus.isComplete());
- waitUntilEncryptionIsComplete(KEY_ID_1);
+ testUtil.waitUntilEncryptionIsComplete(KEY_ID_1);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -169,11 +164,11 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("foggy weather");
// Send an encrypt request with another key id.
- EncryptionStatus encryptionStatus = encrypt(KEY_ID_2);
- assertTrue(encryptionStatus.success);
- assertFalse(encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(KEY_ID_2);
+ assertTrue(encryptionStatus.isSuccess());
+ assertFalse(encryptionStatus.isComplete());
- waitUntilEncryptionIsComplete(KEY_ID_2);
+ testUtil.waitUntilEncryptionIsComplete(KEY_ID_2);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -192,11 +187,11 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("foggy weather");
// Send an encrypt request with no key id.
- EncryptionStatus encryptionStatus = encrypt(NO_KEY_ID);
- assertTrue(encryptionStatus.success);
- assertFalse(encryptionStatus.complete);
+ EncryptionStatus encryptionStatus = testUtil.encrypt(NO_KEY_ID);
+ assertTrue(encryptionStatus.isSuccess());
+ assertFalse(encryptionStatus.isComplete());
- waitUntilEncryptionIsComplete(NO_KEY_ID);
+ testUtil.waitUntilEncryptionIsComplete(NO_KEY_ID);
// Verify that the segment is cleartext.
forceClearText = true;
@@ -208,11 +203,11 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.indexDocsAndCommit("cloudy weather");
// Send an encrypt request with another key id.
- encryptionStatus = encrypt(KEY_ID_2);
- assertTrue(encryptionStatus.success);
- assertFalse(encryptionStatus.complete);
+ encryptionStatus = testUtil.encrypt(KEY_ID_2);
+ assertTrue(encryptionStatus.isSuccess());
+ assertFalse(encryptionStatus.isComplete());
- waitUntilEncryptionIsComplete(KEY_ID_2);
+ testUtil.waitUntilEncryptionIsComplete(KEY_ID_2);
// Verify that the segment is encrypted.
forceClearText = true;
@@ -223,36 +218,6 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
testUtil.assertQueryReturns("weather", 4);
}
- private EncryptionStatus encrypt(String keyId) {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(PARAM_KEY_ID, keyId);
- GenericSolrRequest encryptRequest = new
GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/encrypt", params);
- EncryptionStatus encryptionStatus = new EncryptionStatus();
- testUtil.forAllReplicas(replica -> {
- NamedList<Object> response = testUtil.requestCore(encryptRequest,
replica);
- encryptionStatus.success &= response.get(STATUS).equals(STATUS_SUCCESS);
- encryptionStatus.complete &=
response.get(ENCRYPTION_STATE).equals(STATE_COMPLETE);
- });
- return encryptionStatus;
- }
-
- private void waitUntilEncryptionIsComplete(String keyId) throws
InterruptedException {
- RetryUtil.retryUntil("Timeout waiting for encryption completion",
- 50,
- 100,
- TimeUnit.MILLISECONDS,
- () -> {
- EncryptionStatus encryptionStatus;
- try {
- encryptionStatus = encrypt(keyId);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- assertTrue(encryptionStatus.success);
- return encryptionStatus.complete;
- });
- }
-
private static void clearMockValues() {
forceClearText = false;
soleKeyIdAllowed = null;
@@ -304,9 +269,4 @@ public class EncryptionRequestHandlerTest extends
SolrCloudTestCase {
}
}
}
-
- public static class EncryptionStatus {
- public boolean success = true;
- public boolean complete = true;
- }
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
index d5c9128..61d36c4 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionTestUtil.java
@@ -23,19 +23,30 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.RetryUtil;
import org.junit.Assert;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import static
org.apache.solr.encryption.EncryptionRequestHandler.ENCRYPTION_STATE;
+import static org.apache.solr.encryption.EncryptionRequestHandler.PARAM_KEY_ID;
+import static
org.apache.solr.encryption.EncryptionRequestHandler.STATE_COMPLETE;
+import static org.apache.solr.encryption.EncryptionRequestHandler.STATUS;
+import static
org.apache.solr.encryption.EncryptionRequestHandler.STATUS_SUCCESS;
+import static org.junit.Assert.assertTrue;
+
/**
* Utility methods for encryption tests.
*/
@@ -92,6 +103,36 @@ public class EncryptionTestUtil {
Assert.assertEquals(expectedNumResults, response.getResults().size());
}
+ public EncryptionStatus encrypt(String keyId) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(PARAM_KEY_ID, keyId);
+ GenericSolrRequest encryptRequest = new
GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/encrypt", params);
+ EncryptionStatus encryptionStatus = new EncryptionStatus();
+ forAllReplicas(replica -> {
+ NamedList<Object> response = requestCore(encryptRequest, replica);
+ encryptionStatus.success &= response.get(STATUS).equals(STATUS_SUCCESS);
+ encryptionStatus.complete &=
response.get(ENCRYPTION_STATE).equals(STATE_COMPLETE);
+ });
+ return encryptionStatus;
+ }
+
+ public void waitUntilEncryptionIsComplete(String keyId) throws
InterruptedException {
+ RetryUtil.retryUntil("Timeout waiting for encryption completion",
+ 50,
+ 100,
+ TimeUnit.MILLISECONDS,
+ () -> {
+ EncryptionStatus encryptionStatus;
+ try {
+ encryptionStatus = encrypt(keyId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ assertTrue(encryptionStatus.success);
+ return encryptionStatus.complete;
+ });
+ }
+
/**
* Reloads the leader replica core of the first shard of the collection.
*/
@@ -155,4 +196,24 @@ public class EncryptionTestUtil {
super(msg, cause);
}
}
+
+ /** Status of the encryption of potentially multiple cores. */
+ public static class EncryptionStatus {
+
+ private boolean success;
+ private boolean complete;
+
+ private EncryptionStatus() {
+ this.success = true;
+ this.complete = true;
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public boolean isComplete() {
+ return complete;
+ }
+ }
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateLogTest.java
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateLogTest.java
index 1dec37c..f174336 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateLogTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/EncryptionUpdateLogTest.java
@@ -16,89 +16,78 @@
*/
package org.apache.solr.encryption;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.RetryUtil;
-import org.apache.solr.embedded.JettySolrRunner;
-import org.hamcrest.MatcherAssert;
+import org.apache.solr.encryption.crypto.AesCtrEncrypterFactory;
+import org.apache.solr.encryption.crypto.DecryptingChannelInputStream;
+import org.apache.solr.encryption.crypto.EncryptingOutputStream;
+import org.apache.solr.update.TransactionLog;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.List;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.util.Collection;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.solr.encryption.EncryptionRequestHandler.*;
-import static
org.apache.solr.encryption.TestingEncryptionUpdateLog.reencryptionCallCount;
-import static org.apache.solr.encryption.TestingKeySupplier.*;
-import static org.hamcrest.Matchers.*;
+import static org.apache.solr.encryption.EncryptionRequestHandler.NO_KEY_ID;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_1;
+import static org.apache.solr.encryption.TestingKeySupplier.KEY_ID_2;
+import static org.apache.solr.encryption.EncryptionTestUtil.EncryptionStatus;
+import static
org.apache.solr.encryption.EncryptionUpdateLogTest.TestingEncryptionUpdateLog.*;
/**
* Tests {@link EncryptionUpdateLog} and {@link EncryptionTransactionLog}.
*/
public class EncryptionUpdateLogTest extends SolrCloudTestCase {
- private static final int NUM_SHARDS = 1;
- private static final int NUM_REPLICAS = 4;
- private static final long TIMEOUT = DEFAULT_TIMEOUT;
+ private static final String COLLECTION_PREFIX =
EncryptionRequestHandlerTest.class.getSimpleName() + "-collection-";
+
+ private static final int NUM_SHARDS = 2;
+ private static final int NUM_REPLICAS = 2;
private String collectionName;
- private List<SolrClient> solrClients;
- private int nonLeaderIndex;
+ private CloudSolrClient solrClient;
+ private EncryptionTestUtil testUtil;
@BeforeClass
- public static void setupClass() throws Exception {
- configureCluster(NUM_SHARDS * NUM_REPLICAS)
+ public static void beforeClass() throws Exception {
+ EncryptionTestUtil.setInstallDirProperty();
+ System.setProperty("solr.updateLog",
TestingEncryptionUpdateLog.class.getName());
+ cluster = new MiniSolrCloudCluster.Builder(NUM_SHARDS, createTempDir())
.addConfig("config", EncryptionTestUtil.getConfigPath("collection1"))
.configure();
}
+ @AfterClass
+ public static void afterClass() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Override
@Before
- public void setupTest() throws Exception {
- collectionName = "collection" + UUID.randomUUID();
- CollectionAdminRequest.createCollection(collectionName, "config",
NUM_SHARDS, NUM_REPLICAS)
- .processAndWait(cluster.getSolrClient(), TIMEOUT);
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(
- collectionName, cluster.getZkStateReader(), false, true, TIMEOUT);
-
- solrClients = new ArrayList<>();
- nonLeaderIndex = 0;
- for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
- if (!jettySolrRunner
- .getBaseUrl()
- .toString()
-
.equals(getCollectionState(collectionName).getLeader("shard1").getBaseUrl())) {
- nonLeaderIndex = solrClients.size();
- }
- solrClients.add(jettySolrRunner.newClient());
- }
+ public void setUp() throws Exception {
+ super.setUp();
+ collectionName = COLLECTION_PREFIX + UUID.randomUUID();
+ solrClient = cluster.getSolrClient();
+ CollectionAdminRequest.createCollection(collectionName, NUM_SHARDS,
NUM_REPLICAS).process(solrClient);
+ cluster.waitForActiveCollection(collectionName, NUM_SHARDS, NUM_SHARDS *
NUM_REPLICAS);
+ testUtil = new EncryptionTestUtil(solrClient, collectionName);
}
+ @Override
@After
- public void tearDownTest() throws Exception {
- try {
- if (solrClients != null) {
- for (SolrClient solrClient : solrClients) {
- solrClient.close();
- }
- }
- } finally {
-
CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
- }
+ public void tearDown() throws Exception {
+
CollectionAdminRequest.deleteCollection(collectionName).process(solrClient);
+ super.tearDown();
}
@Test
@@ -117,149 +106,133 @@ public class EncryptionUpdateLogTest extends
SolrCloudTestCase {
}
private void checkEncryptionFromNoKeysToOneKey(String keyId) throws
Exception {
+ EncryptionStatus encryptionStatus = testUtil.encrypt(keyId);
+ assertTrue(encryptionStatus.isSuccess());
+ assertTrue(encryptionStatus.isComplete());
+
+ resetCounters();
+ testUtil.indexDocsAndCommit("doc0");
+ assertEquals(NUM_REPLICAS, encryptedLogWriteCount.get());
+ assertEquals(0, encryptedLogReadCount.get());
+ assertEquals(0, reencryptionCallCount.get());
- reencryptionCallCount.set(0);
- EncryptionStatus encryptionStatus = encrypt(keyId);
- assertTrue(encryptionStatus.statusSuccess);
- assertTrue(encryptionStatus.stateComplete);
-
- new UpdateRequest()
- .add(sdoc("id", "0", "text", "zero"))
- .commit(cluster.getSolrClient(), collectionName);
-
- for (SolrClient solrClient : solrClients) {
- checkNumUpdates(solrClient, 1);
- }
-
- cluster.getJettySolrRunner(nonLeaderIndex).stop();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(
- collectionName, cluster.getZkStateReader(), false, true, TIMEOUT);
-
- new UpdateRequest()
- .add(sdoc("id", "1", "text", "one"))
- .deleteById("2")
- .deleteByQuery("text:three")
- .commit(cluster.getSolrClient(), collectionName);
-
- cluster.getJettySolrRunner(nonLeaderIndex).start();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(
- collectionName, cluster.getZkStateReader(), false, true, TIMEOUT);
-
- for (SolrClient solrClient : solrClients) {
- checkNumUpdates(solrClient, 4);
- }
-
+ resetCounters();
+ testUtil.reloadCores();
+ assertEquals(0, encryptedLogWriteCount.get());
+ assertEquals(NUM_REPLICAS, encryptedLogReadCount.get());
assertEquals(0, reencryptionCallCount.get());
}
private void checkEncryptionFromOneKeyToAnotherKey(String fromKeyId, String
toKeyId) throws Exception {
checkEncryptionFromNoKeysToOneKey(fromKeyId);
- reencryptionCallCount.set(0);
- encrypt(fromKeyId);
- waitUntilEncryptionIsComplete(toKeyId);
-
- new UpdateRequest()
- .add(sdoc("id", "2", "text", "four"))
- .commit(cluster.getSolrClient(), collectionName);
-
- for (SolrClient solrClient : solrClients) {
- checkNumUpdates(solrClient, 5);
+ for (int i = 1 ; i <= 3; i++) {
+ resetCounters();
+ testUtil.indexDocsAndCommit("doc" + i);
+ assertEquals(NUM_REPLICAS, encryptedLogWriteCount.get());
+ assertEquals(0, encryptedLogReadCount.get());
+ assertEquals(0, reencryptionCallCount.get());
}
- cluster.getJettySolrRunner(nonLeaderIndex).stop();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(
- collectionName, cluster.getZkStateReader(), false, true, TIMEOUT);
+ resetCounters();
+ EncryptionStatus encryptionStatus = testUtil.encrypt(toKeyId);
+ assertTrue(encryptionStatus.isSuccess());
+ testUtil.waitUntilEncryptionIsComplete(toKeyId);
+ assertEquals(0, encryptedLogWriteCount.get());
+ assertEquals(0, encryptedLogReadCount.get());
+ assertEquals(4 * NUM_REPLICAS, reencryptionCallCount.get());
+
+ resetCounters();
+ testUtil.reloadCores();
+ assertEquals(0, encryptedLogWriteCount.get());
+ assertEquals(4 * NUM_REPLICAS, encryptedLogReadCount.get());
+ assertEquals(0, reencryptionCallCount.get());
+ }
- new UpdateRequest()
- .add(sdoc("id", "3", "text", "five"))
- .deleteById("6")
- .deleteByQuery("text:seven")
- .commit(cluster.getSolrClient(), collectionName);
+ public static class TestingEncryptionUpdateLog extends EncryptionUpdateLog {
- cluster.getJettySolrRunner(nonLeaderIndex).start();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(
- collectionName, cluster.getZkStateReader(), false, true, TIMEOUT);
+ static final AtomicInteger reencryptionCallCount = new AtomicInteger();
+ static final AtomicInteger encryptedLogWriteCount = new AtomicInteger();
+ static final AtomicInteger encryptedLogReadCount = new AtomicInteger();
- for (SolrClient solrClient : solrClients) {
- checkNumUpdates(solrClient, 8);
+ static void resetCounters() {
+ reencryptionCallCount.set(0);
+ encryptedLogWriteCount.set(0);
+ encryptedLogReadCount.set(0);
}
- MatcherAssert.assertThat(reencryptionCallCount.get(),
greaterThanOrEqualTo(1));
- }
+ @Override
+ public TransactionLog newTransactionLog(Path tlogFile, Collection<String>
globalStrings, boolean openExisting) {
+ return new TestingTransactionLog(tlogFile, globalStrings, openExisting,
directorySupplier);
+ }
- private EncryptionStatus encrypt(String keyId) throws Exception {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(PARAM_KEY_ID, keyId);
-
- boolean statusSuccess = true;
- boolean stateComplete = true;
- for (SolrClient solrClient : solrClients) {
- NamedList<Object> response = solrClient.request(
- new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/encrypt",
params), collectionName);
- statusSuccess &= STATUS_SUCCESS.equals(response.get(STATUS));
- stateComplete &= STATE_COMPLETE.equals(response.get(ENCRYPTION_STATE));
+ protected void reencrypt(FileChannel inputChannel,
+ String inputKeyRef,
+ OutputStream outputStream,
+ String activeKeyRef,
+ EncryptionDirectory directory)
+ throws IOException {
+ reencryptionCallCount.incrementAndGet();
+ super.reencrypt(inputChannel, inputKeyRef, outputStream, activeKeyRef,
directory);
}
- return new EncryptionStatus(statusSuccess, stateComplete);
- }
- private void waitUntilEncryptionIsComplete(String keyId) throws
InterruptedException {
- RetryUtil.retryUntil("Timeout waiting for encryption completion",
- 50,
- 100,
- TimeUnit.MILLISECONDS,
- () -> {
- EncryptionStatus encryptionStatus;
- try {
- encryptionStatus = encrypt(keyId);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- assertTrue(encryptionStatus.statusSuccess);
- return encryptionStatus.stateComplete;
- });
- }
+ private static class TestingTransactionLog extends
EncryptionTransactionLog {
- @SuppressWarnings("unchecked")
- private void checkNumUpdates(SolrClient solrClient, int numExpected) throws
Exception {
+ TestingTransactionLog(Path tlogFile,
+ Collection<String> globalStrings,
+ boolean openExisting,
+ EncryptionDirectorySupplier directorySupplier) {
+ this(tlogFile, globalStrings, openExisting, directorySupplier, new
IvHolder());
+ }
- final QueryRequest reqV = new QueryRequest(params("qt", "/get",
"getVersions", "12345"));
- final NamedList<?> rspV = solrClient.request(reqV, collectionName);
- final List<Long> versions = (List<Long>) rspV.get("versions");
- assertEquals(versions.toString(), numExpected, versions.size());
- if (numExpected == 0) {
- return;
- }
+ TestingTransactionLog(Path tlogFile,
+ Collection<String> globalStrings,
+ boolean openExisting,
+ EncryptionDirectorySupplier directorySupplier,
+ IvHolder ivHolder) {
+ super(tlogFile,
+ globalStrings,
+ openExisting,
+ new TestingEncryptionOutputStreamOpener(directorySupplier,
ivHolder),
+ new TestingEncryptionChannelInputStreamOpener(directorySupplier,
ivHolder));
+ }
- final Deque<Long> absVersions =
-
versions.stream().map(Math::abs).sorted().collect(Collectors.toCollection(ArrayDeque::new));
- final Long minVersion = absVersions.getFirst();
- final Long maxVersion = absVersions.getLast();
-
- for (boolean skipDbq : new boolean[]{false, true}) {
- final QueryRequest reqU =
- new QueryRequest(
- params(
- "qt",
- "/get",
- "getUpdates",
- minVersion + "..." + maxVersion,
- "skipDbq",
- Boolean.toString(skipDbq)));
- final NamedList<?> rspU = solrClient.request(reqU, collectionName);
- final List<?> updatesList = (List<?>) rspU.get("updates");
- assertEquals(updatesList.toString(), numExpected, updatesList.size());
- }
- }
+ static class TestingEncryptionOutputStreamOpener extends
EncryptionOutputStreamOpener {
+
+ TestingEncryptionOutputStreamOpener(EncryptionDirectorySupplier
directorySupplier, IvHolder ivHolder) {
+ super(directorySupplier, ivHolder);
+ }
- private static class EncryptionStatus {
+ @Override
+ protected EncryptingOutputStream
createEncryptingOutputStream(OutputStream outputStream,
+ long
position,
+ byte[]
iv,
+ byte[]
key,
+
AesCtrEncrypterFactory factory)
+ throws IOException {
+ encryptedLogWriteCount.incrementAndGet();
+ return super.createEncryptingOutputStream(outputStream, position,
iv, key, factory);
+ }
+ }
- final boolean statusSuccess;
- final boolean stateComplete;
+ static class TestingEncryptionChannelInputStreamOpener extends
EncryptionChannelInputStreamOpener {
- EncryptionStatus(boolean statusSuccess, boolean stateComplete) {
- this.statusSuccess = statusSuccess;
- this.stateComplete = stateComplete;
+ TestingEncryptionChannelInputStreamOpener(EncryptionDirectorySupplier
directorySupplier,
+ IvHolder ivHolder)
{
+ super(directorySupplier, ivHolder);
+ }
+
+ @Override
+ protected DecryptingChannelInputStream
createDecryptingChannelInputStream(FileChannel channel,
+
long offset,
+
long position,
+
byte[] key,
+
AesCtrEncrypterFactory factory)
+ throws IOException {
+ encryptedLogReadCount.incrementAndGet();
+ return super.createDecryptingChannelInputStream(channel, offset,
position, key, factory);
+ }
+ }
}
}
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/TestingKeySupplier.java
b/encryption/src/test/java/org/apache/solr/encryption/TestingKeySupplier.java
index 6c4556e..3dafea0 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/TestingKeySupplier.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/TestingKeySupplier.java
@@ -18,6 +18,7 @@ package org.apache.solr.encryption;
import org.apache.lucene.index.IndexFileNames;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -177,7 +178,7 @@ public class TestingKeySupplier implements KeySupplier {
private static final KeySupplier SINGLETON = new TestingKeySupplier();
@Override
- public void init(NamedList<?> args) {
+ public void init(NamedList<?> args, CoreContainer coreContainer) {
// Do nothing.
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/AesCtrEncrypterTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/AesCtrEncrypterTest.java
index c8f2809..8ec4eff 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/AesCtrEncrypterTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/AesCtrEncrypterTest.java
@@ -20,6 +20,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest;
import org.junit.Test;
import static org.apache.solr.encryption.crypto.AesCtrUtil.*;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
import static org.junit.Assert.assertArrayEquals;
/**
@@ -55,13 +56,6 @@ public class AesCtrEncrypterTest extends RandomizedTest {
}
}
- private AesCtrEncrypterFactory encrypterFactory() {
- if (LightAesCtrEncrypter.isSupported()) {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
- return CipherAesCtrEncrypter.FACTORY;
- }
-
private static byte[] generateRandomBytes(int numBytes) {
byte[] b = new byte[numBytes];
// Random.nextBytes(byte[]) does not produce good enough randomness here,
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/CharStreamEncrypterTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/CharStreamEncrypterTest.java
index d6aa764..ed9f098 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/CharStreamEncrypterTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/CharStreamEncrypterTest.java
@@ -21,6 +21,7 @@ import org.junit.Test;
import java.io.IOException;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
import static org.junit.Assert.assertEquals;
/** Tests {@link CharStreamEncrypter}. */
@@ -54,11 +55,4 @@ public class CharStreamEncrypterTest extends RandomizedTest {
encrypter.decrypt(encryptedBuilder.toString(), key, decryptedBuilder);
assertEquals(inputString, decryptedBuilder.toString());
}
-
- private AesCtrEncrypterFactory encrypterFactory() {
- if (LightAesCtrEncrypter.isSupported()) {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
- return CipherAesCtrEncrypter.FACTORY;
- }
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/TestingEncryptionUpdateLog.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/CryptoTestUtil.java
similarity index 52%
rename from
encryption/src/test/java/org/apache/solr/encryption/TestingEncryptionUpdateLog.java
rename to
encryption/src/test/java/org/apache/solr/encryption/crypto/CryptoTestUtil.java
index 497c368..7a4044f 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/TestingEncryptionUpdateLog.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/CryptoTestUtil.java
@@ -14,24 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.encryption;
+package org.apache.solr.encryption.crypto;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.channels.FileChannel;
-import java.util.concurrent.atomic.AtomicInteger;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
-public class TestingEncryptionUpdateLog extends EncryptionUpdateLog {
-
- public static AtomicInteger reencryptionCallCount = new AtomicInteger();
+/**
+ * Tools for encryption tests.
+ */
+public class CryptoTestUtil {
- protected void reencrypt(FileChannel inputChannel,
- String inputKeyRef,
- OutputStream outputStream,
- String activeKeyRef,
- EncryptionDirectory directory)
- throws IOException {
- super.reencrypt(inputChannel, inputKeyRef, outputStream, activeKeyRef,
directory);
- reencryptionCallCount.incrementAndGet();
+ /**
+ * Provides an {@link AesCtrEncrypterFactory} selected randomly.
+ */
+ public static AesCtrEncrypterFactory encrypterFactory() {
+ if (LightAesCtrEncrypter.isSupported()) {
+ return RandomizedTest.randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
+ }
+ return CipherAesCtrEncrypter.FACTORY;
}
}
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingChannelInputStreamTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingChannelInputStreamTest.java
index 592037a..621e193 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingChannelInputStreamTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingChannelInputStreamTest.java
@@ -30,6 +30,7 @@ import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import static org.apache.solr.encryption.crypto.AesCtrUtil.IV_LENGTH;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -80,10 +81,6 @@ public class DecryptingChannelInputStreamTest extends
RandomizedTest {
assertArrayEquals(source, decryptedBytes);
}
- private static AesCtrEncrypterFactory encrypterFactory() {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
-
private static void write(byte[] source, int offset, int length,
OutputStream os) throws IOException {
if (length > 0) {
byte[] buffer = new byte[randomIntBetween(2, Math.max(length, 2))];
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingIndexInputTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingIndexInputTest.java
index 7caf30b..0e6337d 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingIndexInputTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/DecryptingIndexInputTest.java
@@ -35,6 +35,7 @@ import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.Before;
import org.junit.Test;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -262,8 +263,4 @@ public class DecryptingIndexInputTest extends
RandomizedTest {
indexInput.seek(offset);
return new DecryptingIndexInput(indexInput, key, encrypterFactory());
}
-
- private AesCtrEncrypterFactory encrypterFactory() {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
}
\ No newline at end of file
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingIndexOutputTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingIndexOutputTest.java
index edb6849..8f69d6d 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingIndexOutputTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingIndexOutputTest.java
@@ -35,6 +35,7 @@ import org.junit.Test;
import static com.carrotsearch.randomizedtesting.RandomizedTest.*;
import static org.apache.solr.encryption.crypto.AesCtrUtil.*;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
/**
* Tests {@link EncryptingIndexOutput}.
@@ -117,10 +118,6 @@ public class EncryptingIndexOutputTest extends
BaseDataOutputTestCase<Encrypting
}
}
- private AesCtrEncrypterFactory encrypterFactory() {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
-
/**
* Replaces the {@link java.security.SecureRandom} by a repeatable {@link
Random} for tests.
* This is used to generate a repeatable random IV.
diff --git
a/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingOutputStreamTest.java
b/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingOutputStreamTest.java
index ad97069..801ffbd 100644
---
a/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingOutputStreamTest.java
+++
b/encryption/src/test/java/org/apache/solr/encryption/crypto/EncryptingOutputStreamTest.java
@@ -26,6 +26,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import static org.apache.solr.encryption.crypto.AesCtrUtil.IV_LENGTH;
+import static
org.apache.solr.encryption.crypto.CryptoTestUtil.encrypterFactory;
import static org.junit.Assert.*;
/**
@@ -117,10 +118,6 @@ public class EncryptingOutputStreamTest extends
RandomizedTest {
assertArrayEquals(source, decryptedBytes);
}
- private static AesCtrEncrypterFactory encrypterFactory() {
- return randomBoolean() ? CipherAesCtrEncrypter.FACTORY :
LightAesCtrEncrypter.FACTORY;
- }
-
private static void write(byte[] source, int offset, int length,
OutputStream os) throws IOException {
if (length > 0) {
byte[] buffer = new byte[randomIntBetween(2, Math.max(length, 2))];
diff --git a/encryption/src/test/resources/configs/collection1/solrconfig.xml
b/encryption/src/test/resources/configs/collection1/solrconfig.xml
index 2680a13..d9a7bb8 100644
--- a/encryption/src/test/resources/configs/collection1/solrconfig.xml
+++ b/encryption/src/test/resources/configs/collection1/solrconfig.xml
@@ -47,7 +47,7 @@
<!-- EncryptionUpdateHandler transfers the encryption key ids from a commit
to the next. -->
<updateHandler class="org.apache.solr.encryption.EncryptionUpdateHandler">
<!-- EncryptionUpdateLog encrypts transaction logs if the index is
encrypted. -->
- <updateLog class="org.apache.solr.encryption.TestingEncryptionUpdateLog"/>
+ <updateLog
class="${solr.updateLog:org.apache.solr.encryption.EncryptionUpdateLog}"/>
</updateHandler>
<!-- Encryption handler can be called to trigger the encryption of the index
and update log. -->
