This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new ab7a30c SOLR-16026: Enable spotless on gcs-repository module
ab7a30c is described below
commit ab7a30c1b089a95d276d540535bfb9959d229c27
Author: Kevin Risden <[email protected]>
AuthorDate: Sat Feb 19 10:26:30 2022 -0500
SOLR-16026: Enable spotless on gcs-repository module
---
gradle/validation/spotless.gradle | 1 -
.../org/apache/solr/gcs/GCSBackupRepository.java | 825 +++++++++++----------
.../java/org/apache/solr/gcs/GCSConfigParser.java | 127 +++-
.../src/java/org/apache/solr/gcs/package-info.java | 6 +-
.../solr/gcs/ConcurrentDelegatingStorage.java | 322 ++++++--
.../apache/solr/gcs/GCSBackupRepositoryTest.java | 74 +-
.../apache/solr/gcs/GCSIncrementalBackupTest.java | 119 +--
.../solr/gcs/LocalStorageGCSBackupRepository.java | 35 +-
8 files changed, 867 insertions(+), 642 deletions(-)
diff --git a/gradle/validation/spotless.gradle
b/gradle/validation/spotless.gradle
index 931ea5d..cd7ccde 100644
--- a/gradle/validation/spotless.gradle
+++ b/gradle/validation/spotless.gradle
@@ -44,7 +44,6 @@ configure(project(":solr").subprojects) { prj ->
// Exclude certain files (generated ones, mostly).
switch (project.path) {
- case ":solr:modules:gcs-repository":
case ":solr:modules:hadoop-auth":
case ":solr:modules:hdfs":
case ":solr:modules:sql":
diff --git
a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java
b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java
index fb3b5a4..6d110f4 100644
---
a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java
+++
b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java
@@ -17,6 +17,8 @@
package org.apache.solr.gcs;
+import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;
+
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.ReadChannel;
import com.google.cloud.WriteChannel;
@@ -26,21 +28,6 @@ import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.BufferedIndexInput;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.solr.common.params.CoreAdminParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory;
-import org.apache.solr.core.backup.repository.BackupRepository;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -58,436 +45,450 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;
-
-/**
- * {@link BackupRepository} implementation that stores files in Google Cloud
Storage ("GCS").
- */
+/** {@link BackupRepository} implementation that stores files in Google Cloud
Storage ("GCS"). */
public class GCSBackupRepository implements BackupRepository {
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static final int LARGE_BLOB_THRESHOLD_BYTE_SIZE = 5 * 1024 * 1024;
- private static final Storage.BlobWriteOption[] NO_WRITE_OPTIONS = new
Storage.BlobWriteOption[0];
-
- protected Storage storage;
-
- private NamedList<?> config = null;
- protected String bucketName = null;
- protected String credentialPath = null;
- protected int writeBufferSizeBytes;
- protected int readBufferSizeBytes;
- protected StorageOptions.Builder storageOptionsBuilder = null;
-
- protected Storage initStorage() {
- if (storage != null)
- return storage;
-
- try {
- if (credentialPath != null) {
- log.info("Creating GCS client using credential at {}",
credentialPath);
- // 'GoogleCredentials.fromStream' closes the input stream, so
we don't
- GoogleCredentials credential =
GoogleCredentials.fromStream(new FileInputStream(credentialPath));
- storageOptionsBuilder.setCredentials(credential);
- } else {
- // nowarn compile time string concatenation
- log.warn(GCSConfigParser.potentiallyMissingCredentialMsg());
//nowarn
- }
- storage = storageOptionsBuilder.build().getService();
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- return storage;
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final int LARGE_BLOB_THRESHOLD_BYTE_SIZE = 5 * 1024 * 1024;
+ private static final Storage.BlobWriteOption[] NO_WRITE_OPTIONS = new
Storage.BlobWriteOption[0];
+
+ protected Storage storage;
+
+ private NamedList<?> config = null;
+ protected String bucketName = null;
+ protected String credentialPath = null;
+ protected int writeBufferSizeBytes;
+ protected int readBufferSizeBytes;
+ protected StorageOptions.Builder storageOptionsBuilder = null;
+
+ protected Storage initStorage() {
+ if (storage != null) return storage;
+
+ try {
+ if (credentialPath != null) {
+ log.info("Creating GCS client using credential at {}", credentialPath);
+ // 'GoogleCredentials.fromStream' closes the input stream, so we don't
+ GoogleCredentials credential =
+ GoogleCredentials.fromStream(new FileInputStream(credentialPath));
+ storageOptionsBuilder.setCredentials(credential);
+ } else {
+ // nowarn compile time string concatenation
+ log.warn(GCSConfigParser.potentiallyMissingCredentialMsg()); // nowarn
+ }
+ storage = storageOptionsBuilder.build().getService();
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
}
-
- @Override
- public void init(NamedList<?> args) {
- this.config = args;
- final GCSConfigParser configReader = new GCSConfigParser();
- final GCSConfigParser.GCSConfig parsedConfig =
configReader.parseConfiguration(config);
-
- this.bucketName = parsedConfig.getBucketName();
- this.credentialPath = parsedConfig.getCredentialPath();
- this.writeBufferSizeBytes = parsedConfig.getWriteBufferSize();
- this.readBufferSizeBytes = parsedConfig.getReadBufferSize();
- this.storageOptionsBuilder = parsedConfig.getStorageOptionsBuilder();
-
- initStorage();
+ return storage;
+ }
+
+ @Override
+ public void init(NamedList<?> args) {
+ this.config = args;
+ final GCSConfigParser configReader = new GCSConfigParser();
+ final GCSConfigParser.GCSConfig parsedConfig =
configReader.parseConfiguration(config);
+
+ this.bucketName = parsedConfig.getBucketName();
+ this.credentialPath = parsedConfig.getCredentialPath();
+ this.writeBufferSizeBytes = parsedConfig.getWriteBufferSize();
+ this.readBufferSizeBytes = parsedConfig.getReadBufferSize();
+ this.storageOptionsBuilder = parsedConfig.getStorageOptionsBuilder();
+
+ initStorage();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T getConfigProperty(String name) {
+ return (T) this.config.get(name);
+ }
+
+ @Override
+ public URI createURI(String location) {
+ Objects.requireNonNull(location);
+
+ URI result;
+ try {
+ result = new URI(location);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException("Error on creating URI", e);
}
- @Override
- @SuppressWarnings("unchecked")
- public <T> T getConfigProperty(String name) {
- return (T) this.config.get(name);
- }
-
- @Override
- public URI createURI(String location) {
- Objects.requireNonNull(location);
+ return result;
+ }
- URI result;
- try {
- result = new URI(location);
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Error on creating URI", e);
- }
+ @Override
+ public URI createDirectoryURI(String location) {
+ Objects.requireNonNull(location);
- return result;
+ if (!location.endsWith("/")) {
+ location += "/";
}
- @Override
- public URI createDirectoryURI(String location) {
- Objects.requireNonNull(location);
+ return createURI(location);
+ }
- if (!location.endsWith("/")) {
- location += "/";
+ @Override
+ public URI resolve(URI baseUri, String... pathComponents) {
+ StringBuilder builder = new StringBuilder(baseUri.toString());
+ for (String path : pathComponents) {
+ if (path != null && !path.isEmpty()) {
+ if (builder.charAt(builder.length() - 1) != '/') {
+ builder.append('/');
}
-
- return createURI(location);
+ builder.append(path);
+ }
}
- @Override
- public URI resolve(URI baseUri, String... pathComponents) {
- StringBuilder builder = new StringBuilder(baseUri.toString());
- for (String path : pathComponents) {
- if (path != null && !path.isEmpty()) {
- if (builder.charAt(builder.length()-1) != '/') {
- builder.append('/');
- }
- builder.append(path);
- }
- }
-
- return URI.create(builder.toString()).normalize();
+ return URI.create(builder.toString()).normalize();
+ }
+
+ @Override
+ public URI resolveDirectory(URI baseUri, String... pathComponents) {
+ if (pathComponents.length > 0) {
+ if (!pathComponents[pathComponents.length - 1].endsWith("/")) {
+ pathComponents[pathComponents.length - 1] =
pathComponents[pathComponents.length - 1] + "/";
+ }
+ } else {
+ if (!baseUri.getPath().endsWith("/")) {
+ baseUri = URI.create(baseUri + "/");
+ }
}
+ return resolve(baseUri, pathComponents);
+ }
- @Override
- public URI resolveDirectory(URI baseUri, String... pathComponents) {
- if (pathComponents.length > 0) {
- if (!pathComponents[pathComponents.length - 1].endsWith("/")) {
- pathComponents[pathComponents.length - 1] =
pathComponents[pathComponents.length - 1] + "/";
- }
- } else {
- if (!baseUri.getPath().endsWith("/")) {
- baseUri = URI.create(baseUri + "/");
- }
- }
- return resolve(baseUri, pathComponents);
- }
+ @Override
+ public boolean exists(URI path) throws IOException {
+ return exists(path.toString());
+ }
- @Override
- public boolean exists(URI path) throws IOException {
- return exists(path.toString());
+ public boolean exists(String path) throws IOException {
+ if (path.equals(getConfigProperty(CoreAdminParams.BACKUP_LOCATION))) {
+ return true;
}
- public boolean exists(String path) throws IOException {
- if (path.equals(getConfigProperty(CoreAdminParams.BACKUP_LOCATION))) {
- return true;
- }
-
- if (path.endsWith("/")) {
- return storage.get(bucketName, path,
Storage.BlobGetOption.fields()) != null;
- } else {
- final String filePath = path;
- final String directoryPath = path + "/";
- return storage.get(bucketName, filePath,
Storage.BlobGetOption.fields()) != null ||
- storage.get(bucketName, directoryPath,
Storage.BlobGetOption.fields()) != null;
- }
- }
-
- @Override
- public PathType getPathType(URI path) throws IOException {
- if (path.toString().endsWith("/"))
- return PathType.DIRECTORY;
-
- Blob blob = storage.get(bucketName, path.toString()+"/",
Storage.BlobGetOption.fields());
- if (blob != null)
- return PathType.DIRECTORY;
-
- return PathType.FILE;
+ if (path.endsWith("/")) {
+ return storage.get(bucketName, path, Storage.BlobGetOption.fields()) !=
null;
+ } else {
+ final String filePath = path;
+ final String directoryPath = path + "/";
+ return storage.get(bucketName, filePath, Storage.BlobGetOption.fields())
!= null
+ || storage.get(bucketName, directoryPath,
Storage.BlobGetOption.fields()) != null;
}
-
- @Override
- public String[] listAll(URI path) throws IOException {
- final String blobName =
appendTrailingSeparatorIfNecessary(path.toString());
-
- final String pathStr = blobName;
- final LinkedList<String> result = new LinkedList<>();
- storage.list(
- bucketName,
- Storage.BlobListOption.currentDirectory(),
- Storage.BlobListOption.prefix(pathStr),
- Storage.BlobListOption.fields())
- .iterateAll().forEach(
- blob -> {
- assert blob.getName().startsWith(pathStr);
- final String suffixName =
blob.getName().substring(pathStr.length());
- if (!suffixName.isEmpty()) {
- // Remove trailing '/' if present
- if (suffixName.endsWith("/")) {
- result.add(suffixName.substring(0,
suffixName.length() - 1));
- } else {
- result.add(suffixName);
- }
- }
- });
-
- return result.toArray(new String[0]);
- }
-
- @Override
- public IndexInput openInput(URI dirPath, String fileName, IOContext ctx)
throws IOException {
- return openInput(dirPath, fileName, ctx, readBufferSizeBytes);
- }
-
- private IndexInput openInput(URI dirPath, String fileName, IOContext ctx,
int bufferSize) {
- String blobName = resolve(dirPath, fileName).toString();
-
- final BlobId blobId = BlobId.of(bucketName, blobName);
- final Blob blob = storage.get(blobId,
Storage.BlobGetOption.fields(Storage.BlobField.SIZE));
- final ReadChannel readChannel = blob.reader();
- readChannel.setChunkSize(bufferSize);
-
- return new BufferedIndexInput(blobName, bufferSize) {
-
- @Override
- public long length() {
- return blob.getSize();
- }
-
- @Override
- protected void readInternal(ByteBuffer b) throws IOException {
- readChannel.read(b);
- }
-
- @Override
- protected void seekInternal(long pos) throws IOException {
- readChannel.seek(pos);
- }
-
- @Override
- public void close() throws IOException {
- readChannel.close();
- }
- };
- }
-
- @Override
- public OutputStream createOutput(URI path) throws IOException {
- final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName,
path.toString()).build();
- final WriteChannel writeChannel = storage.writer(blobInfo,
getDefaultBlobWriteOptions());
-
- return Channels.newOutputStream(new WritableByteChannel() {
- @Override
- public int write(ByteBuffer src) throws IOException {
- return writeChannel.write(src);
- }
-
- @Override
- public boolean isOpen() {
- return writeChannel.isOpen();
- }
-
- @Override
- public void close() throws IOException {
- writeChannel.close();
- }
+ }
+
+ @Override
+ public PathType getPathType(URI path) throws IOException {
+ if (path.toString().endsWith("/")) return PathType.DIRECTORY;
+
+ Blob blob = storage.get(bucketName, path.toString() + "/",
Storage.BlobGetOption.fields());
+ if (blob != null) return PathType.DIRECTORY;
+
+ return PathType.FILE;
+ }
+
+ @Override
+ public String[] listAll(URI path) throws IOException {
+ final String blobName =
appendTrailingSeparatorIfNecessary(path.toString());
+
+ final String pathStr = blobName;
+ final LinkedList<String> result = new LinkedList<>();
+ storage
+ .list(
+ bucketName,
+ Storage.BlobListOption.currentDirectory(),
+ Storage.BlobListOption.prefix(pathStr),
+ Storage.BlobListOption.fields())
+ .iterateAll()
+ .forEach(
+ blob -> {
+ assert blob.getName().startsWith(pathStr);
+ final String suffixName =
blob.getName().substring(pathStr.length());
+ if (!suffixName.isEmpty()) {
+ // Remove trailing '/' if present
+ if (suffixName.endsWith("/")) {
+ result.add(suffixName.substring(0, suffixName.length() - 1));
+ } else {
+ result.add(suffixName);
+ }
+ }
+ });
+
+ return result.toArray(new String[0]);
+ }
+
+ @Override
+ public IndexInput openInput(URI dirPath, String fileName, IOContext ctx)
throws IOException {
+ return openInput(dirPath, fileName, ctx, readBufferSizeBytes);
+ }
+
+ private IndexInput openInput(URI dirPath, String fileName, IOContext ctx,
int bufferSize) {
+ String blobName = resolve(dirPath, fileName).toString();
+
+ final BlobId blobId = BlobId.of(bucketName, blobName);
+ final Blob blob = storage.get(blobId,
Storage.BlobGetOption.fields(Storage.BlobField.SIZE));
+ final ReadChannel readChannel = blob.reader();
+ readChannel.setChunkSize(bufferSize);
+
+ return new BufferedIndexInput(blobName, bufferSize) {
+
+ @Override
+ public long length() {
+ return blob.getSize();
+ }
+
+ @Override
+ protected void readInternal(ByteBuffer b) throws IOException {
+ readChannel.read(b);
+ }
+
+ @Override
+ protected void seekInternal(long pos) throws IOException {
+ readChannel.seek(pos);
+ }
+
+ @Override
+ public void close() throws IOException {
+ readChannel.close();
+ }
+ };
+ }
+
+ @Override
+ public OutputStream createOutput(URI path) throws IOException {
+ final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName,
path.toString()).build();
+ final WriteChannel writeChannel = storage.writer(blobInfo,
getDefaultBlobWriteOptions());
+
+ return Channels.newOutputStream(
+ new WritableByteChannel() {
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ return writeChannel.write(src);
+ }
+
+ @Override
+ public boolean isOpen() {
+ return writeChannel.isOpen();
+ }
+
+ @Override
+ public void close() throws IOException {
+ writeChannel.close();
+ }
});
- }
+ }
- @Override
- public void createDirectory(URI path) throws IOException {
- final String name =
appendTrailingSeparatorIfNecessary(path.toString());
- if (!exists(name)) {
- storage.create(BlobInfo.newBuilder(bucketName, name).build());
- }
+ @Override
+ public void createDirectory(URI path) throws IOException {
+ final String name = appendTrailingSeparatorIfNecessary(path.toString());
+ if (!exists(name)) {
+ storage.create(BlobInfo.newBuilder(bucketName, name).build());
}
-
- @Override
- public void deleteDirectory(URI path) throws IOException {
- List<BlobId> blobIds = allBlobsAtDir(path);
- if (!blobIds.isEmpty()) {
- storage.delete(blobIds);
- } else {
- log.debug("Path:{} doesn't have any blobs", path);
- }
+ }
+
+ @Override
+ public void deleteDirectory(URI path) throws IOException {
+ List<BlobId> blobIds = allBlobsAtDir(path);
+ if (!blobIds.isEmpty()) {
+ storage.delete(blobIds);
+ } else {
+ log.debug("Path:{} doesn't have any blobs", path);
}
-
- protected List<BlobId> allBlobsAtDir(URI path) throws IOException {
- final String blobName =
appendTrailingSeparatorIfNecessary(path.toString());
-
- final List<BlobId> result = new ArrayList<>();
- final String pathStr = blobName;
- storage.list(
- bucketName,
- Storage.BlobListOption.prefix(pathStr),
- Storage.BlobListOption.fields())
- .iterateAll().forEach(
- blob -> result.add(blob.getBlobId())
- );
-
- return result;
-
+ }
+
+ protected List<BlobId> allBlobsAtDir(URI path) throws IOException {
+ final String blobName =
appendTrailingSeparatorIfNecessary(path.toString());
+
+ final List<BlobId> result = new ArrayList<>();
+ final String pathStr = blobName;
+ storage
+ .list(bucketName, Storage.BlobListOption.prefix(pathStr),
Storage.BlobListOption.fields())
+ .iterateAll()
+ .forEach(blob -> result.add(blob.getBlobId()));
+
+ return result;
+ }
+
+ @Override
+ public void delete(URI path, Collection<String> files, boolean
ignoreNoSuchFileException)
+ throws IOException {
+ if (files.isEmpty()) {
+ return;
}
-
- @Override
- public void delete(URI path, Collection<String> files, boolean
ignoreNoSuchFileException) throws IOException {
- if (files.isEmpty()) {
- return;
- }
- final String prefix =
appendTrailingSeparatorIfNecessary(path.toString());
- List<BlobId> blobDeletes = files.stream()
- .map(file -> BlobId.of(bucketName, prefix + file))
- .collect(Collectors.toList());
- List<Boolean> result = storage.delete(blobDeletes);
- if (!ignoreNoSuchFileException) {
- int failedDelete = result.indexOf(Boolean.FALSE);
- if (failedDelete != -1) {
- throw new NoSuchFileException("File " +
blobDeletes.get(failedDelete).getName() + " was not found");
- }
- }
+ final String prefix = appendTrailingSeparatorIfNecessary(path.toString());
+ List<BlobId> blobDeletes =
+ files.stream()
+ .map(file -> BlobId.of(bucketName, prefix + file))
+ .collect(Collectors.toList());
+ List<Boolean> result = storage.delete(blobDeletes);
+ if (!ignoreNoSuchFileException) {
+ int failedDelete = result.indexOf(Boolean.FALSE);
+ if (failedDelete != -1) {
+ throw new NoSuchFileException(
+ "File " + blobDeletes.get(failedDelete).getName() + " was not
found");
+ }
}
-
- @Override
- public void copyIndexFileFrom(Directory sourceDir, String sourceFileName,
URI destDir, String destFileName) throws IOException {
- String blobName = destDir.toString();
- blobName = appendTrailingSeparatorIfNecessary(blobName);
- blobName += destFileName;
- final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName,
blobName).build();
- try (ChecksumIndexInput input =
sourceDir.openChecksumInput(sourceFileName,
DirectoryFactory.IOCONTEXT_NO_CACHE)) {
- if (input.length() <= CodecUtil.footerLength()) {
- throw new CorruptIndexException("file is too small:" +
input.length(), input);
- }
- if (input.length() > LARGE_BLOB_THRESHOLD_BYTE_SIZE) {
- writeBlobResumable(blobInfo, input);
- } else {
- writeBlobMultipart(blobInfo, input, (int) input.length());
- }
- }
+ }
+
+ @Override
+ public void copyIndexFileFrom(
+ Directory sourceDir, String sourceFileName, URI destDir, String
destFileName)
+ throws IOException {
+ String blobName = destDir.toString();
+ blobName = appendTrailingSeparatorIfNecessary(blobName);
+ blobName += destFileName;
+ final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName,
blobName).build();
+ try (ChecksumIndexInput input =
+ sourceDir.openChecksumInput(sourceFileName,
DirectoryFactory.IOCONTEXT_NO_CACHE)) {
+ if (input.length() <= CodecUtil.footerLength()) {
+ throw new CorruptIndexException("file is too small:" + input.length(),
input);
+ }
+ if (input.length() > LARGE_BLOB_THRESHOLD_BYTE_SIZE) {
+ writeBlobResumable(blobInfo, input);
+ } else {
+ writeBlobMultipart(blobInfo, input, (int) input.length());
+ }
}
-
- @Override
- public void copyIndexFileTo(URI sourceRepo, String sourceFileName,
Directory dest, String destFileName) throws IOException {
- try {
- String blobName = sourceRepo.toString();
- blobName = appendTrailingSeparatorIfNecessary(blobName);
- blobName += sourceFileName;
- final BlobId blobId = BlobId.of(bucketName, blobName);
- try (final ReadChannel readChannel = storage.reader(blobId);
- IndexOutput output = dest.createOutput(destFileName,
DirectoryFactory.IOCONTEXT_NO_CACHE)) {
- ByteBuffer buffer = ByteBuffer.allocate(readBufferSizeBytes);
- while (readChannel.read(buffer) > 0) {
- buffer.flip();
- byte[] arr = buffer.array();
- output.writeBytes(arr, buffer.position(), buffer.limit() -
buffer.position());
- buffer.clear();
- }
- }
- } catch (Exception e) {
- log.info("Here's an exception e", e);
+ }
+
+ @Override
+ public void copyIndexFileTo(
+ URI sourceRepo, String sourceFileName, Directory dest, String
destFileName)
+ throws IOException {
+ try {
+ String blobName = sourceRepo.toString();
+ blobName = appendTrailingSeparatorIfNecessary(blobName);
+ blobName += sourceFileName;
+ final BlobId blobId = BlobId.of(bucketName, blobName);
+ try (final ReadChannel readChannel = storage.reader(blobId);
+ IndexOutput output =
+ dest.createOutput(destFileName,
DirectoryFactory.IOCONTEXT_NO_CACHE)) {
+ ByteBuffer buffer = ByteBuffer.allocate(readBufferSizeBytes);
+ while (readChannel.read(buffer) > 0) {
+ buffer.flip();
+ byte[] arr = buffer.array();
+ output.writeBytes(arr, buffer.position(), buffer.limit() -
buffer.position());
+ buffer.clear();
}
+ }
+ } catch (Exception e) {
+ log.info("Here's an exception e", e);
}
-
-
- @Override
- public void close() throws IOException {
-
- }
-
- private void writeBlobMultipart(BlobInfo blobInfo, ChecksumIndexInput
indexInput, int blobSize)
- throws IOException {
- byte[] bytes = new byte[blobSize];
- indexInput.readBytes(bytes, 0, blobSize - CodecUtil.footerLength());
- long checksum = CodecUtil.checkFooter(indexInput);
- ByteBuffer footerBuffer = ByteBuffer.wrap(bytes, blobSize -
CodecUtil.footerLength(), CodecUtil.footerLength());
- writeFooter(checksum, footerBuffer);
- try {
- storage.create(blobInfo, bytes,
Storage.BlobTargetOption.doesNotExist());
- } catch (final StorageException se) {
- if (se.getCode() == HTTP_PRECON_FAILED) {
- throw new
FileAlreadyExistsException(blobInfo.getBlobId().getName(), null,
se.getMessage());
- }
- throw se;
- }
- }
-
- private void writeBlobResumable(BlobInfo blobInfo, ChecksumIndexInput
indexInput) throws IOException {
- try {
- final WriteChannel writeChannel = storage.writer(blobInfo,
getDefaultBlobWriteOptions());
-
- ByteBuffer buffer = ByteBuffer.allocate(writeBufferSizeBytes);
- writeChannel.setChunkSize(writeBufferSizeBytes);
-
- long remain = indexInput.length() - CodecUtil.footerLength();
- while (remain > 0) {
- // reading
- int byteReads = (int) Math.min(buffer.capacity(), remain);
- indexInput.readBytes(buffer.array(), 0, byteReads);
- buffer.position(byteReads);
- buffer.flip();
-
- // writing
- writeChannel.write(buffer);
- buffer.clear();
- remain -= byteReads;
- }
- long checksum = CodecUtil.checkFooter(indexInput);
- ByteBuffer bytes = getFooter(checksum);
- writeChannel.write(bytes);
- writeChannel.close();
- } catch (final StorageException se) {
- if (se.getCode() == HTTP_PRECON_FAILED) {
- throw new
FileAlreadyExistsException(blobInfo.getBlobId().getName(), null,
se.getMessage());
- }
- throw se;
- }
+ }
+
+ @Override
+ public void close() throws IOException {}
+
+ private void writeBlobMultipart(BlobInfo blobInfo, ChecksumIndexInput
indexInput, int blobSize)
+ throws IOException {
+ byte[] bytes = new byte[blobSize];
+ indexInput.readBytes(bytes, 0, blobSize - CodecUtil.footerLength());
+ long checksum = CodecUtil.checkFooter(indexInput);
+ ByteBuffer footerBuffer =
+ ByteBuffer.wrap(bytes, blobSize - CodecUtil.footerLength(),
CodecUtil.footerLength());
+ writeFooter(checksum, footerBuffer);
+ try {
+ storage.create(blobInfo, bytes, Storage.BlobTargetOption.doesNotExist());
+ } catch (final StorageException se) {
+ if (se.getCode() == HTTP_PRECON_FAILED) {
+ throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(),
null, se.getMessage());
+ }
+ throw se;
}
+ }
+
+ private void writeBlobResumable(BlobInfo blobInfo, ChecksumIndexInput
indexInput)
+ throws IOException {
+ try {
+ final WriteChannel writeChannel = storage.writer(blobInfo,
getDefaultBlobWriteOptions());
+
+ ByteBuffer buffer = ByteBuffer.allocate(writeBufferSizeBytes);
+ writeChannel.setChunkSize(writeBufferSizeBytes);
+
+ long remain = indexInput.length() - CodecUtil.footerLength();
+ while (remain > 0) {
+ // reading
+ int byteReads = (int) Math.min(buffer.capacity(), remain);
+ indexInput.readBytes(buffer.array(), 0, byteReads);
+ buffer.position(byteReads);
+ buffer.flip();
- private ByteBuffer getFooter(long checksum) throws IOException {
- ByteBuffer buffer = ByteBuffer.allocate(CodecUtil.footerLength());
- writeFooter(checksum, buffer);
- return buffer;
+ // writing
+ writeChannel.write(buffer);
+ buffer.clear();
+ remain -= byteReads;
+ }
+ long checksum = CodecUtil.checkFooter(indexInput);
+ ByteBuffer bytes = getFooter(checksum);
+ writeChannel.write(bytes);
+ writeChannel.close();
+ } catch (final StorageException se) {
+ if (se.getCode() == HTTP_PRECON_FAILED) {
+ throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(),
null, se.getMessage());
+ }
+ throw se;
}
-
- private void writeFooter(long checksum, ByteBuffer buffer) throws
IOException {
- IndexOutput out = new IndexOutput("", "") {
-
- @Override
- public void writeByte(byte b) throws IOException {
- buffer.put(b);
- }
-
- @Override
- public void writeBytes(byte[] b, int offset, int length) throws
IOException {
- buffer.put(b, offset, length);
- }
-
- @Override
- public void close() throws IOException {
-
- }
-
- @Override
- public long getFilePointer() {
- return 0;
- }
-
- @Override
- public long getChecksum() throws IOException {
- return checksum;
- }
+ }
+
+ private ByteBuffer getFooter(long checksum) throws IOException {
+ ByteBuffer buffer = ByteBuffer.allocate(CodecUtil.footerLength());
+ writeFooter(checksum, buffer);
+ return buffer;
+ }
+
+ private void writeFooter(long checksum, ByteBuffer buffer) throws
IOException {
+ IndexOutput out =
+ new IndexOutput("", "") {
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ buffer.put(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws
IOException {
+ buffer.put(b, offset, length);
+ }
+
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public long getFilePointer() {
+ return 0;
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return checksum;
+ }
};
- CodecUtil.writeFooter(out);
- buffer.flip();
- }
+ CodecUtil.writeFooter(out);
+ buffer.flip();
+ }
- protected Storage.BlobWriteOption[] getDefaultBlobWriteOptions() {
- return NO_WRITE_OPTIONS;
- }
+ protected Storage.BlobWriteOption[] getDefaultBlobWriteOptions() {
+ return NO_WRITE_OPTIONS;
+ }
- private String appendTrailingSeparatorIfNecessary(String blobName) {
- if (! blobName.endsWith("/")) {
- return blobName + "/";
- }
- return blobName;
+ private String appendTrailingSeparatorIfNecessary(String blobName) {
+ if (!blobName.endsWith("/")) {
+ return blobName + "/";
}
+ return blobName;
+ }
}
diff --git
a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java
b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java
index ad7f3de..9003808 100644
---
a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java
+++
b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java
@@ -19,14 +19,11 @@ package org.apache.solr.gcs;
import com.google.api.gax.retrying.RetrySettings;
import com.google.cloud.storage.StorageOptions;
+import java.util.Map;
import org.apache.solr.common.util.NamedList;
import org.threeten.bp.Duration;
-import java.util.Map;
-
-/**
- * Parses configuration for {@link GCSBackupRepository} from NamedList and
environment variables
- */
+/** Parses configuration for {@link GCSBackupRepository} from NamedList and
environment variables */
public class GCSConfigParser {
protected static final String GCS_BUCKET_ENV_VAR_NAME = "GCS_BUCKET";
protected static final String GCS_CREDENTIAL_ENV_VAR_NAME =
"GCS_CREDENTIAL_PATH";
@@ -35,15 +32,19 @@ public class GCSConfigParser {
private static final String GCS_CREDENTIAL_PARAM_NAME = "gcsCredentialPath";
private static final String GCS_WRITE_BUFFER_SIZE_PARAM_NAME =
"gcsWriteBufferSizeBytes";
private static final String GCS_READ_BUFFER_SIZE_PARAM_NAME =
"gcsReadBufferSizeBytes";
- private static final String HTTP_CONNECT_TIMEOUT_MILLIS_NAME =
"gcsClientHttpConnectTimeoutMillis";
+ private static final String HTTP_CONNECT_TIMEOUT_MILLIS_NAME =
+ "gcsClientHttpConnectTimeoutMillis";
private static final String HTTP_READ_TIMEOUT_MILLIS_NAME =
"gcsClientHttpReadTimeoutMillis";
private static final String MAX_REQUEST_RETRIES_NAME = "gcsClientMaxRetries";
private static final String TOTAL_TIMEOUT_MILLIS_NAME =
"gcsClientMaxRequestTimeoutMillis";
- private static final String HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME =
"gcsClientHttpInitialRetryDelayMillis";
- private static final String HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME =
"gcsClientHttpRetryDelayMultiplier";
+ private static final String HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME =
+ "gcsClientHttpInitialRetryDelayMillis";
+ private static final String HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME =
+ "gcsClientHttpRetryDelayMultiplier";
private static final String HTTP_MAX_RETRY_DELAY_MILLIS_NAME =
"gcsClientHttpMaxRetryDelayMillis";
private static final String RPC_INITIAL_TIMEOUT_MILLIS_NAME =
"gcsClientRpcInitialTimeoutMillis";
- private static final String RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME =
"gcsClientRpcTimeoutMultiplier";
+ private static final String RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME =
+ "gcsClientRpcTimeoutMultiplier";
private static final String RPC_MAX_TIMEOUT_MILLIS_NAME =
"gcsClientRpcMaxTimeoutMillis";
private static final String DEFAULT_GCS_BUCKET_VALUE = "solrBackupsBucket";
@@ -67,10 +68,19 @@ public class GCSConfigParser {
public GCSConfig parseConfiguration(NamedList<?> repoConfig, Map<String,
String> envVars) {
final String bucketName = parseBucket(repoConfig, envVars);
final String credentialPathStr = parseCredentialPath(repoConfig, envVars);
- final int writeBufferSizeBytes = getIntOrDefault(repoConfig,
GCS_WRITE_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_WRITE_BUFFER_SIZE_VALUE);
- final int readBufferSizeBytes = getIntOrDefault(repoConfig,
GCS_READ_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_READ_BUFFER_SIZE_VALUE);
+ final int writeBufferSizeBytes =
+ getIntOrDefault(
+ repoConfig, GCS_WRITE_BUFFER_SIZE_PARAM_NAME,
DEFAULT_GCS_WRITE_BUFFER_SIZE_VALUE);
+ final int readBufferSizeBytes =
+ getIntOrDefault(
+ repoConfig, GCS_READ_BUFFER_SIZE_PARAM_NAME,
DEFAULT_GCS_READ_BUFFER_SIZE_VALUE);
final StorageOptions.Builder storageOptionsBuilder =
parseStorageOptions(repoConfig);
- return new GCSConfig(bucketName, credentialPathStr, writeBufferSizeBytes,
readBufferSizeBytes, storageOptionsBuilder);
+ return new GCSConfig(
+ bucketName,
+ credentialPathStr,
+ writeBufferSizeBytes,
+ readBufferSizeBytes,
+ storageOptionsBuilder);
}
/*
@@ -94,12 +104,15 @@ public class GCSConfigParser {
}
public static String potentiallyMissingCredentialMsg() {
- return "No explicit credential path was provided for this
GCSBackupRepository. If Solr is running inside GCP, this " +
- "may be expected if GCP's \"Workload Identity\" or a related
feature is configured. Solr instances " +
- "running outside of GCP however must provide a valid credential
path in order to access GCS. These users " +
- "should specify their credential path by adding a '" +
GCS_CREDENTIAL_PARAM_NAME + "' property to the " +
- "repository definition in solconfig, or by setting the path value
in an env-var named '" +
- GCS_CREDENTIAL_ENV_VAR_NAME + "'.";
+ return "No explicit credential path was provided for this
GCSBackupRepository. If Solr is running inside GCP, this "
+ + "may be expected if GCP's \"Workload Identity\" or a related feature
is configured. Solr instances "
+ + "running outside of GCP however must provide a valid credential path
in order to access GCS. These users "
+ + "should specify their credential path by adding a '"
+ + GCS_CREDENTIAL_PARAM_NAME
+ + "' property to the "
+ + "repository definition in solconfig, or by setting the path value in
an env-var named '"
+ + GCS_CREDENTIAL_ENV_VAR_NAME
+ + "'.";
}
private int getIntOrDefault(NamedList<?> config, String propName, int
defaultValue) {
@@ -118,27 +131,64 @@ public class GCSConfigParser {
private StorageOptions.Builder parseStorageOptions(NamedList<?> repoConfig) {
final StorageOptions.Builder builder = StorageOptions.newBuilder();
-
builder.setTransportOptions(StorageOptions.getDefaultHttpTransportOptions().toBuilder()
- .setConnectTimeout(getIntOrDefault(repoConfig,
HTTP_CONNECT_TIMEOUT_MILLIS_NAME, DEFAULT_HTTP_CONNECT_TIMEOUT_MILLIS))
- .setReadTimeout(getIntOrDefault(repoConfig,
HTTP_READ_TIMEOUT_MILLIS_NAME, DEFAULT_HTTP_READ_TIMEOUT_MILLIS))
+ builder.setTransportOptions(
+ StorageOptions.getDefaultHttpTransportOptions().toBuilder()
+ .setConnectTimeout(
+ getIntOrDefault(
+ repoConfig,
+ HTTP_CONNECT_TIMEOUT_MILLIS_NAME,
+ DEFAULT_HTTP_CONNECT_TIMEOUT_MILLIS))
+ .setReadTimeout(
+ getIntOrDefault(
+ repoConfig, HTTP_READ_TIMEOUT_MILLIS_NAME,
DEFAULT_HTTP_READ_TIMEOUT_MILLIS))
+ .build());
+ builder.setRetrySettings(
+ RetrySettings.newBuilder()
+ // All retries
+ .setMaxAttempts(
+ getIntOrDefault(repoConfig, MAX_REQUEST_RETRIES_NAME,
DEFAULT_MAX_RETRIES))
+ .setTotalTimeout(
+ Duration.ofMillis(
+ getIntOrDefault(
+ repoConfig, TOTAL_TIMEOUT_MILLIS_NAME,
DEFAULT_TOTAL_TIMEOUT_MILLIS)))
+ // http requests
+ .setInitialRetryDelay(
+ Duration.ofMillis(
+ getIntOrDefault(
+ repoConfig,
+ HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME,
+ DEFAULT_HTTP_INITIAL_RETRY_DELAY_MILLIS)))
+ .setMaxRetryDelay(
+ Duration.ofMillis(
+ getIntOrDefault(
+ repoConfig,
+ HTTP_MAX_RETRY_DELAY_MILLIS_NAME,
+ DEFAULT_HTTP_MAX_RETRY_DELAY_MILLIS)))
+ .setRetryDelayMultiplier(
+ getDoubleOrDefault(
+ repoConfig,
+ HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME,
+ DEFAULT_HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER))
+ // rpc requests
+ .setInitialRpcTimeout(
+ Duration.ofMillis(
+ getIntOrDefault(
+ repoConfig,
+ RPC_INITIAL_TIMEOUT_MILLIS_NAME,
+ DEFAULT_RPC_INITIAL_TIMEOUT_MILLIS)))
+ .setMaxRpcTimeout(
+ Duration.ofMillis(
+ getIntOrDefault(
+ repoConfig, RPC_MAX_TIMEOUT_MILLIS_NAME,
DEFAULT_RPC_MAX_TIMEOUT_MILLIS)))
+ .setRpcTimeoutMultiplier(
+ getDoubleOrDefault(
+ repoConfig,
+ RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME,
+ DEFAULT_RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER))
.build());
- builder.setRetrySettings(RetrySettings.newBuilder()
- // All retries
- .setMaxAttempts(getIntOrDefault(repoConfig,
MAX_REQUEST_RETRIES_NAME, DEFAULT_MAX_RETRIES))
-
.setTotalTimeout(Duration.ofMillis(getIntOrDefault(repoConfig,
TOTAL_TIMEOUT_MILLIS_NAME, DEFAULT_TOTAL_TIMEOUT_MILLIS)))
- //http requests
-
.setInitialRetryDelay(Duration.ofMillis(getIntOrDefault(repoConfig,
HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME, DEFAULT_HTTP_INITIAL_RETRY_DELAY_MILLIS)))
-
.setMaxRetryDelay(Duration.ofMillis(getIntOrDefault(repoConfig,
HTTP_MAX_RETRY_DELAY_MILLIS_NAME, DEFAULT_HTTP_MAX_RETRY_DELAY_MILLIS)))
- .setRetryDelayMultiplier(getDoubleOrDefault(repoConfig,
HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME,
DEFAULT_HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER))
- //rpc requests
-
.setInitialRpcTimeout(Duration.ofMillis(getIntOrDefault(repoConfig,
RPC_INITIAL_TIMEOUT_MILLIS_NAME, DEFAULT_RPC_INITIAL_TIMEOUT_MILLIS)))
-
.setMaxRpcTimeout(Duration.ofMillis(getIntOrDefault(repoConfig,
RPC_MAX_TIMEOUT_MILLIS_NAME, DEFAULT_RPC_MAX_TIMEOUT_MILLIS)))
- .setRpcTimeoutMultiplier(getDoubleOrDefault(repoConfig,
RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME,
DEFAULT_RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER))
- .build());
return builder;
}
-
public static class GCSConfig {
private final StorageOptions.Builder optionsBuilder;
private final String bucketName;
@@ -146,7 +196,12 @@ public class GCSConfigParser {
private final int writeBufferSizeBytes;
private final int readBufferSizeBytes;
- public GCSConfig(String bucketName, String gcsCredentialPath, int
writeBufferSizeBytes, int readBufferSizeBytes, StorageOptions.Builder
optionsBuilder) {
+ public GCSConfig(
+ String bucketName,
+ String gcsCredentialPath,
+ int writeBufferSizeBytes,
+ int readBufferSizeBytes,
+ StorageOptions.Builder optionsBuilder) {
this.bucketName = bucketName;
this.gcsCredentialPath = gcsCredentialPath;
this.writeBufferSizeBytes = writeBufferSizeBytes;
diff --git
a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java
b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java
index e209187..c30db9c 100644
--- a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java
+++ b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java
@@ -15,7 +15,5 @@
* limitations under the License.
*/
-/**
- * GCS Backup Repository for Solr
- */
-package org.apache.solr.gcs;
\ No newline at end of file
+/** GCS Backup Repository for Solr */
+package org.apache.solr.gcs;
diff --git
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java
index a1074f5..d6f5d22 100644
---
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java
+++
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java
@@ -34,7 +34,6 @@ import com.google.cloud.storage.ServiceAccount;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageBatch;
import com.google.cloud.storage.StorageOptions;
-
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
@@ -44,13 +43,15 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
/**
- * Provides a thread-safe wrapper around any {@link Storage} implementations
which may not offer thread safety.
+ * Provides a thread-safe wrapper around any {@link Storage} implementations
which may not offer
+ * thread safety.
*
- * The {@link Storage} implementation provided by google-cloud-nio for
in-memory testing is thread-unsafe. This causes
- * a problem for backup-restore tests, which have threads reading and writing
cloud-store data from the test threads,
- * overseer nodes, as well as any nodes hosting individual cores. This class
takes a heavyhanded approach to solving
- * this by adding synchronization across all instance methods. This allows
google-cloud-nio's in-memory fake to be used
- * from multiple threads simultaneously as our tests require.
+ * <p>The {@link Storage} implementation provided by google-cloud-nio for
in-memory testing is
+ * thread-unsafe. This causes a problem for backup-restore tests, which have
threads reading and
+ * writing cloud-store data from the test threads, overseer nodes, as well as
any nodes hosting
+ * individual cores. This class takes a heavyhanded approach to solving this
by adding
+ * synchronization across all instance methods. This allows google-cloud-nio's
in-memory fake to be
+ * used from multiple threads simultaneously as our tests require.
*/
public class ConcurrentDelegatingStorage implements Storage {
@@ -61,210 +62,375 @@ public class ConcurrentDelegatingStorage implements
Storage {
}
@Override
- public synchronized Bucket create(BucketInfo bucketInfo,
BucketTargetOption... options) { return delegate.create(bucketInfo, options); }
+ public synchronized Bucket create(BucketInfo bucketInfo,
BucketTargetOption... options) {
+ return delegate.create(bucketInfo, options);
+ }
@Override
- public synchronized Blob create(BlobInfo blobInfo, BlobTargetOption...
options) { return delegate.create(blobInfo, options); }
+ public synchronized Blob create(BlobInfo blobInfo, BlobTargetOption...
options) {
+ return delegate.create(blobInfo, options);
+ }
@Override
- public synchronized Blob create(BlobInfo blobInfo, byte[] content,
BlobTargetOption... options) { return delegate.create(blobInfo, content,
options); }
+ public synchronized Blob create(BlobInfo blobInfo, byte[] content,
BlobTargetOption... options) {
+ return delegate.create(blobInfo, content, options);
+ }
@Override
- public synchronized Blob create(BlobInfo blobInfo, byte[] content, int
offset, int length, BlobTargetOption... options) { return
delegate.create(blobInfo, content, offset, length, options); }
+ public synchronized Blob create(
+ BlobInfo blobInfo, byte[] content, int offset, int length,
BlobTargetOption... options) {
+ return delegate.create(blobInfo, content, offset, length, options);
+ }
@Override
- public synchronized Blob create(BlobInfo blobInfo, InputStream content,
BlobWriteOption... options) { return delegate.create(blobInfo, content,
options); }
+ public synchronized Blob create(
+ BlobInfo blobInfo, InputStream content, BlobWriteOption... options) {
+ return delegate.create(blobInfo, content, options);
+ }
@Override
- public synchronized Blob createFrom(BlobInfo blobInfo, Path path,
BlobWriteOption... options) throws IOException { return
delegate.createFrom(blobInfo, path, options); }
+ public synchronized Blob createFrom(BlobInfo blobInfo, Path path,
BlobWriteOption... options)
+ throws IOException {
+ return delegate.createFrom(blobInfo, path, options);
+ }
@Override
- public synchronized Blob createFrom(BlobInfo blobInfo, Path path, int
bufferSize, BlobWriteOption... options) throws IOException { return
delegate.createFrom(blobInfo, path, bufferSize, options); }
+ public synchronized Blob createFrom(
+ BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption...
options) throws IOException {
+ return delegate.createFrom(blobInfo, path, bufferSize, options);
+ }
@Override
- public synchronized Blob createFrom(BlobInfo blobInfo, InputStream content,
BlobWriteOption... options) throws IOException { return
delegate.createFrom(blobInfo, content, options); }
+ public synchronized Blob createFrom(
+ BlobInfo blobInfo, InputStream content, BlobWriteOption... options)
throws IOException {
+ return delegate.createFrom(blobInfo, content, options);
+ }
@Override
- public synchronized Blob createFrom(BlobInfo blobInfo, InputStream content,
int bufferSize, BlobWriteOption... options) throws IOException { return
delegate.createFrom(blobInfo, content, bufferSize, options); }
+ public synchronized Blob createFrom(
+ BlobInfo blobInfo, InputStream content, int bufferSize,
BlobWriteOption... options)
+ throws IOException {
+ return delegate.createFrom(blobInfo, content, bufferSize, options);
+ }
@Override
- public synchronized Bucket get(String bucket, BucketGetOption... options) {
return delegate.get(bucket, options); }
+ public synchronized Bucket get(String bucket, BucketGetOption... options) {
+ return delegate.get(bucket, options);
+ }
@Override
- public synchronized Bucket lockRetentionPolicy(BucketInfo bucket,
BucketTargetOption... options) { return delegate.lockRetentionPolicy(bucket,
options); }
+ public synchronized Bucket lockRetentionPolicy(BucketInfo bucket,
BucketTargetOption... options) {
+ return delegate.lockRetentionPolicy(bucket, options);
+ }
@Override
- public synchronized Blob get(String bucket, String blob, BlobGetOption...
options) { return delegate.get(bucket, blob, options); }
+ public synchronized Blob get(String bucket, String blob, BlobGetOption...
options) {
+ return delegate.get(bucket, blob, options);
+ }
@Override
- public synchronized Blob get(BlobId blob, BlobGetOption... options) { return
delegate.get(blob, options); }
+ public synchronized Blob get(BlobId blob, BlobGetOption... options) {
+ return delegate.get(blob, options);
+ }
@Override
- public synchronized Blob get(BlobId blob) { return delegate.get(blob); }
+ public synchronized Blob get(BlobId blob) {
+ return delegate.get(blob);
+ }
@Override
- public synchronized Page<Bucket> list(BucketListOption... options) { return
delegate.list(options); }
+ public synchronized Page<Bucket> list(BucketListOption... options) {
+ return delegate.list(options);
+ }
@Override
- public synchronized Page<Blob> list(String bucket, BlobListOption...
options) { return delegate.list(bucket, options); }
+ public synchronized Page<Blob> list(String bucket, BlobListOption...
options) {
+ return delegate.list(bucket, options);
+ }
@Override
- public synchronized Bucket update(BucketInfo bucketInfo,
BucketTargetOption... options) { return delegate.update(bucketInfo, options); }
+ public synchronized Bucket update(BucketInfo bucketInfo,
BucketTargetOption... options) {
+ return delegate.update(bucketInfo, options);
+ }
@Override
- public synchronized Blob update(BlobInfo blobInfo, BlobTargetOption...
options) { return delegate.update(blobInfo, options); }
+ public synchronized Blob update(BlobInfo blobInfo, BlobTargetOption...
options) {
+ return delegate.update(blobInfo, options);
+ }
@Override
- public synchronized Blob update(BlobInfo blobInfo) { return
delegate.update(blobInfo); }
+ public synchronized Blob update(BlobInfo blobInfo) {
+ return delegate.update(blobInfo);
+ }
@Override
- public synchronized boolean delete(String bucket, BucketSourceOption...
options) { return delegate.delete(bucket, options); }
+ public synchronized boolean delete(String bucket, BucketSourceOption...
options) {
+ return delegate.delete(bucket, options);
+ }
@Override
- public synchronized boolean delete(String bucket, String blob,
BlobSourceOption... options) { return delegate.delete(bucket, blob, options); }
+ public synchronized boolean delete(String bucket, String blob,
BlobSourceOption... options) {
+ return delegate.delete(bucket, blob, options);
+ }
@Override
- public synchronized boolean delete(BlobId blob, BlobSourceOption... options)
{ return delegate.delete(blob, options); }
+ public synchronized boolean delete(BlobId blob, BlobSourceOption... options)
{
+ return delegate.delete(blob, options);
+ }
@Override
- public synchronized boolean delete(BlobId blob) { return
delegate.delete(blob); }
+ public synchronized boolean delete(BlobId blob) {
+ return delegate.delete(blob);
+ }
@Override
- public synchronized Blob compose(ComposeRequest composeRequest) { return
delegate.compose(composeRequest); }
+ public synchronized Blob compose(ComposeRequest composeRequest) {
+ return delegate.compose(composeRequest);
+ }
@Override
- public synchronized CopyWriter copy(CopyRequest copyRequest) { return
delegate.copy(copyRequest); }
+ public synchronized CopyWriter copy(CopyRequest copyRequest) {
+ return delegate.copy(copyRequest);
+ }
@Override
- public synchronized byte[] readAllBytes(String bucket, String blob,
BlobSourceOption... options) { return delegate.readAllBytes(bucket, blob,
options); }
+ public synchronized byte[] readAllBytes(String bucket, String blob,
BlobSourceOption... options) {
+ return delegate.readAllBytes(bucket, blob, options);
+ }
@Override
- public synchronized byte[] readAllBytes(BlobId blob, BlobSourceOption...
options) { return delegate.readAllBytes(blob, options); }
+ public synchronized byte[] readAllBytes(BlobId blob, BlobSourceOption...
options) {
+ return delegate.readAllBytes(blob, options);
+ }
@Override
- public synchronized StorageBatch batch() { return delegate.batch(); }
+ public synchronized StorageBatch batch() {
+ return delegate.batch();
+ }
@Override
- public synchronized ReadChannel reader(String bucket, String blob,
BlobSourceOption... options) { return new ConcurrentReadChannel(this,
delegate.reader(bucket, blob, options)); }
+ public synchronized ReadChannel reader(String bucket, String blob,
BlobSourceOption... options) {
+ return new ConcurrentReadChannel(this, delegate.reader(bucket, blob,
options));
+ }
@Override
- public synchronized ReadChannel reader(BlobId blob, BlobSourceOption...
options) { return new ConcurrentReadChannel(this, delegate.reader(blob,
options)); }
+ public synchronized ReadChannel reader(BlobId blob, BlobSourceOption...
options) {
+ return new ConcurrentReadChannel(this, delegate.reader(blob, options));
+ }
@Override
- public synchronized WriteChannel writer(BlobInfo blobInfo,
BlobWriteOption... options) { return new ConcurrentWriteChannel(this,
delegate.writer(blobInfo, options)); }
+ public synchronized WriteChannel writer(BlobInfo blobInfo,
BlobWriteOption... options) {
+ return new ConcurrentWriteChannel(this, delegate.writer(blobInfo,
options));
+ }
@Override
- public synchronized WriteChannel writer(URL signedURL) { return new
ConcurrentWriteChannel(this, delegate.writer(signedURL)); }
+ public synchronized WriteChannel writer(URL signedURL) {
+ return new ConcurrentWriteChannel(this, delegate.writer(signedURL));
+ }
@Override
- public synchronized URL signUrl(BlobInfo blobInfo, long duration, TimeUnit
unit, SignUrlOption... options) { return delegate.signUrl(blobInfo, duration,
unit, options); }
+ public synchronized URL signUrl(
+ BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOption...
options) {
+ return delegate.signUrl(blobInfo, duration, unit, options);
+ }
@Override
- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo
blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostFieldsV4 fields,
PostPolicyV4.PostConditionsV4 conditions, PostPolicyV4Option... options) {
return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, fields,
conditions, options); }
+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(
+ BlobInfo blobInfo,
+ long duration,
+ TimeUnit unit,
+ PostPolicyV4.PostFieldsV4 fields,
+ PostPolicyV4.PostConditionsV4 conditions,
+ PostPolicyV4Option... options) {
+ return delegate.generateSignedPostPolicyV4(
+ blobInfo, duration, unit, fields, conditions, options);
+ }
@Override
- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo
blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostFieldsV4 fields,
PostPolicyV4Option... options) { return
delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, fields, options);
}
+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(
+ BlobInfo blobInfo,
+ long duration,
+ TimeUnit unit,
+ PostPolicyV4.PostFieldsV4 fields,
+ PostPolicyV4Option... options) {
+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit,
fields, options);
+ }
@Override
- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo
blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostConditionsV4
conditions, PostPolicyV4Option... options) { return
delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, conditions,
options); }
+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(
+ BlobInfo blobInfo,
+ long duration,
+ TimeUnit unit,
+ PostPolicyV4.PostConditionsV4 conditions,
+ PostPolicyV4Option... options) {
+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit,
conditions, options);
+ }
@Override
- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo
blobInfo, long duration, TimeUnit unit, PostPolicyV4Option... options) { return
delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, options); }
+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(
+ BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4Option...
options) {
+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit,
options);
+ }
@Override
- public synchronized List<Blob> get(BlobId... blobIds) { return
delegate.get(blobIds); }
+ public synchronized List<Blob> get(BlobId... blobIds) {
+ return delegate.get(blobIds);
+ }
@Override
- public synchronized List<Blob> get(Iterable<BlobId> blobIds) { return
delegate.get(blobIds); }
+ public synchronized List<Blob> get(Iterable<BlobId> blobIds) {
+ return delegate.get(blobIds);
+ }
@Override
- public synchronized List<Blob> update(BlobInfo... blobInfos) { return
delegate.update(blobInfos); }
+ public synchronized List<Blob> update(BlobInfo... blobInfos) {
+ return delegate.update(blobInfos);
+ }
@Override
- public synchronized List<Blob> update(Iterable<BlobInfo> blobInfos) { return
delegate.update(blobInfos); }
+ public synchronized List<Blob> update(Iterable<BlobInfo> blobInfos) {
+ return delegate.update(blobInfos);
+ }
@Override
- public synchronized List<Boolean> delete(BlobId... blobIds) { return
delegate.delete(blobIds); }
+ public synchronized List<Boolean> delete(BlobId... blobIds) {
+ return delegate.delete(blobIds);
+ }
@Override
- public synchronized List<Boolean> delete(Iterable<BlobId> blobIds) { return
delegate.delete(blobIds); }
+ public synchronized List<Boolean> delete(Iterable<BlobId> blobIds) {
+ return delegate.delete(blobIds);
+ }
@Override
- public synchronized Acl getAcl(String bucket, Acl.Entity entity,
BucketSourceOption... options) { return delegate.getAcl(bucket, entity,
options); }
+ public synchronized Acl getAcl(String bucket, Acl.Entity entity,
BucketSourceOption... options) {
+ return delegate.getAcl(bucket, entity, options);
+ }
@Override
- public synchronized Acl getAcl(String bucket, Acl.Entity entity) { return
delegate.getAcl(bucket, entity); }
+ public synchronized Acl getAcl(String bucket, Acl.Entity entity) {
+ return delegate.getAcl(bucket, entity);
+ }
@Override
- public synchronized boolean deleteAcl(String bucket, Acl.Entity entity,
BucketSourceOption... options) { return delegate.deleteAcl(bucket, entity,
options); }
+ public synchronized boolean deleteAcl(
+ String bucket, Acl.Entity entity, BucketSourceOption... options) {
+ return delegate.deleteAcl(bucket, entity, options);
+ }
@Override
- public synchronized boolean deleteAcl(String bucket, Acl.Entity entity) {
return delegate.deleteAcl(bucket, entity); }
+ public synchronized boolean deleteAcl(String bucket, Acl.Entity entity) {
+ return delegate.deleteAcl(bucket, entity);
+ }
@Override
- public synchronized Acl createAcl(String bucket, Acl acl,
BucketSourceOption... options) { return delegate.createAcl(bucket, acl,
options); }
+ public synchronized Acl createAcl(String bucket, Acl acl,
BucketSourceOption... options) {
+ return delegate.createAcl(bucket, acl, options);
+ }
@Override
- public synchronized Acl createAcl(String bucket, Acl acl) { return
delegate.createAcl(bucket, acl); }
+ public synchronized Acl createAcl(String bucket, Acl acl) {
+ return delegate.createAcl(bucket, acl);
+ }
@Override
- public synchronized Acl updateAcl(String bucket, Acl acl,
BucketSourceOption... options) { return delegate.updateAcl(bucket, acl,
options); }
+ public synchronized Acl updateAcl(String bucket, Acl acl,
BucketSourceOption... options) {
+ return delegate.updateAcl(bucket, acl, options);
+ }
@Override
- public synchronized Acl updateAcl(String bucket, Acl acl) { return
delegate.updateAcl(bucket, acl); }
+ public synchronized Acl updateAcl(String bucket, Acl acl) {
+ return delegate.updateAcl(bucket, acl);
+ }
@Override
- public synchronized List<Acl> listAcls(String bucket, BucketSourceOption...
options) { return delegate.listAcls(bucket, options); }
+ public synchronized List<Acl> listAcls(String bucket, BucketSourceOption...
options) {
+ return delegate.listAcls(bucket, options);
+ }
@Override
- public synchronized List<Acl> listAcls(String bucket) { return
delegate.listAcls(bucket); }
+ public synchronized List<Acl> listAcls(String bucket) {
+ return delegate.listAcls(bucket);
+ }
@Override
- public synchronized Acl getDefaultAcl(String bucket, Acl.Entity entity) {
return delegate.getDefaultAcl(bucket, entity); }
+ public synchronized Acl getDefaultAcl(String bucket, Acl.Entity entity) {
+ return delegate.getDefaultAcl(bucket, entity);
+ }
@Override
- public synchronized boolean deleteDefaultAcl(String bucket, Acl.Entity
entity) { return delegate.deleteDefaultAcl(bucket, entity); }
+ public synchronized boolean deleteDefaultAcl(String bucket, Acl.Entity
entity) {
+ return delegate.deleteDefaultAcl(bucket, entity);
+ }
@Override
- public synchronized Acl createDefaultAcl(String bucket, Acl acl) { return
delegate.createDefaultAcl(bucket, acl); }
+ public synchronized Acl createDefaultAcl(String bucket, Acl acl) {
+ return delegate.createDefaultAcl(bucket, acl);
+ }
@Override
- public synchronized Acl updateDefaultAcl(String bucket, Acl acl) { return
delegate.updateDefaultAcl(bucket, acl); }
+ public synchronized Acl updateDefaultAcl(String bucket, Acl acl) {
+ return delegate.updateDefaultAcl(bucket, acl);
+ }
@Override
- public synchronized List<Acl> listDefaultAcls(String bucket) { return
delegate.listDefaultAcls(bucket); }
+ public synchronized List<Acl> listDefaultAcls(String bucket) {
+ return delegate.listDefaultAcls(bucket);
+ }
@Override
- public synchronized Acl getAcl(BlobId blob, Acl.Entity entity) { return
delegate.getAcl(blob, entity); }
+ public synchronized Acl getAcl(BlobId blob, Acl.Entity entity) {
+ return delegate.getAcl(blob, entity);
+ }
@Override
- public synchronized boolean deleteAcl(BlobId blob, Acl.Entity entity) {
return delegate.deleteAcl(blob, entity); }
+ public synchronized boolean deleteAcl(BlobId blob, Acl.Entity entity) {
+ return delegate.deleteAcl(blob, entity);
+ }
@Override
- public synchronized Acl createAcl(BlobId blob, Acl acl) { return
delegate.createAcl(blob, acl); }
+ public synchronized Acl createAcl(BlobId blob, Acl acl) {
+ return delegate.createAcl(blob, acl);
+ }
@Override
- public synchronized Acl updateAcl(BlobId blob, Acl acl) { return
delegate.updateAcl(blob, acl); }
+ public synchronized Acl updateAcl(BlobId blob, Acl acl) {
+ return delegate.updateAcl(blob, acl);
+ }
@Override
- public synchronized List<Acl> listAcls(BlobId blob) { return
delegate.listAcls(blob); }
+ public synchronized List<Acl> listAcls(BlobId blob) {
+ return delegate.listAcls(blob);
+ }
@Override
- public synchronized HmacKey createHmacKey(ServiceAccount serviceAccount,
CreateHmacKeyOption... options) { return delegate.createHmacKey(serviceAccount,
options); }
+ public synchronized HmacKey createHmacKey(
+ ServiceAccount serviceAccount, CreateHmacKeyOption... options) {
+ return delegate.createHmacKey(serviceAccount, options);
+ }
@Override
- public synchronized Page<HmacKey.HmacKeyMetadata>
listHmacKeys(ListHmacKeysOption... options) { return
delegate.listHmacKeys(options); }
+ public synchronized Page<HmacKey.HmacKeyMetadata>
listHmacKeys(ListHmacKeysOption... options) {
+ return delegate.listHmacKeys(options);
+ }
@Override
- public synchronized HmacKey.HmacKeyMetadata getHmacKey(String accessId,
GetHmacKeyOption... options) { return delegate.getHmacKey(accessId, options); }
+ public synchronized HmacKey.HmacKeyMetadata getHmacKey(
+ String accessId, GetHmacKeyOption... options) {
+ return delegate.getHmacKey(accessId, options);
+ }
@Override
- public synchronized void deleteHmacKey(HmacKey.HmacKeyMetadata
hmacKeyMetadata, DeleteHmacKeyOption... options) {
+ public synchronized void deleteHmacKey(
+ HmacKey.HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption... options)
{
delegate.deleteHmacKey(hmacKeyMetadata, options);
}
@Override
- public synchronized HmacKey.HmacKeyMetadata
updateHmacKeyState(HmacKey.HmacKeyMetadata hmacKeyMetadata,
HmacKey.HmacKeyState state, UpdateHmacKeyOption... options) {
+ public synchronized HmacKey.HmacKeyMetadata updateHmacKeyState(
+ HmacKey.HmacKeyMetadata hmacKeyMetadata,
+ HmacKey.HmacKeyState state,
+ UpdateHmacKeyOption... options) {
return delegate.updateHmacKeyState(hmacKeyMetadata, state, options);
}
@@ -274,12 +440,14 @@ public class ConcurrentDelegatingStorage implements
Storage {
}
@Override
- public synchronized Policy setIamPolicy(String bucket, Policy policy,
BucketSourceOption... options) {
+ public synchronized Policy setIamPolicy(
+ String bucket, Policy policy, BucketSourceOption... options) {
return delegate.setIamPolicy(bucket, policy, options);
}
@Override
- public synchronized List<Boolean> testIamPermissions(String bucket,
List<String> permissions, BucketSourceOption... options) {
+ public synchronized List<Boolean> testIamPermissions(
+ String bucket, List<String> permissions, BucketSourceOption... options) {
return delegate.testIamPermissions(bucket, permissions, options);
}
diff --git
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java
index 4676ef7..03aa0af 100644
---
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java
+++
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java
@@ -17,57 +17,53 @@
package org.apache.solr.gcs;
-import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.backup.repository.BackupRepository;
-import org.junit.AfterClass;
-import org.junit.Test;
+import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;
+import static org.apache.solr.gcs.GCSConfigParser.GCS_BUCKET_ENV_VAR_NAME;
+import static org.apache.solr.gcs.GCSConfigParser.GCS_CREDENTIAL_ENV_VAR_NAME;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.junit.AfterClass;
+import org.junit.Test;
-import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;
-import static org.apache.solr.gcs.GCSConfigParser.GCS_BUCKET_ENV_VAR_NAME;
-import static org.apache.solr.gcs.GCSConfigParser.GCS_CREDENTIAL_ENV_VAR_NAME;
-
-/**
- * Unit tests for {@link GCSBackupRepository} that use an in-memory Storage
object
- */
+/** Unit tests for {@link GCSBackupRepository} that use an in-memory Storage
object */
public class GCSBackupRepositoryTest extends AbstractBackupRepositoryTest {
- @AfterClass
- public static void tearDownClass() throws Exception {
- LocalStorageGCSBackupRepository.clearStashedStorage();
- }
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ LocalStorageGCSBackupRepository.clearStashedStorage();
+ }
- @Override
- protected BackupRepository getRepository() {
- final NamedList<Object> config = new NamedList<>();
- config.add(BACKUP_LOCATION, "backup1");
- final GCSBackupRepository repository = new
LocalStorageGCSBackupRepository();
- repository.init(config);
+ @Override
+ protected BackupRepository getRepository() {
+ final NamedList<Object> config = new NamedList<>();
+ config.add(BACKUP_LOCATION, "backup1");
+ final GCSBackupRepository repository = new
LocalStorageGCSBackupRepository();
+ repository.init(config);
- return repository;
- }
+ return repository;
+ }
- @Override
- protected URI getBaseUri() throws URISyntaxException {
- return new URI("tmp");
- }
+ @Override
+ protected URI getBaseUri() throws URISyntaxException {
+ return new URI("tmp");
+ }
- @Test
- public void testInitStoreDoesNotFailWithMissingCredentials()
- {
- Map<String, String> config = new HashMap<>();
- config.put(GCS_BUCKET_ENV_VAR_NAME, "a_bucket_name");
- // explicitly setting credential name to null; will work inside
google-cloud project
- config.put(GCS_CREDENTIAL_ENV_VAR_NAME, null);
- config.put(BACKUP_LOCATION, "/==");
+ @Test
+ public void testInitStoreDoesNotFailWithMissingCredentials() {
+ Map<String, String> config = new HashMap<>();
+ config.put(GCS_BUCKET_ENV_VAR_NAME, "a_bucket_name");
+ // explicitly setting credential name to null; will work inside
google-cloud project
+ config.put(GCS_CREDENTIAL_ENV_VAR_NAME, null);
+ config.put(BACKUP_LOCATION, "/==");
- BackupRepository gcsBackupRepository = getRepository();
+ BackupRepository gcsBackupRepository = getRepository();
- gcsBackupRepository.init(new NamedList<>(config));
- }
+ gcsBackupRepository.init(new NamedList<>(config));
+ }
}
diff --git
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java
index b567252..d72c0d4 100644
---
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java
+++
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java
@@ -18,6 +18,7 @@
package org.apache.solr.gcs;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import java.lang.invoke.MethodHandles;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest;
import org.junit.AfterClass;
@@ -25,71 +26,71 @@ import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.invoke.MethodHandles;
-
@ThreadLeakLingering(linger = 10)
[email protected]({"SimpleText"}) // Backups do checksum
validation against a footer value not present in 'SimpleText'
[email protected]({
+ "SimpleText"
+}) // Backups do checksum validation against a footer value not present in
'SimpleText'
public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest {
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String SOLR_XML = "<solr>\n" +
- "\n" +
- " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
- " <str
name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
- " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
+
- "\n" +
- " <shardHandlerFactory name=\"shardHandlerFactory\"
class=\"HttpShardHandlerFactory\">\n" +
- " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
- " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
- " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
- " </shardHandlerFactory>\n" +
- "\n" +
- " <solrcloud>\n" +
- " <str name=\"host\">127.0.0.1</str>\n" +
- " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
- " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
- " <int
name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
- " <bool
name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
- " <int name=\"leaderVoteWait\">10000</int>\n" +
- " <int
name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
- " <int
name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
- " </solrcloud>\n" +
- " \n" +
- " <backup>\n" +
- " <repository name=\"trackingBackupRepository\"
class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
- " <str name=\"delegateRepoName\">localfs</str>\n" +
- " </repository>\n" +
- " <repository name=\"localfs\"
class=\"org.apache.solr.gcs.LocalStorageGCSBackupRepository\"> \n" +
- " <str name=\"gcsBucket\">someBucketName</str>\n" +
- " <str name=\"location\">backup1</str>\n" +
- " </repository>\n" +
- " </backup>\n" +
- " \n" +
- "</solr>\n";
-
- private static String backupLocation;
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String SOLR_XML =
+ "<solr>\n"
+ + "\n"
+ + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"
+ + " <str
name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"
+ + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"
+ + "\n"
+ + " <shardHandlerFactory name=\"shardHandlerFactory\"
class=\"HttpShardHandlerFactory\">\n"
+ + " <str name=\"urlScheme\">${urlScheme:}</str>\n"
+ + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"
+ + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"
+ + " </shardHandlerFactory>\n"
+ + "\n"
+ + " <solrcloud>\n"
+ + " <str name=\"host\">127.0.0.1</str>\n"
+ + " <int name=\"hostPort\">${hostPort:8983}</int>\n"
+ + " <str name=\"hostContext\">${hostContext:solr}</str>\n"
+ + " <int
name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"
+ + " <bool
name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"
+ + " <int name=\"leaderVoteWait\">10000</int>\n"
+ + " <int
name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"
+ + " <int
name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"
+ + " </solrcloud>\n"
+ + " \n"
+ + " <backup>\n"
+ + " <repository name=\"trackingBackupRepository\"
class=\"org.apache.solr.core.TrackingBackupRepository\"> \n"
+ + " <str name=\"delegateRepoName\">localfs</str>\n"
+ + " </repository>\n"
+ + " <repository name=\"localfs\"
class=\"org.apache.solr.gcs.LocalStorageGCSBackupRepository\"> \n"
+ + " <str name=\"gcsBucket\">someBucketName</str>\n"
+ + " <str name=\"location\">backup1</str>\n"
+ + " </repository>\n"
+ + " </backup>\n"
+ + " \n"
+ + "</solr>\n";
- @BeforeClass
- public static void setupClass() throws Exception {
+ private static String backupLocation;
- configureCluster(NUM_SHARDS)// nodes
- .addConfig("conf1",
getFile("conf/solrconfig.xml").getParentFile().toPath())
- .withSolrXml(SOLR_XML)
- .configure();
- }
+ @BeforeClass
+ public static void setupClass() throws Exception {
- @AfterClass
- public static void tearDownClass() throws Exception {
- LocalStorageGCSBackupRepository.clearStashedStorage();
- }
+ configureCluster(NUM_SHARDS) // nodes
+ .addConfig("conf1",
getFile("conf/solrconfig.xml").getParentFile().toPath())
+ .withSolrXml(SOLR_XML)
+ .configure();
+ }
- @Override
- public String getCollectionNamePrefix() {
- return "backuprestore";
- }
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ LocalStorageGCSBackupRepository.clearStashedStorage();
+ }
- @Override
- public String getBackupLocation() {
- return "backup1";
- }
+ @Override
+ public String getCollectionNamePrefix() {
+ return "backuprestore";
+ }
+ @Override
+ public String getBackupLocation() {
+ return "backup1";
+ }
}
diff --git
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java
index 30d0df1..84460e5 100644
---
a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java
+++
b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java
@@ -17,12 +17,13 @@
package org.apache.solr.gcs;
+import static org.apache.lucene.util.LuceneTestCase.assumeFalse;
+
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.collect.Lists;
-
import java.io.IOException;
import java.net.URI;
import java.nio.file.NoSuchFileException;
@@ -30,16 +31,16 @@ import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
-import static org.apache.lucene.util.LuceneTestCase.assumeFalse;
-
public class LocalStorageGCSBackupRepository extends GCSBackupRepository {
protected static Storage stashedStorage = null;
@Override
public Storage initStorage() {
- // LocalStorageHelper produces 'Storage' objects that track blob store
state in non-static memory unique to each
- // Storage instance. For various components in Solr to have a coherent
view of the blob-space then, they need to
+ // LocalStorageHelper produces 'Storage' objects that track blob store
state in non-static
+ // memory unique to each
+ // Storage instance. For various components in Solr to have a coherent
view of the blob-space
+ // then, they need to
// share a single 'Storage' object.
synchronized (LocalStorageGCSBackupRepository.class) {
storage = getSingletonStorage();
@@ -54,10 +55,12 @@ public class LocalStorageGCSBackupRepository extends
GCSBackupRepository {
}
}
- // A reimplementation of delete functionality that avoids batching.
Batching is ideal in production use cases, but
+ // A reimplementation of delete functionality that avoids batching.
Batching is ideal in
+ // production use cases, but
// isn't supported by the in-memory Storage implementation provided by
LocalStorageHelper
@Override
- public void delete(URI path, Collection<String> files, boolean
ignoreNoSuchFileException) throws IOException {
+ public void delete(URI path, Collection<String> files, boolean
ignoreNoSuchFileException)
+ throws IOException {
final List<Boolean> results = Lists.newArrayList();
final List<String> filesOrdered =
files.stream().collect(Collectors.toList());
@@ -89,9 +92,12 @@ public class LocalStorageGCSBackupRepository extends
GCSBackupRepository {
return stashedStorage;
}
- // FakeStorageRpc isn't thread-safe, which causes flaky test failures when
multiple cores attempt to backup files
- // simultaneously. We work around this here by wrapping it in a
delegating instance that adds a measure of thread safety.
- stashedStorage = new
ConcurrentDelegatingStorage(LocalStorageHelper.customOptions(false).getService());
+ // FakeStorageRpc isn't thread-safe, which causes flaky test failures when
multiple cores
+ // attempt to backup files
+ // simultaneously. We work around this here by wrapping it in a
delegating instance that adds a
+ // measure of thread safety.
+ stashedStorage =
+ new
ConcurrentDelegatingStorage(LocalStorageHelper.customOptions(false).getService());
return stashedStorage;
}
@@ -103,10 +109,11 @@ public class LocalStorageGCSBackupRepository extends
GCSBackupRepository {
} catch (Exception e) {
final Throwable cause = e.getCause();
if (cause != null) {
- assumeFalse("This test uses a GCS mock library that is incompatible
with the current default locale",
- e instanceof StorageException &&
- cause.getMessage().contains("Invalid date/time
format") &&
- cause instanceof NumberFormatException);
+ assumeFalse(
+ "This test uses a GCS mock library that is incompatible with the
current default locale",
+ e instanceof StorageException
+ && cause.getMessage().contains("Invalid date/time format")
+ && cause instanceof NumberFormatException);
}
}
}
