>From Michael Blow <mb...@apache.org>:
Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19536 )
Change subject: [NO ISSUE][*DB][STO] Support a no-op cloud storage scheme (none)
......................................................................
[NO ISSUE][*DB][STO] Support a no-op cloud storage scheme (none)
When configuring a cloud.storage.scheme of none, do not actually
perform reads or writes to a cloud provider. All reads will be empty
and writes will not actually be performed. Additionally, do not
reconcile local cache with the cloud during recovery, since the
cloud does not have anything stored.
Ext-ref: MB-65877
Change-Id: Ib2911f35177d2acde3ce1e98f94e9a2384df16ab
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
A
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
8 files changed, 218 insertions(+), 29 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/36/19536/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index 95cdb1c..1aefadf 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -103,6 +103,7 @@
} catch (Exception ex) {
// do nothing
}
+ s3Mock = null;
}
}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index e4224f2..3e5622e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -87,6 +87,7 @@
protected final IOManager localIoManager;
protected final INamespacePathResolver nsPathResolver;
private final List<FileStore> drivePaths;
+ private final String storageScheme;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
@@ -102,6 +103,7 @@
partitionPaths = new ArrayList<>();
this.localIoManager = ioManager;
drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
+ storageScheme = cloudProperties.getStorageScheme();
}
/*
@@ -143,6 +145,11 @@
partitionPaths.add(resolve(STORAGE_ROOT_DIR_NAME + File.separator
+ partitionDir));
}
+ if (CloudClientProvider.NONE.equals(storageScheme)) {
+ LOGGER.info("Cloud storage scheme is '{}', nothing to reconcile /
download", storageScheme);
+ return;
+ }
+
LOGGER.info("Initializing cloud manager with ({}) storage partitions:
{}", partitions.size(), partitions);
if (!currentOnDiskPartitions.isEmpty()) {
deleteUnkeptPartitionDirs(currentOnDiskPartitions);
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
index 9b832a2..ab8c6c4 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
@@ -71,11 +71,6 @@
*/
@Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
- @Override
public int write(ByteBuffer page) throws HyracksDataException {
open();
return write(page.array(), page.position(), page.remaining());
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index c98c6b4..03a5b46 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -33,6 +33,7 @@
public static final String S3 = "s3";
public static final String GCS = "gs";
public static final String AZ_BLOB = "azblob";
+ public static final String NONE = "none";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
@@ -41,19 +42,13 @@
public static ICloudClient getClient(CloudProperties cloudProperties,
ICloudGuardian guardian)
throws HyracksDataException {
String storageScheme = cloudProperties.getStorageScheme();
- ICloudClient cloudClient;
- if (S3.equalsIgnoreCase(storageScheme)) {
- S3ClientConfig config = S3ClientConfig.of(cloudProperties);
- cloudClient = new S3CloudClient(config, guardian);
- } else if (GCS.equalsIgnoreCase(storageScheme)) {
- GCSClientConfig config = GCSClientConfig.of(cloudProperties);
- cloudClient = new GCSCloudClient(config, guardian);
- } else if (AZ_BLOB.equalsIgnoreCase(storageScheme)) {
- AzBlobStorageClientConfig config =
AzBlobStorageClientConfig.of(cloudProperties);
- cloudClient = new AzBlobStorageCloudClient(config, guardian);
- } else {
- throw new IllegalStateException("unsupported cloud storage scheme:
" + storageScheme);
- }
+ ICloudClient cloudClient = switch (storageScheme.toLowerCase()) {
+ case S3 -> new S3CloudClient(S3ClientConfig.of(cloudProperties),
guardian);
+ case GCS -> new
GCSCloudClient(GCSClientConfig.of(cloudProperties), guardian);
+ case AZ_BLOB -> new
AzBlobStorageCloudClient(AzBlobStorageClientConfig.of(cloudProperties),
guardian);
+ case NONE -> NoopCloudClient.INSTANCE;
+ default -> throw new IllegalStateException("unsupported cloud
storage scheme: " + storageScheme);
+ };
return UNSTABLE ? new UnstableCloudClient(cloudClient) : cloudClient;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
index 920be9c..5299b5d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
@@ -33,7 +33,9 @@
* @param page to write
* @return written bytes
*/
- int write(ByteBuffer header, ByteBuffer page) throws HyracksDataException;
+ default int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
+ return write(header) + write(page);
+ }
/**
* Write a page
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
new file mode 100644
index 0000000..fb83aa9
--- /dev/null
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.cloud.IWriteBufferProvider;
+import org.apache.asterix.cloud.clients.profiler.IRequestProfilerLimiter;
+import org.apache.asterix.cloud.clients.profiler.NoOpRequestProfilerLimiter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class NoopCloudClient implements ICloudClient {
+
+ public static final ICloudClient INSTANCE = new NoopCloudClient();
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private NoopCloudClient() {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // no-op
+ }
+
+ @Override
+ public int getWriteBufferSize() {
+ return 0;
+ }
+
+ @Override
+ public IRequestProfilerLimiter getProfilerLimiter() {
+ return NoOpRequestProfilerLimiter.INSTANCE;
+ }
+
+ @Override
+ public ICloudWriter createWriter(String bucket, String path,
IWriteBufferProvider bufferProvider) {
+ return new NoOpCloudWriter();
+ }
+
+ @Override
+ public Set<CloudFile> listObjects(String bucket, String path,
FilenameFilter filter) {
+ return Set.of();
+ }
+
+ @Override
+ public int read(String bucket, String path, long offset, ByteBuffer
buffer) throws HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public byte[] readAllBytes(String bucket, String path) throws
HyracksDataException {
+ return new byte[0];
+ }
+
+ @Override
+ public InputStream getObjectStream(String bucket, String path, long
offset, long length) {
+ return InputStream.nullInputStream();
+ }
+
+ @Override
+ public void write(String bucket, String path, byte[] data) {
+ }
+
+ @Override
+ public void copy(String bucket, String srcPath, FileReference destPath) {
+ }
+
+ @Override
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
+ }
+
+ @Override
+ public long getObjectSize(String bucket, String path) throws
HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public boolean exists(String bucket, String path) throws
HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean isEmptyPrefix(String bucket, String path) throws
HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public IParallelDownloader createParallelDownloader(String bucket,
IOManager ioManager)
+ throws HyracksDataException {
+ return NoOpParallelDownloader.INSTANCE;
+ }
+
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ return OBJECT_MAPPER.createArrayNode();
+ }
+
+ private static class NoOpCloudWriter implements ICloudWriter {
+
+ long position = 0L;
+
+ public NoOpCloudWriter() {
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ position = 0L;
+ }
+
+ @Override
+ public int write(ByteBuffer page) throws HyracksDataException {
+ int written = page.remaining();
+ position += written;
+ page.position(page.limit());
+ return written;
+ }
+
+ @Override
+ public void write(int b) throws HyracksDataException {
+ position++;
+ }
+
+ @Override
+ public int write(byte[] b, int off, int len) throws
HyracksDataException {
+ position += len;
+ return len;
+ }
+
+ @Override
+ public long position() {
+ return position;
+ }
+
+ @Override
+ public void finish() throws HyracksDataException {
+ position = 0;
+ }
+ }
+
+ private static class NoOpParallelDownloader implements IParallelDownloader
{
+ private static final NoOpParallelDownloader INSTANCE = new
NoOpParallelDownloader();
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void downloadFiles(Collection<FileReference> toDownload) throws
HyracksDataException {
+ }
+
+ @Override
+ public Collection<FileReference>
downloadDirectories(Collection<FileReference> toDownload)
+ throws HyracksDataException {
+ return List.of();
+ }
+ }
+}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
index 1b39251..cb24650 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
@@ -170,11 +170,6 @@
}
@Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
- @Override
public int write(ByteBuffer page) throws HyracksDataException {
if (position() == 0) {
fail();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index 3a83786..04b2de1 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
@@ -58,11 +58,6 @@
}
@Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
- @Override
public int write(ByteBuffer page) throws HyracksDataException {
guardian.checkIsolatedWriteAccess(bucket, path);
// The GCS library triggers a new upload when its internal buffer is
full, not on each call to writer.write().
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19536
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ib2911f35177d2acde3ce1e98f94e9a2384df16ab
Gerrit-Change-Number: 19536
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-MessageType: newchange
