This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4741bc8b07 [NO ISSUE][*DB][STO] Support a no-op cloud storage scheme
(none)
4741bc8b07 is described below
commit 4741bc8b070352a7d34874432ac7d54ea3bbf5f0
Author: Michael Blow <[email protected]>
AuthorDate: Thu Mar 20 11:57:28 2025 -0400
[NO ISSUE][*DB][STO] Support a no-op cloud storage scheme (none)
When configuring a cloud.storage.scheme of none, do not actually
perform reads or writes to a cloud provider. All reads will be empty
and writes will not actually be performed. Additionally, do not
reconcile local cache with the cloud during recovery, since the
cloud does not have anything stored.
Ext-ref: MB-65877
Change-Id: Ib2911f35177d2acde3ce1e98f94e9a2384df16ab
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19536
Integration-Tests: Jenkins <[email protected]>
Tested-by: Michael Blow <[email protected]>
Reviewed-by: Michael Blow <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
---
.../api/common/LocalCloudUtilAdobeMock.java | 1 +
.../asterix/cloud/AbstractCloudIOManager.java | 7 +
.../asterix/cloud/CloudResettableInputStream.java | 5 -
.../asterix/cloud/clients/CloudClientProvider.java | 21 +--
.../apache/asterix/cloud/clients/ICloudWriter.java | 4 +-
.../asterix/cloud/clients/NoopCloudClient.java | 183 +++++++++++++++++++++
.../asterix/cloud/clients/UnstableCloudClient.java | 5 -
.../cloud/clients/google/gcs/GCSWriter.java | 5 -
8 files changed, 202 insertions(+), 29 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index 95cdb1cfea..1aefadfc66 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -103,6 +103,7 @@ public class LocalCloudUtilAdobeMock {
} catch (Exception ex) {
// do nothing
}
+ s3Mock = null;
}
}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index e4224f2877..3e5622e7b1 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -87,6 +87,7 @@ public abstract class AbstractCloudIOManager extends
IOManager implements IParti
protected final IOManager localIoManager;
protected final INamespacePathResolver nsPathResolver;
private final List<FileStore> drivePaths;
+ private final String storageScheme;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
@@ -102,6 +103,7 @@ public abstract class AbstractCloudIOManager extends
IOManager implements IParti
partitionPaths = new ArrayList<>();
this.localIoManager = ioManager;
drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
+ storageScheme = cloudProperties.getStorageScheme();
}
/*
@@ -143,6 +145,11 @@ public abstract class AbstractCloudIOManager extends
IOManager implements IParti
partitionPaths.add(resolve(STORAGE_ROOT_DIR_NAME + File.separator
+ partitionDir));
}
+ if (CloudClientProvider.NONE.equals(storageScheme)) {
+ LOGGER.info("Cloud storage scheme is '{}', nothing to reconcile /
download", storageScheme);
+ return;
+ }
+
LOGGER.info("Initializing cloud manager with ({}) storage partitions:
{}", partitions.size(), partitions);
if (!currentOnDiskPartitions.isEmpty()) {
deleteUnkeptPartitionDirs(currentOnDiskPartitions);
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
index 9b832a2018..ab8c6c4671 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudResettableInputStream.java
@@ -70,11 +70,6 @@ public class CloudResettableInputStream extends InputStream
implements ICloudWri
* ************************************************************
*/
- @Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
@Override
public int write(ByteBuffer page) throws HyracksDataException {
open();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index c98c6b4ba7..03a5b4663b 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -33,6 +33,7 @@ public class CloudClientProvider {
public static final String S3 = "s3";
public static final String GCS = "gs";
public static final String AZ_BLOB = "azblob";
+ public static final String NONE = "none";
private CloudClientProvider() {
throw new AssertionError("do not instantiate");
@@ -41,19 +42,13 @@ public class CloudClientProvider {
public static ICloudClient getClient(CloudProperties cloudProperties,
ICloudGuardian guardian)
throws HyracksDataException {
String storageScheme = cloudProperties.getStorageScheme();
- ICloudClient cloudClient;
- if (S3.equalsIgnoreCase(storageScheme)) {
- S3ClientConfig config = S3ClientConfig.of(cloudProperties);
- cloudClient = new S3CloudClient(config, guardian);
- } else if (GCS.equalsIgnoreCase(storageScheme)) {
- GCSClientConfig config = GCSClientConfig.of(cloudProperties);
- cloudClient = new GCSCloudClient(config, guardian);
- } else if (AZ_BLOB.equalsIgnoreCase(storageScheme)) {
- AzBlobStorageClientConfig config =
AzBlobStorageClientConfig.of(cloudProperties);
- cloudClient = new AzBlobStorageCloudClient(config, guardian);
- } else {
- throw new IllegalStateException("unsupported cloud storage scheme:
" + storageScheme);
- }
+ ICloudClient cloudClient = switch (storageScheme.toLowerCase()) {
+ case S3 -> new S3CloudClient(S3ClientConfig.of(cloudProperties),
guardian);
+ case GCS -> new
GCSCloudClient(GCSClientConfig.of(cloudProperties), guardian);
+ case AZ_BLOB -> new
AzBlobStorageCloudClient(AzBlobStorageClientConfig.of(cloudProperties),
guardian);
+ case NONE -> NoopCloudClient.INSTANCE;
+ default -> throw new IllegalStateException("unsupported cloud
storage scheme: " + storageScheme);
+ };
return UNSTABLE ? new UnstableCloudClient(cloudClient) : cloudClient;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
index 920be9ce2e..5299b5d63d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudWriter.java
@@ -33,7 +33,9 @@ public interface ICloudWriter {
* @param page to write
* @return written bytes
*/
- int write(ByteBuffer header, ByteBuffer page) throws HyracksDataException;
+ default int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
+ return write(header) + write(page);
+ }
/**
* Write a page
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
new file mode 100644
index 0000000000..fb83aa98a4
--- /dev/null
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.cloud.IWriteBufferProvider;
+import org.apache.asterix.cloud.clients.profiler.IRequestProfilerLimiter;
+import org.apache.asterix.cloud.clients.profiler.NoOpRequestProfilerLimiter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class NoopCloudClient implements ICloudClient {
+
+ public static final ICloudClient INSTANCE = new NoopCloudClient();
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private NoopCloudClient() {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // no-op
+ }
+
+ @Override
+ public int getWriteBufferSize() {
+ return 0;
+ }
+
+ @Override
+ public IRequestProfilerLimiter getProfilerLimiter() {
+ return NoOpRequestProfilerLimiter.INSTANCE;
+ }
+
+ @Override
+ public ICloudWriter createWriter(String bucket, String path,
IWriteBufferProvider bufferProvider) {
+ return new NoOpCloudWriter();
+ }
+
+ @Override
+ public Set<CloudFile> listObjects(String bucket, String path,
FilenameFilter filter) {
+ return Set.of();
+ }
+
+ @Override
+ public int read(String bucket, String path, long offset, ByteBuffer
buffer) throws HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public byte[] readAllBytes(String bucket, String path) throws
HyracksDataException {
+ return new byte[0];
+ }
+
+ @Override
+ public InputStream getObjectStream(String bucket, String path, long
offset, long length) {
+ return InputStream.nullInputStream();
+ }
+
+ @Override
+ public void write(String bucket, String path, byte[] data) {
+ }
+
+ @Override
+ public void copy(String bucket, String srcPath, FileReference destPath) {
+ }
+
+ @Override
+ public void deleteObjects(String bucket, Collection<String> paths) throws
HyracksDataException {
+ }
+
+ @Override
+ public long getObjectSize(String bucket, String path) throws
HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public boolean exists(String bucket, String path) throws
HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean isEmptyPrefix(String bucket, String path) throws
HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public IParallelDownloader createParallelDownloader(String bucket,
IOManager ioManager)
+ throws HyracksDataException {
+ return NoOpParallelDownloader.INSTANCE;
+ }
+
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ return OBJECT_MAPPER.createArrayNode();
+ }
+
+ private static class NoOpCloudWriter implements ICloudWriter {
+
+ long position = 0L;
+
+ public NoOpCloudWriter() {
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ position = 0L;
+ }
+
+ @Override
+ public int write(ByteBuffer page) throws HyracksDataException {
+ int written = page.remaining();
+ position += written;
+ page.position(page.limit());
+ return written;
+ }
+
+ @Override
+ public void write(int b) throws HyracksDataException {
+ position++;
+ }
+
+ @Override
+ public int write(byte[] b, int off, int len) throws
HyracksDataException {
+ position += len;
+ return len;
+ }
+
+ @Override
+ public long position() {
+ return position;
+ }
+
+ @Override
+ public void finish() throws HyracksDataException {
+ position = 0;
+ }
+ }
+
+ private static class NoOpParallelDownloader implements IParallelDownloader
{
+ private static final NoOpParallelDownloader INSTANCE = new
NoOpParallelDownloader();
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void downloadFiles(Collection<FileReference> toDownload) throws
HyracksDataException {
+ }
+
+ @Override
+ public Collection<FileReference>
downloadDirectories(Collection<FileReference> toDownload)
+ throws HyracksDataException {
+ return List.of();
+ }
+ }
+}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
index 1b39251334..cb246503fa 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
@@ -169,11 +169,6 @@ public class UnstableCloudClient implements ICloudClient {
this.writeBufferSize = writeBufferSize;
}
- @Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
@Override
public int write(ByteBuffer page) throws HyracksDataException {
if (position() == 0) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
index 3a837867c6..04b2de10a9 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSWriter.java
@@ -57,11 +57,6 @@ public class GCSWriter implements ICloudWriter {
writtenBytes = 0;
}
- @Override
- public int write(ByteBuffer header, ByteBuffer page) throws
HyracksDataException {
- return write(header) + write(page);
- }
-
@Override
public int write(ByteBuffer page) throws HyracksDataException {
guardian.checkIsolatedWriteAccess(bucket, path);
