>From Michael Blow <[email protected]>:
Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18293 )
Change subject: [NO ISSUE] Introduce ICloudGuardian, for validating cloud access
......................................................................
[NO ISSUE] Introduce ICloudGuardian, for validating cloud access
Change-Id: I07eab4697803103d6cf626580f09512de5fd71a2
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
M
asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/EagerCloudIOManager.java
A
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudGuardian.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
M
asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/gcs/LSMGCSTest.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
M
hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudManagerProvider.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
16 files changed, 173 insertions(+), 29 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/93/18293/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index cd52b9d..bee20e1 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -35,6 +35,7 @@
import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.cloud.CloudManagerProvider;
import org.apache.asterix.cloud.LocalPartitionBootstrapper;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.common.api.IConfigValidator;
import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.ICoordinationService;
@@ -214,8 +215,8 @@
IDiskCachedPageAllocator pageAllocator =
DefaultDiskCachedPageAllocator.INSTANCE;
IBufferCacheReadContext defaultContext =
DefaultBufferCacheReadContextProvider.DEFAULT;
if (isCloudDeployment()) {
- persistenceIOManager =
- CloudManagerProvider.createIOManager(cloudProperties,
ioManager, namespacePathResolver);
+ persistenceIOManager =
CloudManagerProvider.createIOManager(cloudProperties, ioManager,
+ namespacePathResolver,
createCloudGuardian(cloudProperties));
partitionBootstrapper =
CloudManagerProvider.getCloudPartitionBootstrapper(persistenceIOManager);
} else {
persistenceIOManager = ioManager;
@@ -338,6 +339,10 @@
diskWriteRateLimiterProvider = new DiskWriteRateLimiterProvider();
}
+ protected ICloudGuardian createCloudGuardian(CloudProperties
cloudProperties) {
+ return ICloudGuardian.NoOpCloudGuardian.INSTANCE;
+ }
+
@Override
public boolean isShuttingdown() {
return isShuttingdown;
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index ba3ffcd..0c8b257 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -62,6 +62,7 @@
import org.apache.asterix.app.replication.NcLifecycleCoordinator;
import org.apache.asterix.app.result.JobResultCallback;
import org.apache.asterix.cloud.CloudManagerProvider;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IConfigValidatorFactory;
import org.apache.asterix.common.api.INamespacePathResolver;
@@ -186,8 +187,8 @@
CloudProperties cloudProperties = null;
if (cloudDeployment) {
cloudProperties = new
CloudProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
- ioManager =
- (IOManager)
CloudManagerProvider.createIOManager(cloudProperties, ioManager,
namespacePathResolver);
+ ioManager = (IOManager)
CloudManagerProvider.createIOManager(cloudProperties, ioManager,
+ namespacePathResolver,
createCloudGuardian(cloudProperties));
}
IGlobalTxManager globalTxManager = createGlobalTxManager(ioManager);
appCtx = createApplicationContext(null, globalRecoveryManager,
lifecycleCoordinator, Receptionist::new,
@@ -217,6 +218,10 @@
jobCapacityController = new
JobCapacityController(controllerService.getResourceManager());
}
+ protected ICloudGuardian createCloudGuardian(CloudProperties
cloudProperties) {
+ return ICloudGuardian.NoOpCloudGuardian.INSTANCE;
+ }
+
private Map<String, String> parseCredentialMap(String credPath) {
File credentialFile = new File(credPath);
Map<String, String> storedCredentials = new HashMap<>();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 9df26f2..2bafade 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -36,6 +36,7 @@
import org.apache.asterix.cloud.clients.CloudClientProvider;
import org.apache.asterix.cloud.clients.CloudFile;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.ICloudWriter;
import org.apache.asterix.cloud.util.CloudFileUtil;
import org.apache.asterix.common.api.INamespacePathResolver;
@@ -60,6 +61,7 @@
public abstract class AbstractCloudIOManager extends IOManager implements
IPartitionBootstrapper, ICloudIOManager {
private static final Logger LOGGER = LogManager.getLogger();
protected final ICloudClient cloudClient;
+ protected final ICloudGuardian guardian;
protected final IWriteBufferProvider writeBufferProvider;
protected final String bucket;
protected final Set<Integer> partitions;
@@ -68,12 +70,13 @@
protected final INamespacePathResolver nsPathResolver;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
- INamespacePathResolver nsPathResolver) throws HyracksDataException
{
+ INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
super(ioManager.getIODevices(), ioManager.getDeviceComputer(),
ioManager.getIOParallelism(),
ioManager.getQueueSize());
this.nsPathResolver = nsPathResolver;
this.bucket = cloudProperties.getStorageBucket();
- cloudClient = CloudClientProvider.getClient(cloudProperties);
+ cloudClient = CloudClientProvider.getClient(cloudProperties, guardian);
+ this.guardian = guardian;
int numOfThreads = getIODevices().size() * getIOParallelism();
writeBufferProvider = new WriteBufferProvider(numOfThreads,
cloudClient.getWriteBufferSize());
partitions = new HashSet<>();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudManagerProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudManagerProvider.java
index fb61d7a..b49f3ce 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudManagerProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudManagerProvider.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.cloud;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.cloud.CloudCachePolicy;
import org.apache.asterix.common.cloud.IPartitionBootstrapper;
@@ -31,13 +32,13 @@
}
public static IIOManager createIOManager(CloudProperties cloudProperties,
IIOManager ioManager,
- INamespacePathResolver nsPathResolver) throws HyracksDataException
{
+ INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
IOManager localIoManager = (IOManager) ioManager;
if (cloudProperties.getCloudCachePolicy() == CloudCachePolicy.LAZY) {
- return new LazyCloudIOManager(localIoManager, cloudProperties,
nsPathResolver, false);
+ return new LazyCloudIOManager(localIoManager, cloudProperties,
nsPathResolver, false, guardian);
}
- return new EagerCloudIOManager(localIoManager, cloudProperties,
nsPathResolver);
+ return new EagerCloudIOManager(localIoManager, cloudProperties,
nsPathResolver, guardian);
}
public static IPartitionBootstrapper
getCloudPartitionBootstrapper(IIOManager ioManager) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/EagerCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/EagerCloudIOManager.java
index 04979e6..b8009d3 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/EagerCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/EagerCloudIOManager.java
@@ -25,6 +25,7 @@
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.IParallelDownloader;
import org.apache.asterix.common.api.INamespacePathResolver;
import org.apache.asterix.common.config.CloudProperties;
@@ -47,8 +48,8 @@
private static final Logger LOGGER = LogManager.getLogger();
public EagerCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
- INamespacePathResolver nsPathResolver) throws HyracksDataException
{
- super(ioManager, cloudProperties, nsPathResolver);
+ INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
+ super(ioManager, cloudProperties, nsPathResolver, guardian);
}
/*
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
index 612237a..d64f373 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
@@ -33,6 +33,7 @@
import org.apache.asterix.cloud.bulk.DeleteBulkCloudOperation;
import org.apache.asterix.cloud.clients.CloudFile;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.IParallelDownloader;
import org.apache.asterix.cloud.lazy.ParallelCacher;
import org.apache.asterix.cloud.lazy.accessor.ILazyAccessor;
@@ -68,8 +69,8 @@
private ILazyAccessor accessor;
public LazyCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
- INamespacePathResolver nsPathResolver, boolean
replaceableAccessor) throws HyracksDataException {
- super(ioManager, cloudProperties, nsPathResolver);
+ INamespacePathResolver nsPathResolver, boolean
replaceableAccessor, ICloudGuardian guardian) throws HyracksDataException {
+ super(ioManager, cloudProperties, nsPathResolver, guardian);
accessor = new InitialCloudAccessor(cloudClient, bucket,
localIoManager);
puncher = HolePuncherProvider.get(this, cloudProperties,
writeBufferProvider);
if (replaceableAccessor) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
index d5f508d..d6b570f 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudClientProvider.java
@@ -33,14 +33,15 @@
throw new AssertionError("do not instantiate");
}
- public static ICloudClient getClient(CloudProperties cloudProperties)
throws HyracksDataException {
+ public static ICloudClient getClient(CloudProperties cloudProperties,
ICloudGuardian guardian)
+ throws HyracksDataException {
String storageScheme = cloudProperties.getStorageScheme();
if (S3.equalsIgnoreCase(storageScheme)) {
S3ClientConfig config = S3ClientConfig.of(cloudProperties);
- return new S3CloudClient(config);
+ return new S3CloudClient(config, guardian);
} else if (GCS.equalsIgnoreCase(storageScheme)) {
GCSClientConfig config = GCSClientConfig.of(cloudProperties);
- return new GCSCloudClient(config);
+ return new GCSCloudClient(config, guardian);
}
throw new IllegalStateException("unsupported cloud storage scheme: " +
storageScheme);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudGuardian.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudGuardian.java
new file mode 100644
index 0000000..2125cdd
--- /dev/null
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudGuardian.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+/**
+ * Interface containing methods to perform IO operation on the Cloud Storage
+ */
+public interface ICloudGuardian {
+ /**
+ * Ensure we have authorization to perform isolated writes on the bucket.
Isolated means that the writes do not
+ * interfere with any other node, at least at present. In the event that
the isolated writes can become interfering
+ * in the future (i.e. committed), the {@link #checkWriteAccess(String,
String)} method should be invoked just
+ * prior.
+ */
+ void checkIsolatedWriteAccess(String bucket, String path);
+
+ /**
+ * Ensure we have authorization to perform writes on the bucket. These
writes are not isolated in that they are
+ * visibly side-effecting immediately.
+ */
+ void checkWriteAccess(String bucket, String path);
+
+ /**
+ * Ensure we have authorization to perform reads on the bucket.
+ */
+ void checkReadAccess(String bucket, String path);
+
+ void setCloudClient(ICloudClient cloudClient);
+
+ class NoOpCloudGuardian implements ICloudGuardian {
+
+ public static final NoOpCloudGuardian INSTANCE = new
NoOpCloudGuardian();
+
+ private NoOpCloudGuardian() {
+ }
+
+ @Override
+ public void checkIsolatedWriteAccess(String bucket, String path) {
+ // no-op
+ }
+
+ @Override
+ public void checkWriteAccess(String bucket, String path) {
+ // no-op
+ }
+
+ @Override
+ public void checkReadAccess(String bucket, String path) {
+ // no-op
+ }
+
+ @Override
+ public void setCloudClient(ICloudClient cloudClient) {
+ }
+ }
+}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
index a6579c2..93be80c 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -24,6 +24,7 @@
import java.util.concurrent.TimeUnit;
import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.profiler.IRequestProfiler;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.logging.log4j.LogManager;
@@ -45,6 +46,7 @@
private static final Logger LOGGER = LogManager.getLogger();
private final S3Client s3Client;
private final IRequestProfiler profiler;
+ private final ICloudGuardian guardian;
private final String bucket;
private final String path;
private final List<CompletedPart> partQueue;
@@ -52,9 +54,11 @@
private String uploadId;
private int partNumber;
- public S3BufferedWriter(S3Client s3client, IRequestProfiler profiler,
String bucket, String path) {
+ public S3BufferedWriter(S3Client s3client, IRequestProfiler profiler,
ICloudGuardian guardian, String bucket,
+ String path) {
this.s3Client = s3client;
this.profiler = profiler;
+ this.guardian = guardian;
this.bucket = bucket;
this.path = path;
partQueue = new ArrayList<>();
@@ -62,6 +66,7 @@
@Override
public int upload(InputStream stream, int length) {
+ guardian.checkIsolatedWriteAccess(bucket, path);
profiler.objectMultipartUpload();
setUploadId();
UploadPartRequest upReq =
@@ -123,6 +128,7 @@
}
private void completeMultipartUpload(CompleteMultipartUploadRequest
request) throws HyracksDataException {
+ guardian.checkWriteAccess(bucket, path);
profiler.objectMultipartUpload();
try {
s3Client.completeMultipartUpload(request);
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 38a6801..f395362 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -40,6 +40,7 @@
import org.apache.asterix.cloud.clients.CloudFile;
import org.apache.asterix.cloud.clients.ICloudBufferedWriter;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.ICloudWriter;
import org.apache.asterix.cloud.clients.IParallelDownloader;
import org.apache.asterix.cloud.clients.profiler.CountRequestProfiler;
@@ -76,21 +77,24 @@
public final class S3CloudClient implements ICloudClient {
private final S3ClientConfig config;
private final S3Client s3Client;
+ private final ICloudGuardian guardian;
private final IRequestProfiler profiler;
- public S3CloudClient(S3ClientConfig config) {
- this(config, buildClient(config));
+ public S3CloudClient(S3ClientConfig config, ICloudGuardian guardian) {
+ this(config, buildClient(config), guardian);
}
- public S3CloudClient(S3ClientConfig config, S3Client s3Client) {
+ public S3CloudClient(S3ClientConfig config, S3Client s3Client,
ICloudGuardian guardian) {
this.config = config;
this.s3Client = s3Client;
+ this.guardian = guardian;
long profilerInterval = config.getProfilerLogInterval();
if (profilerInterval > 0) {
profiler = new CountRequestProfiler(profilerInterval);
} else {
profiler = NoOpRequestProfiler.INSTANCE;
}
+ guardian.setCloudClient(this);
}
@Override
@@ -100,12 +104,13 @@
@Override
public ICloudWriter createWriter(String bucket, String path,
IWriteBufferProvider bufferProvider) {
- ICloudBufferedWriter bufferedWriter = new S3BufferedWriter(s3Client,
profiler, bucket, path);
+ ICloudBufferedWriter bufferedWriter = new S3BufferedWriter(s3Client,
profiler, guardian, bucket, path);
return new CloudResettableInputStream(bufferedWriter, bufferProvider);
}
@Override
public Set<CloudFile> listObjects(String bucket, String path,
FilenameFilter filter) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectsList();
path = config.isLocalS3Provider() ? encodeURI(path) : path;
return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
@@ -113,6 +118,7 @@
@Override
public int read(String bucket, String path, long offset, ByteBuffer
buffer) throws HyracksDataException {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
long readTo = offset + buffer.remaining();
GetObjectRequest rangeGetObjectRequest =
@@ -140,6 +146,7 @@
@Override
public byte[] readAllBytes(String bucket, String path) throws
HyracksDataException {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
GetObjectRequest getReq =
GetObjectRequest.builder().bucket(bucket).key(path).build();
@@ -154,6 +161,7 @@
@Override
public InputStream getObjectStream(String bucket, String path, long
offset, long length) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
long readTo = offset + length;
GetObjectRequest getReq =
@@ -168,6 +176,7 @@
@Override
public void write(String bucket, String path, byte[] data) {
+ guardian.checkWriteAccess(bucket, path);
profiler.objectWrite();
PutObjectRequest putReq =
PutObjectRequest.builder().bucket(bucket).key(path).build();
s3Client.putObject(putReq, RequestBody.fromBytes(data));
@@ -175,11 +184,13 @@
@Override
public void copy(String bucket, String srcPath, FileReference destPath) {
+ guardian.checkReadAccess(bucket, srcPath);
srcPath = config.isLocalS3Provider() ? encodeURI(srcPath) : srcPath;
List<S3Object> objects = listS3Objects(s3Client, bucket, srcPath);
profiler.objectsList();
for (S3Object object : objects) {
+ guardian.checkWriteAccess(bucket, destPath.getRelativePath());
profiler.objectCopy();
String srcKey = object.key();
String destKey =
destPath.getChildPath(IoUtil.getFileNameFromPath(srcKey));
@@ -201,7 +212,9 @@
while (pathIter.hasNext()) {
objectIdentifiers.clear();
for (int i = 0; pathIter.hasNext() && i < DELETE_BATCH_SIZE; i++) {
- objectIdentifiers.add(builder.key(pathIter.next()).build());
+ String path = pathIter.next();
+ guardian.checkWriteAccess(bucket, path);
+ objectIdentifiers.add(builder.key(path).build());
}
Delete delete =
Delete.builder().objects(objectIdentifiers).build();
@@ -213,6 +226,7 @@
@Override
public long getObjectSize(String bucket, String path) throws
HyracksDataException {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
try {
return
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build()).contentLength();
@@ -225,6 +239,7 @@
@Override
public boolean exists(String bucket, String path) throws
HyracksDataException {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
try {
s3Client.headObject(HeadObjectRequest.builder().bucket(bucket).key(path).build());
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index 9bcd386..accf3c9 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -35,6 +35,7 @@
import org.apache.asterix.cloud.IWriteBufferProvider;
import org.apache.asterix.cloud.clients.CloudFile;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.ICloudWriter;
import org.apache.asterix.cloud.clients.IParallelDownloader;
import org.apache.asterix.cloud.clients.profiler.CountRequestProfiler;
@@ -64,21 +65,24 @@
public class GCSCloudClient implements ICloudClient {
private final Storage gcsClient;
private final GCSClientConfig config;
+ private final ICloudGuardian guardian;
private final IRequestProfiler profiler;
- public GCSCloudClient(GCSClientConfig config, Storage gcsClient) {
+ public GCSCloudClient(GCSClientConfig config, Storage gcsClient,
ICloudGuardian guardian) {
this.gcsClient = gcsClient;
this.config = config;
+ this.guardian = guardian;
long profilerInterval = config.getProfilerLogInterval();
if (profilerInterval > 0) {
profiler = new CountRequestProfiler(profilerInterval);
} else {
profiler = NoOpRequestProfiler.INSTANCE;
}
+ guardian.setCloudClient(this);
}
- public GCSCloudClient(GCSClientConfig config) throws HyracksDataException {
- this(config, buildClient(config));
+ public GCSCloudClient(GCSClientConfig config, ICloudGuardian guardian)
throws HyracksDataException {
+ this(config, buildClient(config), guardian);
}
@Override
@@ -93,6 +97,7 @@
@Override
public Set<CloudFile> listObjects(String bucket, String path,
FilenameFilter filter) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectsList();
Page<Blob> blobs =
gcsClient.list(bucket, BlobListOption.prefix(path),
BlobListOption.fields(Storage.BlobField.SIZE));
@@ -151,6 +156,7 @@
@Override
public void write(String bucket, String path, byte[] data) {
+ guardian.checkWriteAccess(bucket, path);
profiler.objectWrite();
BlobInfo blobInfo = BlobInfo.newBuilder(bucket, path).build();
gcsClient.create(blobInfo, data);
@@ -165,6 +171,7 @@
BlobId source = blob.getBlobId();
String targetName =
destPath.getChildPath(IoUtil.getFileNameFromPath(source.getName()));
BlobId target = BlobId.of(bucket, targetName);
+ guardian.checkWriteAccess(bucket, targetName);
CopyRequest copyReq =
CopyRequest.newBuilder().setSource(source).setTarget(target).build();
gcsClient.copy(copyReq);
}
@@ -182,6 +189,7 @@
batchRequest = gcsClient.batch();
for (int i = 0; pathIter.hasNext() && i < DELETE_BATCH_SIZE; i++) {
BlobId blobId = BlobId.of(bucket, pathIter.next());
+ guardian.checkWriteAccess(bucket, blobId.getName());
batchRequest.delete(blobId);
}
@@ -192,6 +200,7 @@
@Override
public long getObjectSize(String bucket, String path) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path,
Storage.BlobGetOption.fields(Storage.BlobField.SIZE));
if (blob == null) {
@@ -202,6 +211,7 @@
@Override
public boolean exists(String bucket, String path) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectGet();
Blob blob = gcsClient.get(bucket, path,
Storage.BlobGetOption.fields(Storage.BlobField.values()));
return blob != null && blob.exists();
@@ -209,6 +219,7 @@
@Override
public boolean isEmptyPrefix(String bucket, String path) {
+ guardian.checkReadAccess(bucket, path);
profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket, BlobListOption.prefix(path));
return !blobs.hasNextPage();
@@ -222,6 +233,8 @@
@Override
public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ guardian.checkReadAccess(bucket, "/");
+ profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket,
BlobListOption.fields(Storage.BlobField.SIZE));
ArrayNode objectsInfo = objectMapper.createArrayNode();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
index 3245869..9e9c003 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/GCSExternalFileWriterFactory.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig;
import org.apache.asterix.cloud.clients.google.gcs.GCSCloudClient;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -62,7 +63,8 @@
@Override
ICloudClient createCloudClient() throws CompilationException {
GCSClientConfig config = GCSClientConfig.of(configuration);
- return new GCSCloudClient(config, GCSUtils.buildClient(configuration));
+ return new GCSCloudClient(config, GCSUtils.buildClient(configuration),
+ ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index 96aa929..dcaf488 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.aws.s3.S3ClientConfig;
import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -62,7 +63,8 @@
@Override
ICloudClient createCloudClient() throws CompilationException {
S3ClientConfig config = S3ClientConfig.of(configuration);
- return new S3CloudClient(config,
S3Utils.buildAwsS3Client(configuration));
+ return new S3CloudClient(config,
S3Utils.buildAwsS3Client(configuration),
+ ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/gcs/LSMGCSTest.java
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/gcs/LSMGCSTest.java
index 86bb5ad..3c62cce 100644
---
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/gcs/LSMGCSTest.java
+++
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/gcs/LSMGCSTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.cloud.gcs;
import org.apache.asterix.cloud.AbstractLSMTest;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.google.gcs.GCSClientConfig;
import org.apache.asterix.cloud.clients.google.gcs.GCSCloudClient;
import org.junit.AfterClass;
@@ -48,7 +49,7 @@
.setLocation(MOCK_SERVER_REGION).build());
LOGGER.info("Client created successfully");
GCSClientConfig config = new GCSClientConfig(MOCK_SERVER_REGION,
MOCK_SERVER_HOSTNAME, "", true, 0);
- CLOUD_CLIENT = new GCSCloudClient(config);
+ CLOUD_CLIENT = new GCSCloudClient(config,
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
private static void cleanup() {
diff --git
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
index c785e57..01d3422 100644
---
a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
+++
b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
@@ -21,6 +21,7 @@
import java.net.URI;
import org.apache.asterix.cloud.AbstractLSMTest;
+import org.apache.asterix.cloud.clients.ICloudGuardian;
import org.apache.asterix.cloud.clients.aws.s3.S3ClientConfig;
import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
import org.junit.AfterClass;
@@ -65,7 +66,7 @@
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
LOGGER.info("Client created successfully");
S3ClientConfig config = new S3ClientConfig(MOCK_SERVER_REGION,
MOCK_SERVER_HOSTNAME, "", true, 0);
- CLOUD_CLIENT = new S3CloudClient(config);
+ CLOUD_CLIENT = new S3CloudClient(config,
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
private static void cleanup() {
diff --git
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
index 1a40360..3e2e955 100644
---
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
+++
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/Span.java
@@ -109,6 +109,12 @@
}
public static Span start(long span, TimeUnit unit) {
+ Span s = new Span(span, unit);
+ s.reset();
+ return s;
+ }
+
+ public static Span init(long span, TimeUnit unit) {
return new Span(span, unit);
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18293
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I07eab4697803103d6cf626580f09512de5fd71a2
Gerrit-Change-Number: 18293
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <[email protected]>
Gerrit-MessageType: newchange
