This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch bucket-browser-gcp
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit 5dc1b13e3f41b74e374646bb3e7ba11360e4bb13
Author: Oleh Fuks <olegfuk...@gmail.com>
AuthorDate: Thu Apr 23 15:31:48 2020 +0300
bucket browser aws
---
.../src/ssn/templates/ssn.yml | 2 +-
services/provisioning-service/pom.xml | 16 +++
.../dlab/backendapi/modules/ProductionModule.java | 29 ++---
.../backendapi/modules/ProvisioningDevModule.java | 9 +-
.../dlab/backendapi/resources/BucketResource.java | 8 --
.../service/impl/aws/BucketServiceAwsImpl.java | 125 +++++++++++++++++++++
6 files changed, 164 insertions(+), 25 deletions(-)
diff --git a/infrastructure-provisioning/src/ssn/templates/ssn.yml
b/infrastructure-provisioning/src/ssn/templates/ssn.yml
index 949ddf9..503116f 100644
--- a/infrastructure-provisioning/src/ssn/templates/ssn.yml
+++ b/infrastructure-provisioning/src/ssn/templates/ssn.yml
@@ -64,7 +64,7 @@ provisioningService:
bucketService:
jerseyClient:
- timeout: 5m
+ timeout: 10m
connectionTimeout: 3s
# Log out user on inactivity
diff --git a/services/provisioning-service/pom.xml
b/services/provisioning-service/pom.xml
index 78bc19b..2fc22bd 100644
--- a/services/provisioning-service/pom.xml
+++ b/services/provisioning-service/pom.xml
@@ -38,6 +38,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bom</artifactId>
+ <version>2.11.9</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -85,6 +92,15 @@
<artifactId>google-cloud-storage</artifactId>
<version>1.106.0</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.9</version>
+ </dependency>
<dependency>
<groupId>commons-io</groupId>
diff --git
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
index d01ebc5..6ba8505 100644
---
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
+++
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
@@ -34,6 +34,7 @@ import
com.epam.dlab.backendapi.service.RestoreCallbackHandlerService;
import com.epam.dlab.backendapi.service.impl.CheckInactivityServiceImpl;
import com.epam.dlab.backendapi.service.impl.ProjectServiceImpl;
import com.epam.dlab.backendapi.service.impl.RestoreCallbackHandlerServiceImpl;
+import com.epam.dlab.backendapi.service.impl.aws.BucketServiceAwsImpl;
import com.epam.dlab.backendapi.service.impl.gcp.BucketServiceGcpImpl;
import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.constants.ServiceConsts;
@@ -67,17 +68,19 @@ public class ProductionModule extends
ModuleBase<ProvisioningServiceApplicationC
.build(environment,
ServiceConsts.SECURITY_SERVICE_NAME, ServiceConsts
.PROVISIONING_USER_AGENT));
-
bind(RESTService.class).toInstance(configuration.getSelfFactory().build(environment,
ServiceConsts
- .SELF_SERVICE_NAME));
- bind(MetadataHolder.class).to(DockerWarmuper.class);
-
bind(ICommandExecutor.class).to(CommandExecutor.class).asEagerSingleton();
- bind(ObjectMapper.class).toInstance(new
ObjectMapper().configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true));
- bind(CallbackHandlerDao.class).to(FileSystemCallbackHandlerDao.class);
-
bind(RestoreCallbackHandlerService.class).to(RestoreCallbackHandlerServiceImpl.class);
-
bind(CheckInactivityService.class).to(CheckInactivityServiceImpl.class);
- bind(ProjectService.class).to(ProjectServiceImpl.class);
- if (configuration.getCloudProvider() == CloudProvider.GCP) {
- bind(BucketService.class).to(BucketServiceGcpImpl.class);
- }
- }
+
bind(RESTService.class).toInstance(configuration.getSelfFactory().build(environment,
ServiceConsts
+ .SELF_SERVICE_NAME));
+ bind(MetadataHolder.class).to(DockerWarmuper.class);
+
bind(ICommandExecutor.class).to(CommandExecutor.class).asEagerSingleton();
+ bind(ObjectMapper.class).toInstance(new
ObjectMapper().configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true));
+
bind(CallbackHandlerDao.class).to(FileSystemCallbackHandlerDao.class);
+
bind(RestoreCallbackHandlerService.class).to(RestoreCallbackHandlerServiceImpl.class);
+
bind(CheckInactivityService.class).to(CheckInactivityServiceImpl.class);
+ bind(ProjectService.class).to(ProjectServiceImpl.class);
+ if (configuration.getCloudProvider() == CloudProvider.GCP) {
+
bind(BucketService.class).to(BucketServiceGcpImpl.class);
+ } else if (configuration.getCloudProvider() ==
CloudProvider.AWS) {
+
bind(BucketService.class).to(BucketServiceAwsImpl.class);
+ }
+ }
}
diff --git
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
index 43685a6..f4bbed5 100644
---
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
+++
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
@@ -37,6 +37,7 @@ import
com.epam.dlab.backendapi.service.RestoreCallbackHandlerService;
import com.epam.dlab.backendapi.service.impl.CheckInactivityServiceImpl;
import com.epam.dlab.backendapi.service.impl.ProjectServiceImpl;
import com.epam.dlab.backendapi.service.impl.RestoreCallbackHandlerServiceImpl;
+import com.epam.dlab.backendapi.service.impl.aws.BucketServiceAwsImpl;
import com.epam.dlab.backendapi.service.impl.gcp.BucketServiceGcpImpl;
import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.constants.ServiceConsts;
@@ -82,9 +83,11 @@ public class ProvisioningDevModule extends
ModuleBase<ProvisioningServiceApplica
bind(RestoreCallbackHandlerService.class).to(RestoreCallbackHandlerServiceImpl.class);
bind(CheckInactivityService.class).to(CheckInactivityServiceImpl.class);
bind(ProjectService.class).to(ProjectServiceImpl.class);
- if (configuration.getCloudProvider() == CloudProvider.GCP) {
- bind(BucketService.class).to(BucketServiceGcpImpl.class);
- }
+ if (configuration.getCloudProvider() == CloudProvider.GCP) {
+
bind(BucketService.class).to(BucketServiceGcpImpl.class);
+ } else if (configuration.getCloudProvider() ==
CloudProvider.AWS) {
+
bind(BucketService.class).to(BucketServiceAwsImpl.class);
+ }
}
/**
diff --git
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
index 7859a0a..a44d21f 100644
---
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
+++
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java
@@ -21,8 +21,6 @@ package com.epam.dlab.backendapi.resources;
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.service.BucketService;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.StorageOptions;
import com.google.inject.Inject;
import io.dropwizard.auth.Auth;
import lombok.extern.slf4j.Slf4j;
@@ -92,10 +90,4 @@ public class BucketResource {
bucketService.deleteObject(bucket, object);
return Response.ok().build();
}
-
- public static void main(String[] args) {
- Storage storage = StorageOptions.getDefaultInstance().getService();
- String bucketName = "ofuks-1304-prj1-local-bucket";
- storage.delete(bucketName, "1.txt");
- }
}
diff --git
a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
new file mode 100644
index 0000000..8193fde
--- /dev/null
+++
b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service.impl.aws;
+
+import com.epam.dlab.backendapi.service.BucketService;
+import com.epam.dlab.dto.bucket.BucketDTO;
+import com.epam.dlab.exceptions.DlabException;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import java.io.InputStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class BucketServiceAwsImpl implements BucketService {
+
+ @Override
+ public List<BucketDTO> getObjects(String bucket) {
+ try {
+ S3Client s3 = S3Client.create();
+ ListObjectsRequest getRequest = ListObjectsRequest
+ .builder()
+ .bucket(bucket)
+ .build();
+
+ return s3.listObjects(getRequest).contents()
+ .stream()
+ .map(o -> toBucketDTO(bucket, o))
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ log.error("Cannot retrieve objects from bucket {}. Reason: {}",
bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot retrieve objects
from bucket %s. Reason: %s", bucket, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void uploadObject(String bucket, String object, InputStream stream)
{
+ try {
+ S3Client s3 = S3Client.create();
+ PutObjectRequest uploadRequest = PutObjectRequest
+ .builder()
+ .bucket(bucket)
+ .key(object)
+ .build();
+ s3.putObject(uploadRequest,
RequestBody.fromBytes(IOUtils.toByteArray(stream)));
+ } catch (Exception e) {
+ log.error("Cannot upload object {} to bucket {}. Reason: {}",
object, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot upload object %s to
bucket %s. Reason: %s", object, bucket, e.getMessage()));
+ }
+ }
+
+ @Override
+ public byte[] downloadObject(String bucket, String object) {
+ try {
+ S3Client s3 = S3Client.create();
+ GetObjectRequest downloadRequest = GetObjectRequest
+ .builder()
+ .bucket(bucket)
+ .key(object)
+ .build();
+ return s3.getObject(downloadRequest,
ResponseTransformer.toBytes()).asByteArray();
+ } catch (Exception e) {
+ log.error("Cannot download object {} from bucket {}. Reason: {}",
object, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot download object %s
from bucket %s. Reason: %s", object, bucket, e.getMessage()));
+ }
+ }
+
+ @Override
+ public void deleteObject(String bucket, String object) {
+ try {
+ S3Client s3 = S3Client.create();
+ DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest
+ .builder()
+ .bucket(bucket)
+ .key(object)
+ .build();
+ s3.deleteObject(deleteObjectRequest);
+ } catch (AwsServiceException e) {
+ log.error("Cannot delete object {} from bucket {}. Reason: {}",
object, bucket, e.getMessage());
+ throw new DlabException(String.format("Cannot delete object %s
from bucket %s. Reason: %s", object, bucket, e.getMessage()));
+ }
+ }
+
+ private BucketDTO toBucketDTO(String bucket, S3Object s3Object) {
+ final String size = FileUtils.byteCountToDisplaySize(s3Object.size());
+ Date date = Date.from(s3Object.lastModified());
+ SimpleDateFormat formatter = new SimpleDateFormat(DATE_FORMAT);
+ return BucketDTO.builder()
+ .bucket(bucket)
+ .object(s3Object.key())
+ .size(size)
+ .lastModifiedDate(formatter.format(date))
+ .build();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org
For additional commands, e-mail: commits-h...@dlab.apache.org
