This is an automated email from the ASF dual-hosted git repository. ofuks pushed a commit to branch bucket-browser-azure in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/bucket-browser-azure by this push: new 54d19cd [DLAB-1552, DLAB-1803] Azure bucket browser 54d19cd is described below commit 54d19cddb93b538ad7e5c26b0b5640007aaf5696 Author: Oleh Fuks <olegfuk...@gmail.com> AuthorDate: Fri May 15 14:50:28 2020 +0300 [DLAB-1552, DLAB-1803] Azure bucket browser --- services/provisioning-service/pom.xml | 11 +++- .../dlab/backendapi/resources/BucketResource.java | 5 +- .../dlab/backendapi/service/BucketService.java | 2 +- .../service/impl/aws/BucketServiceAwsImpl.java | 9 ++-- .../service/impl/azure/BucketServiceAzureImpl.java | 63 ++++++++++++++++++++-- .../service/impl/gcp/BucketServiceGcpImpl.java | 2 +- .../dlab/backendapi/resources/BucketResource.java | 5 +- .../dlab/backendapi/service/BucketService.java | 2 +- .../backendapi/service/impl/BucketServiceImpl.java | 7 +-- 9 files changed, 86 insertions(+), 20 deletions(-) diff --git a/services/provisioning-service/pom.xml b/services/provisioning-service/pom.xml index fc01af1..043f596 100644 --- a/services/provisioning-service/pom.xml +++ b/services/provisioning-service/pom.xml @@ -101,7 +101,16 @@ <artifactId>httpclient</artifactId> <version>4.5.9</version> </dependency> - + <dependency> + <groupId>com.azure</groupId> + <artifactId>azure-storage-blob</artifactId> + <version>12.6.0</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>2.11.0</version> + </dependency> <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java index d81bf61..180d324 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java @@ -65,8 +65,9 @@ public class BucketResource { public Response uploadObject(@Auth UserInfo userInfo, @FormDataParam("object") String object, @FormDataParam("bucket") String bucket, - @FormDataParam("file") InputStream inputStream) { - bucketService.uploadObject(bucket, object, inputStream); + @FormDataParam("file") InputStream inputStream, + @FormDataParam("file-size") long fileSize) { + bucketService.uploadObject(bucket, object, inputStream, fileSize); return Response.ok().build(); } diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java index edaf29c..f69dfe7 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java @@ -30,7 +30,7 @@ public interface BucketService { List<BucketDTO> getObjects(String bucket); - void uploadObject(String bucket, String object, InputStream stream); + void uploadObject(String bucket, String object, InputStream stream, long fileSize); void downloadObject(String bucket, String object, HttpServletResponse resp); diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java index 83f65e3..d7cef73 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/aws/BucketServiceAwsImpl.java @@ -24,8 +24,6 @@ import com.epam.dlab.dto.bucket.BucketDTO; import com.epam.dlab.exceptions.DlabException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import software.amazon.awssdk.awscore.exception.AwsServiceException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3Client; @@ -68,7 +66,7 @@ public class BucketServiceAwsImpl implements BucketService { } @Override - public void uploadObject(String bucket, String object, InputStream stream) { + public void uploadObject(String bucket, String object, InputStream stream, long fileSize) { try { S3Client s3 = S3Client.create(); PutObjectRequest uploadRequest = PutObjectRequest @@ -76,7 +74,7 @@ public class BucketServiceAwsImpl implements BucketService { .bucket(bucket) .key(object) .build(); - s3.putObject(uploadRequest, RequestBody.fromBytes(IOUtils.toByteArray(stream))); + s3.putObject(uploadRequest, RequestBody.fromInputStream(stream, fileSize)); } catch (Exception e) { log.error("Cannot upload object {} to bucket {}. Reason: {}", object, bucket, e.getMessage()); throw new DlabException(String.format("Cannot upload object %s to bucket %s. Reason: %s", object, bucket, e.getMessage())); @@ -118,8 +116,7 @@ public class BucketServiceAwsImpl implements BucketService { .build(); s3.deleteObjects(deleteObjectsRequests); - - } catch (AwsServiceException e) { + } catch (Exception e) { log.error("Cannot delete objects {} from bucket {}. Reason: {}", objects, bucket, e.getMessage()); throw new DlabException(String.format("Cannot delete objects %s from bucket %s. Reason: %s", objects, bucket, e.getMessage())); } diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java index 1112b5e..c650c6e 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/azure/BucketServiceAzureImpl.java @@ -19,30 +19,87 @@ package com.epam.dlab.backendapi.service.impl.azure; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobItem; import com.epam.dlab.backendapi.service.BucketService; import com.epam.dlab.dto.bucket.BucketDTO; +import com.epam.dlab.exceptions.DlabException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; import java.io.InputStream; +import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.stream.Collectors; +@Slf4j public class BucketServiceAzureImpl implements BucketService { @Override public List<BucketDTO> getObjects(String bucket) { - return null; + try { + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + return blobContainerClient.listBlobs() + .stream() + .map(blob -> toBucketDTO(bucket, blob)) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Cannot retrieve objects from bucket {}. Reason: {}", bucket, e.getMessage()); + throw new DlabException(String.format("Cannot retrieve objects from bucket %s. Reason: %s", bucket, e.getMessage())); + } } @Override - public void uploadObject(String bucket, String object, InputStream stream) { - + public void uploadObject(String bucket, String object, InputStream stream, long fileSize) { + try { + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + BlobClient blobClient = blobContainerClient.getBlobClient(object); + blobClient.upload(stream, fileSize); + } catch (Exception e) { + log.error("Cannot upload object {} to bucket {}. Reason: {}", object, bucket, e.getMessage()); + throw new DlabException(String.format("Cannot upload object %s to bucket %s. Reason: %s", object, bucket, e.getMessage())); + } } @Override public void downloadObject(String bucket, String object, HttpServletResponse resp) { + try (ServletOutputStream outputStream = resp.getOutputStream()) { + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + BlobClient blobClient = blobContainerClient.getBlobClient(object); + blobClient.download(outputStream); + } catch (Exception e) { + log.error("Cannot download object {} from bucket {}. Reason: {}", object, bucket, e.getMessage()); + throw new DlabException(String.format("Cannot download object %s from bucket %s. Reason: %s", object, bucket, e.getMessage())); + } } @Override public void deleteObjects(String bucket, List<String> objects) { + try { + BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(System.getenv("AZURE_STORAGE_CONNECTION_STRING")).buildClient(); + BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(bucket); + objects.forEach(object -> blobContainerClient.getBlobClient(object).delete()); + } catch (Exception e) { + log.error("Cannot delete objects {} from bucket {}. Reason: {}", objects, bucket, e.getMessage()); + throw new DlabException(String.format("Cannot delete objects %s from bucket %s. Reason: %s", objects, bucket, e.getMessage())); + } + } + private BucketDTO toBucketDTO(String bucket, BlobItem blob) { + final String size = FileUtils.byteCountToDisplaySize(blob.getProperties().getContentLength()); + String lastModifiedDate = blob.getProperties().getLastModified().format(DateTimeFormatter.ofPattern(DATE_FORMAT)); + return BucketDTO.builder() + .bucket(bucket) + .object(blob.getName()) + .lastModifiedDate(lastModifiedDate) + .size(size) + .build(); } } diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java index dc068fc..bd9abf6 100644 --- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java +++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/gcp/BucketServiceGcpImpl.java @@ -58,7 +58,7 @@ public class BucketServiceGcpImpl implements BucketService { } @Override - public void uploadObject(String bucket, String object, InputStream stream) { + public void uploadObject(String bucket, String object, InputStream stream, long fileSize) { log.info("Uploading file {} to bucket {}", object, bucket); try { Storage storage = StorageOptions.getDefaultInstance().getService(); diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java index f34c3f3..f757950 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/BucketResource.java @@ -73,8 +73,9 @@ public class BucketResource { @FormDataParam("object") String object, @FormDataParam("bucket") String bucket, @FormDataParam("endpoint") String endpoint, - @FormDataParam("file") InputStream fileInputStream) { - bucketService.uploadObjects(userInfo, bucket, object, endpoint, fileInputStream); + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("size") long fileSize) { + bucketService.uploadObjects(userInfo, bucket, object, endpoint, fileInputStream, fileSize); return Response.ok().build(); } diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java index 62a0e2e..6e5345b 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BucketService.java @@ -29,7 +29,7 @@ import java.util.List; public interface BucketService { List<BucketDTO> getObjects(UserInfo userInfo, String bucket, String endpoint); - void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream); + void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream, long fileSize); void downloadObject(UserInfo userInfo, String bucket, String object, String endpoint, HttpServletResponse resp); diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java index 1372ada..d305532 100644 --- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java +++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BucketServiceImpl.java @@ -79,11 +79,11 @@ public class BucketServiceImpl implements BucketService { } @Override - public void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream) { + public void uploadObjects(UserInfo userInfo, String bucket, String object, String endpoint, InputStream inputStream, long fileSize) { log.info("Uploading file {} for user {} to bucket {}", object, userInfo.getName(), bucket); try { EndpointDTO endpointDTO = endpointService.get(endpoint); - FormDataMultiPart formData = getFormDataMultiPart(bucket, object, inputStream); + FormDataMultiPart formData = getFormDataMultiPart(bucket, object, inputStream, fileSize); Response response = provisioningService.postForm(String.format(BUCKET_UPLOAD_OBJECT, endpointDTO.getUrl()), userInfo.getAccessToken(), formData, Response.class); if (response.getStatus() != HttpStatus.SC_OK) { throw new DlabException(String.format("Something went wrong. Response status is %s ", response.getStatus())); @@ -128,11 +128,12 @@ public class BucketServiceImpl implements BucketService { return URLEncoder.encode(object, StandardCharsets.UTF_8.toString()).replace("+", "%20"); } - private FormDataMultiPart getFormDataMultiPart(String bucket, String object, InputStream inputStream) { + private FormDataMultiPart getFormDataMultiPart(String bucket, String object, InputStream inputStream, long fileSize) { StreamDataBodyPart filePart = new StreamDataBodyPart("file", inputStream, object, MediaType.valueOf(APPLICATION_OCTET_STREAM)); FormDataMultiPart formData = new FormDataMultiPart(); formData.field("bucket", bucket); formData.field("object", object); + formData.field("file-size", String.valueOf(fileSize)); formData.bodyPart(filePart); return formData; } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@dlab.apache.org For additional commands, e-mail: commits-h...@dlab.apache.org