Author: mattryan
Date: Tue Aug 27 07:30:21 2019
New Revision: 1865972
URL: http://svn.apache.org/viewvc?rev=1865972&view=rev
Log:
OAK-8574: Reduce cloud service API calls in completeHttpUpload()
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1865972&r1=1865971&r2=1865972&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
Tue Aug 27 07:30:21 2019
@@ -365,12 +365,12 @@ public class AzureBlobStoreBackend exten
CloudBlockBlob blob =
getAzureContainer().getBlockBlobReference(key);
blob.downloadAttributes();
AzureBlobStoreDataRecord record = new AzureBlobStoreDataRecord(
- this,
- connectionString,
- containerName,
- new DataIdentifier(getIdentifierName(blob.getName())),
- blob.getProperties().getLastModified().getTime(),
- blob.getProperties().getLength());
+ this,
+ connectionString,
+ containerName,
+ new DataIdentifier(getIdentifierName(blob.getName())),
+ blob.getProperties().getLastModified().getTime(),
+ blob.getProperties().getLength());
LOG.debug("Data record read for blob. identifier={} duration={}
record={}",
key, (System.currentTimeMillis() - start), record);
return record;
@@ -945,34 +945,54 @@ public class AzureBlobStoreBackend exten
String key = uploadToken.getBlobId();
DataIdentifier blobId = new DataIdentifier(getIdentifierName(key));
- if (! exists(blobId)) {
+ DataRecord record = null;
+ try {
+ record = getRecord(blobId);
+ // If this succeeds this means either it was a "single put" upload
+ // (we don't need to do anything in this case - blob is already
uploaded)
+ // or it was completed before with the same token.
+ }
+ catch (DataStoreException e) {
+ // record doesn't exist - so this means we are safe to do the
complete request
try {
if (uploadToken.getUploadId().isPresent()) {
- // An existing upload ID means this is a multi-part upload
CloudBlockBlob blob =
getAzureContainer().getBlockBlobReference(key);
+ // An existing upload ID means this is a multi-part upload
List<BlockEntry> blocks = blob.downloadBlockList(
BlockListingFilter.UNCOMMITTED,
AccessCondition.generateEmptyCondition(),
null,
null);
blob.commitBlockList(blocks);
- }
- // else do nothing - single put is already complete
-
- if (!exists(blobId)) {
+ long size = 0L;
+ for (BlockEntry block : blocks) {
+ size += block.getSize();
+ }
+ record = new AzureBlobStoreDataRecord(
+ this,
+ connectionString,
+ containerName,
+ blobId,
+ blob.getProperties().getLastModified().getTime(),
+ size);
+ }
+ else {
+ // Something is wrong - upload ID missing from upload token
+ // but record doesn't exist already, so this is invalid
throw new DataRecordUploadException(
- String.format("Unable to finalize direct write of
binary %s", blobId));
+ String.format("Unable to finalize direct write of
binary %s - upload ID missing from upload token",
+ blobId)
+ );
}
- } catch (URISyntaxException | StorageException e) {
+ } catch (URISyntaxException | StorageException e2) {
throw new DataRecordUploadException(
String.format("Unable to finalize direct write of
binary %s", blobId),
e
);
}
}
- // else return the already existing record for this blob ID
- return getRecord(blobId);
+ return record;
}
private URI createPresignedURI(String key,
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1865972&r1=1865971&r2=1865972&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Tue Aug 27 07:30:21 2019
@@ -905,41 +905,64 @@ public class S3Backend extends AbstractS
}
DataRecordUploadToken uploadToken =
DataRecordUploadToken.fromEncodedToken(uploadTokenStr,
getOrCreateReferenceKey());
- String blobId = uploadToken.getBlobId();
- DataIdentifier dataIdentifier = new
DataIdentifier(getIdentifierName(blobId));
+ String key = uploadToken.getBlobId();
+ DataIdentifier blobId = new DataIdentifier(getIdentifierName(key));
- if (! exists(dataIdentifier)) {
+ DataRecord record = null;
+ try {
+ record = getRecord(blobId);
+ // If this succeeds this means either it was a "single put" upload
+ // (we don't need to do anything in this case - blob is already
uploaded)
+ // or it was completed before with the same token.
+ }
+ catch (DataStoreException e) {
+ // record doesn't exist - so this means we are safe to do the
complete request
if (uploadToken.getUploadId().isPresent()) {
// An existing upload ID means this is a multi-part upload
String uploadId = uploadToken.getUploadId().get();
- ListPartsRequest listPartsRequest = new
ListPartsRequest(bucket, blobId, uploadId);
+ ListPartsRequest listPartsRequest = new
ListPartsRequest(bucket, key, uploadId);
PartListing listing = s3service.listParts(listPartsRequest);
List<PartETag> eTags = Lists.newArrayList();
+ long size = 0L;
+ Date lastModified = null;
for (PartSummary partSummary : listing.getParts()) {
PartETag eTag = new PartETag(partSummary.getPartNumber(),
partSummary.getETag());
eTags.add(eTag);
+ size += partSummary.getSize();
+ if (null == lastModified ||
partSummary.getLastModified().after(lastModified)) {
+ lastModified = partSummary.getLastModified();
+ }
}
CompleteMultipartUploadRequest completeReq = new
CompleteMultipartUploadRequest(
bucket,
- blobId,
+ key,
uploadId,
eTags
);
s3service.completeMultipartUpload(completeReq);
- }
- // else do nothing - single-put upload is already complete
-
- if (!s3service.doesObjectExist(bucket, blobId)) {
+ record = new S3DataRecord(
+ this,
+ s3service,
+ bucket,
+ blobId,
+ lastModified.getTime(),
+ size
+ );
+ }
+ else {
+ // Something is wrong - upload ID missing from upload token
+ // but record doesn't exist already, so this is invalid
throw new DataRecordUploadException(
- String.format("Unable to finalize direct write of
binary %s", blobId)
+ String.format("Unable to finalize direct write of
binary %s - upload ID missing from upload token",
+ blobId)
);
}
}
- return getRecord(new DataIdentifier(getIdentifierName(blobId)));
+ return record;
}
private URI createPresignedURI(DataIdentifier identifier,
