Author: mattryan
Date: Thu Aug 29 08:36:52 2019
New Revision: 1866061
URL: http://svn.apache.org/viewvc?rev=1866061&view=rev
Log:
OAK-8580: Add distinct cloud data store loggers for stream activity
This change allows for specific monitoring of specific named loggers
that indicate where cloud data stores are streaming bytes to or
from cloud storage. Useful for instances that are trying to
identify where upload or download streaming through the repository
is taking place for those that are attempting to replace such
activity with direct binary access in clients.
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1866061&r1=1866060&r2=1866061&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
Thu Aug 29 08:36:52 2019
@@ -91,6 +91,8 @@ import org.slf4j.LoggerFactory;
public class AzureBlobStoreBackend extends AbstractSharedBackend {
private static final Logger LOG =
LoggerFactory.getLogger(AzureBlobStoreBackend.class);
+ private static final Logger LOG_STREAMS_DOWNLOAD =
LoggerFactory.getLogger("oak.datastore.download.streams");
+ private static final Logger LOG_STREAMS_UPLOAD =
LoggerFactory.getLogger("oak.datastore.upload.streams");
private static final String META_DIR_NAME = "META";
private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
@@ -220,6 +222,10 @@ public class AzureBlobStoreBackend exten
InputStream is = blob.openInputStream();
LOG.debug("Got input stream for blob. identifier={} duration={}",
key, (System.currentTimeMillis() - start));
+ if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace to see
where the call came from
+ LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure Blob
Storage - identifier={}", key, new Exception());
+ }
return is;
}
catch (StorageException e) {
@@ -262,6 +268,10 @@ public class AzureBlobStoreBackend exten
try {
blob.upload(in, len, null, options, null);
LOG.debug("Blob created. identifier={} length={}
duration={} buffered={}", key, len, (System.currentTimeMillis() - start),
useBufferedStream);
+ if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace
to see where the call came from
+ LOG_STREAMS_UPLOAD.debug("Binary uploaded to Azure
Blob Storage - identifier={}", key, new Exception());
+ }
} finally {
in.close();
}
@@ -1203,11 +1213,12 @@ public class AzureBlobStoreBackend exten
if (isMeta) {
id = addMetaKeyPrefix(getIdentifier().toString());
}
- if (LOG.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see
where the call
- // came from
- LOG.debug("binary downloaded from Azure Blob Storage: " +
getIdentifier(),
- new Exception());
+ else {
+ // Don't worry about stream logging for metadata records
+ if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace to
see where the call came from
+ LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from Azure
Blob Storage - identifier={} ", id, new Exception());
+ }
}
try {
return container.getBlockBlobReference(id).openInputStream();
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1866061&r1=1866060&r2=1866061&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Thu Aug 29 08:36:52 2019
@@ -106,6 +106,8 @@ public class S3Backend extends AbstractS
* Logger instance.
*/
private static final Logger LOG = LoggerFactory.getLogger(S3Backend.class);
+ private static final Logger LOG_STREAMS_DOWNLOAD =
LoggerFactory.getLogger("oak.datastore.download.streams");
+ private static final Logger LOG_STREAMS_UPLOAD =
LoggerFactory.getLogger("oak.datastore.upload.streams");
private static final String KEY_PREFIX = "dataStore_";
@@ -333,6 +335,10 @@ public class S3Backend extends AbstractS
// start multipart parallel upload using amazon sdk
Upload up = tmx.upload(s3ReqDecorator.decorate(new
PutObjectRequest(
bucket, key, file)));
+ if (LOG_STREAMS_UPLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace
to see where the call came from
+ LOG_STREAMS_UPLOAD.debug("Binary uploaded to S3 -
identifier={}", key, new Exception());
+ }
// wait for upload to finish
up.waitForUploadResult();
LOG.debug("synchronous upload to identifier [{}]
completed.", identifier);
@@ -393,8 +399,9 @@ public class S3Backend extends AbstractS
S3Object object = s3service.getObject(bucket, key);
InputStream in = object.getObjectContent();
LOG.debug("[{}] read took [{}]ms", identifier,
(System.currentTimeMillis() - start));
- if (LOG.isDebugEnabled()) {
- LOG.debug("binary downloaded from S3: " + identifier, new
Exception());
+ if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace to see
where the call came from
+ LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from S3 -
identifier={}", key, new Exception());
}
return in;
} catch (AmazonServiceException e) {
@@ -1156,10 +1163,12 @@ public class S3Backend extends AbstractS
if (isMeta) {
id = addMetaKeyPrefix(getIdentifier().toString());
}
- if (LOG.isDebugEnabled()) {
- // Log message, with exception so we can get a trace to see
where the call
- // came from
- LOG.debug("binary downloaded from S3: " + getIdentifier(), new
Exception());
+ else {
+ // Don't worry about stream logging for metadata records
+ if (LOG_STREAMS_DOWNLOAD.isDebugEnabled()) {
+ // Log message, with exception so we can get a trace to
see where the call came from
+ LOG_STREAMS_DOWNLOAD.debug("Binary downloaded from S3 -
identifier={}", id, new Exception());
+ }
}
return s3service.getObject(bucket, id).getObjectContent();
}
