exceptionfactory commented on code in PR #8088:
URL: https://github.com/apache/nifi/pull/8088#discussion_r1435309260
##
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java:
##
@@ -451,11 +456,92 @@ public void onTrigger(final ProcessContext context, final
ProcessSession session
listByTrackingTimestamps(context, session);
} else if (BY_ENTITIES.equals(listingStrategy)) {
listByTrackingEntities(context, session);
+} else if (NO_TRACKING.equals(listingStrategy)) {
+listNoTracking(context, session);
} else {
throw new ProcessException("Unknown listing strategy: " +
listingStrategy);
}
}
+private void listNoTracking(ProcessContext context, ProcessSession
session) {
+final AmazonS3 client = getClient(context);
+
+S3BucketLister bucketLister = getS3BucketLister(context, client);
+
+final long startNanos = System.nanoTime();
+final long minAgeMilliseconds =
context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
+final Long maxAgeMilliseconds = context.getProperty(MAX_AGE) != null ?
context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS) : null;
+final long listingTimestamp = System.currentTimeMillis();
+
+final String bucket =
context.getProperty(BUCKET_WITHOUT_DEFAULT_VALUE).evaluateAttributeExpressions().getValue();
+final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
+
+int listCount = 0;
+int totalListCount = 0;
+
+getLogger().trace("Start listing, listingTimestamp={}", new
Object[]{listingTimestamp});
+
+final S3ObjectWriter writer;
+final RecordSetWriterFactory writerFactory =
context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
+if (writerFactory == null) {
+writer = new AttributeObjectWriter(session);
+} else {
+writer = new RecordObjectWriter(session, writerFactory,
getLogger(), context.getProperty(S3_REGION).getValue());
+}
+
+try {
+writer.beginListing();
+
+do {
+VersionListing versionListing = bucketLister.listVersions();
+for (S3VersionSummary versionSummary :
versionListing.getVersionSummaries()) {
+long lastModified =
versionSummary.getLastModified().getTime();
+if ((maxAgeMilliseconds != null && (lastModified <
(listingTimestamp - maxAgeMilliseconds)))
+|| lastModified > (listingTimestamp -
minAgeMilliseconds)) {
+continue;
+}
+
+getLogger().trace("Listed key={}, lastModified={}", new
Object[]{versionSummary.getKey(), lastModified});
Review Comment:
```suggestion
getLogger().trace("Listed key={}, lastModified={}",
versionSummary.getKey(), lastModified);
```
##
nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java:
##
@@ -451,11 +456,92 @@ public void onTrigger(final ProcessContext context, final
ProcessSession session
listByTrackingTimestamps(context, session);
} else if (BY_ENTITIES.equals(listingStrategy)) {
listByTrackingEntities(context, session);
+} else if (NO_TRACKING.equals(listingStrategy)) {
+listNoTracking(context, session);
} else {
throw new ProcessException("Unknown listing strategy: " +
listingStrategy);
}
}
+private void listNoTracking(ProcessContext context, ProcessSession
session) {
+final AmazonS3 client = getClient(context);
+
+S3BucketLister bucketLister = getS3BucketLister(context, client);
+
+final long startNanos = System.nanoTime();
+final long minAgeMilliseconds =
context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
+final Long maxAgeMilliseconds = context.getProperty(MAX_AGE) != null ?
context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS) : null;
+final long listingTimestamp = System.currentTimeMillis();
+
+final String bucket =
context.getProperty(BUCKET_WITHOUT_DEFAULT_VALUE).evaluateAttributeExpressions().getValue();
+final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
+
+int listCount = 0;
+int totalListCount = 0;
+
+getLogger().trace("Start listing, listingTimestamp={}", new
Object[]{listingTimestamp});
Review Comment:
The `Object[]` wrapper is not necessary for log arguments, although not all
components have been updated to reflect the recommended usage, new code should
follow the pattern.
```suggestion
getLogger().trace("Start listing, listingTimestamp={}",
listingTimestamp);
```
##
