bgaborg commented on a change in pull request #802: HADOOP-16279. S3Guard:
Implement time-based (TTL) expiry for entries …
URL: https://github.com/apache/hadoop/pull/802#discussion_r291482234
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
##########
@@ -551,17 +553,82 @@ public static void putWithTtl(MetadataStore ms,
DirListingMetadata dirMeta,
ms.put(dirMeta);
}
- public static DirListingMetadata listChildrenWithTtl(MetadataStore ms,
- Path path, ITtlTimeProvider timeProvider)
+ public static void putWithTtl(MetadataStore ms, PathMetadata fileMeta,
+ @Nullable ITtlTimeProvider timeProvider) throws IOException {
+ if (timeProvider != null) {
+ fileMeta.setLastUpdated(timeProvider.getNow());
+ } else {
+ LOG.debug("timeProvider is null, put {} without setting last_updated",
+ fileMeta);
+ }
+ ms.put(fileMeta);
+ }
+
+ public static void putWithTtl(MetadataStore ms,
+ Collection<PathMetadata> fileMetas,
+ @Nullable ITtlTimeProvider timeProvider)
throws IOException {
- long ttl = timeProvider.getAuthoritativeDirTtl();
+ if (timeProvider != null) {
+ final long now = timeProvider.getNow();
+ fileMetas.forEach(fileMeta -> fileMeta.setLastUpdated(now));
+ } else {
+ LOG.debug("timeProvider is null, put {} without setting last_updated",
+ fileMetas);
+ }
+ ms.put(fileMetas);
+ }
+
+ public static PathMetadata getWithTtl(MetadataStore ms, Path path,
+ @Nullable ITtlTimeProvider timeProvider) throws IOException {
+ final PathMetadata pathMetadata = ms.get(path);
+ // if timeProvider is null let's return with what the ms has
+ if (timeProvider == null) {
+ LOG.debug("timeProvider is null, returning pathMetadata as is");
+ return pathMetadata;
+ }
+ long ttl = timeProvider.getMetadataTtl();
+
+ if (pathMetadata != null) {
+ // Special case: the pathmetadata's last updated is 0. This can happen
+ // eg. with an old db using this implementation
+ if (pathMetadata.getLastUpdated() == 0) {
Review comment:
there's space between every `if` and `(`. I'll resolve this, I was not able
to find an `if(` here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
