This is an automated email from the ASF dual-hosted git repository. kfaraz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push: new b1c1937e94f Change last update timestamp granularity of GCS objects from seconds to milliseconds (#16083) b1c1937e94f is described below commit b1c1937e94fef7cada74e908b3fe7cc073c14d4d Author: Vishesh Garg <gargvish...@gmail.com> AuthorDate: Sat Mar 9 07:54:33 2024 +0530 Change last update timestamp granularity of GCS objects from seconds to milliseconds (#16083) The previously used GCS API client library returned last update time for objects directly in milliseconds. The new library returns it in OffsetDateTime format which was being converted to seconds and stored against the object. This fix converts the time back to ms before storing it. --- .../apache/druid/storage/google/GoogleStorage.java | 4 ++-- .../google/GoogleStorageObjectMetadata.java | 26 +++++++++++++--------- .../druid/storage/google/GoogleTaskLogs.java | 6 ++--- .../google/GoogleTimestampVersionedDataFinder.java | 2 +- .../druid/storage/google/GoogleStorageTest.java | 11 +++++---- .../GoogleTimestampVersionedDataFinderTest.java | 16 ++++++------- .../overlord/duty/TaskLogAutoCleanerConfig.java | 4 ++++ .../org/apache/druid/tasklogs/TaskLogKiller.java | 6 +++++ 8 files changed, 47 insertions(+), 28 deletions(-) diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java index 15cca5e08d5..57c2ac1843b 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java @@ -141,7 +141,7 @@ public class GoogleStorage blob.getName(), blob.getSize(), blob.getUpdateTimeOffsetDateTime() - .toEpochSecond() + .toEpochSecond() * 1000 ); } @@ -250,7 +250,7 @@ public class GoogleStorage blob.getName(), blob.getSize(), blob.getUpdateTimeOffsetDateTime() - .toEpochSecond() + .toEpochSecond() * 1000 )) .collect(Collectors.toList()); diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java index 87feb774a5d..ce4b4ca3064 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java @@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata final String bucket; final String name; final Long size; - Long lastUpdateTime; + Long lastUpdateTimeMillis; - public GoogleStorageObjectMetadata(final String bucket, final String name, final Long size, final Long lastUpdateTime) + public GoogleStorageObjectMetadata( + final String bucket, + final String name, + final Long size, + final Long lastUpdateTimeMillis + ) { this.bucket = bucket; this.name = name; this.size = size; - this.lastUpdateTime = lastUpdateTime; + this.lastUpdateTimeMillis = lastUpdateTimeMillis; } - public void setLastUpdateTime(Long lastUpdateTime) + public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis) { - this.lastUpdateTime = lastUpdateTime; + this.lastUpdateTimeMillis = lastUpdateTimeMillis; } @@ -57,9 +62,9 @@ public class GoogleStorageObjectMetadata return size; } - public Long getLastUpdateTime() + public Long getLastUpdateTimeMillis() { - return lastUpdateTime; + return lastUpdateTimeMillis; } @Override @@ -74,13 +79,14 @@ public class GoogleStorageObjectMetadata GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o; return Objects.equals(bucket, that.bucket) && Objects.equals(name, that.name) - && Objects.equals(size, that.size); + && Objects.equals(size, that.size) + && Objects.equals(lastUpdateTimeMillis, that.getLastUpdateTimeMillis()); } @Override public int hashCode() { - return Objects.hash(bucket, name, size); + return Objects.hash(bucket, name, size, lastUpdateTimeMillis); } @Override @@ -90,7 +96,7 @@ public class GoogleStorageObjectMetadata "bucket='" + bucket + '\'' + ", name='" + name + '\'' + ", size=" + size + - ", lastUpdateTime=" + lastUpdateTime + + ", lastUpdateTimeMillis=" + lastUpdateTimeMillis + '}'; } } diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java index 4f7444f8ea9..a11694f4a2f 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java @@ -190,13 +190,13 @@ public class GoogleTaskLogs implements TaskLogs } @Override - public void killOlderThan(long timestamp) throws IOException + public void killOlderThan(long timestampMs) throws IOException { LOG.info( "Deleting all task logs from gs location [bucket: '%s' prefix: '%s'] older than %s.", config.getBucket(), config.getPrefix(), - new Date(timestamp) + new Date(timestampMs) ); try { GoogleUtils.deleteObjectsInPath( @@ -204,7 +204,7 @@ public class GoogleTaskLogs implements TaskLogs inputDataConfig, config.getBucket(), config.getPrefix(), - (object) -> object.getLastUpdateTime() < timestamp + (object) -> object.getLastUpdateTimeMillis() < timestampMs ); } catch (Exception e) { diff --git a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java index b93128cc2fa..01ae094912c 100644 --- a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java @@ -67,7 +67,7 @@ public class GoogleTimestampVersionedDataFinder extends GoogleDataSegmentPuller if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } - final long latestModified = objectMetadata.getLastUpdateTime(); + final long latestModified = objectMetadata.getLastUpdateTimeMillis(); if (latestModified >= mostRecent) { mostRecent = latestModified; latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS); diff --git a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java index f227d593ce6..0a6d346d1c4 100644 --- a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java +++ b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java @@ -143,7 +143,7 @@ public class GoogleStorageTest } @Test - public void testGetMetadata() throws IOException + public void testGetMetadataMatch() throws IOException { EasyMock.expect(mockStorage.get( EasyMock.eq(BUCKET), @@ -159,7 +159,10 @@ public class GoogleStorageTest EasyMock.replay(mockStorage, blob); GoogleStorageObjectMetadata objectMetadata = googleStorage.getMetadata(BUCKET, PATH); - assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond())); + assertEquals( + objectMetadata, + new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond() * 1000) + ); } @@ -262,13 +265,13 @@ public class GoogleStorageTest bucket1, path1, size1, - updateTime1.toEpochSecond() + updateTime1.toEpochSecond() * 1000 ); GoogleStorageObjectMetadata objectMetadata2 = new GoogleStorageObjectMetadata( bucket2, path2, size2, - updateTime2.toEpochSecond() + updateTime2.toEpochSecond() * 1000 ); GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH, null, null); diff --git a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java index b9417b7f7f0..2c65d386379 100644 --- a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java +++ b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java @@ -37,13 +37,13 @@ public class GoogleTimestampVersionedDataFinderTest // object for directory prefix/dir/0/ final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0); - storageObject1.setLastUpdateTime(System.currentTimeMillis()); + storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1); - storageObject2.setLastUpdateTime(System.currentTimeMillis()); + storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1); - storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4); - storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4)); final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage); @@ -61,13 +61,13 @@ public class GoogleTimestampVersionedDataFinderTest // object for directory prefix/dir/0/ final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0); - storageObject1.setLastUpdateTime(System.currentTimeMillis()); + storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1); - storageObject2.setLastUpdateTime(System.currentTimeMillis()); + storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis()); final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1); - storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4); - storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100); + storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100); final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4)); final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java index 71945ad9230..26c2a9362a4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java @@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig @JsonProperty private final long durationToRetain; + /** + * Config for Task logs auto-cleaner. + * All time-related parameters should be in milliseconds. + */ @JsonCreator public TaskLogAutoCleanerConfig( @JsonProperty("enabled") boolean enabled, diff --git a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java index d2a3d0e92fd..76dc7b8e92f 100644 --- a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java +++ b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java @@ -30,5 +30,11 @@ import java.io.IOException; public interface TaskLogKiller { void killAll() throws IOException; + + /** + * Removes logs older than the provided timestamp + * @param timestamp Timestamp in milliseconds + * @throws IOException + */ void killOlderThan(long timestamp) throws IOException; } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org