This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch 29.0.1
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/29.0.1 by this push:
new 7029caa40b2 [Backport] Change last update timestamp granularity of GCS
objects from seconds to milliseconds (#16086)
7029caa40b2 is described below
commit 7029caa40b22ff22f3c876ac5470d9ce25251490
Author: Vishesh Garg <[email protected]>
AuthorDate: Sat Mar 9 08:44:40 2024 +0530
[Backport] Change last update timestamp granularity of GCS objects from
seconds to milliseconds (#16086)
Backport of PR #16083
---
.../apache/druid/storage/google/GoogleStorage.java | 4 ++--
.../google/GoogleStorageObjectMetadata.java | 26 +++++++++++++---------
.../druid/storage/google/GoogleTaskLogs.java | 6 ++---
.../google/GoogleTimestampVersionedDataFinder.java | 2 +-
.../druid/storage/google/GoogleStorageTest.java | 11 +++++----
.../GoogleTimestampVersionedDataFinderTest.java | 16 ++++++-------
.../overlord/duty/TaskLogAutoCleanerConfig.java | 4 ++++
.../org/apache/druid/tasklogs/TaskLogKiller.java | 6 +++++
8 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
index 91d290b1785..301f848339f 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
@@ -138,7 +138,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
- .toEpochSecond()
+ .toEpochSecond() * 1000
);
}
@@ -234,7 +234,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
- .toEpochSecond()
+ .toEpochSecond() * 1000
))
.collect(Collectors.toList());
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
index 87feb774a5d..ce4b4ca3064 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
@@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata
final String bucket;
final String name;
final Long size;
- Long lastUpdateTime;
+ Long lastUpdateTimeMillis;
- public GoogleStorageObjectMetadata(final String bucket, final String name,
final Long size, final Long lastUpdateTime)
+ public GoogleStorageObjectMetadata(
+ final String bucket,
+ final String name,
+ final Long size,
+ final Long lastUpdateTimeMillis
+ )
{
this.bucket = bucket;
this.name = name;
this.size = size;
- this.lastUpdateTime = lastUpdateTime;
+ this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
- public void setLastUpdateTime(Long lastUpdateTime)
+ public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis)
{
- this.lastUpdateTime = lastUpdateTime;
+ this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
@@ -57,9 +62,9 @@ public class GoogleStorageObjectMetadata
return size;
}
- public Long getLastUpdateTime()
+ public Long getLastUpdateTimeMillis()
{
- return lastUpdateTime;
+ return lastUpdateTimeMillis;
}
@Override
@@ -74,13 +79,14 @@ public class GoogleStorageObjectMetadata
GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o;
return Objects.equals(bucket, that.bucket)
&& Objects.equals(name, that.name)
- && Objects.equals(size, that.size);
+ && Objects.equals(size, that.size)
+ && Objects.equals(lastUpdateTimeMillis,
that.getLastUpdateTimeMillis());
}
@Override
public int hashCode()
{
- return Objects.hash(bucket, name, size);
+ return Objects.hash(bucket, name, size, lastUpdateTimeMillis);
}
@Override
@@ -90,7 +96,7 @@ public class GoogleStorageObjectMetadata
"bucket='" + bucket + '\'' +
", name='" + name + '\'' +
", size=" + size +
- ", lastUpdateTime=" + lastUpdateTime +
+ ", lastUpdateTimeMillis=" + lastUpdateTimeMillis +
'}';
}
}
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
index 4f7444f8ea9..a11694f4a2f 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
@@ -190,13 +190,13 @@ public class GoogleTaskLogs implements TaskLogs
}
@Override
- public void killOlderThan(long timestamp) throws IOException
+ public void killOlderThan(long timestampMs) throws IOException
{
LOG.info(
"Deleting all task logs from gs location [bucket: '%s' prefix: '%s']
older than %s.",
config.getBucket(),
config.getPrefix(),
- new Date(timestamp)
+ new Date(timestampMs)
);
try {
GoogleUtils.deleteObjectsInPath(
@@ -204,7 +204,7 @@ public class GoogleTaskLogs implements TaskLogs
inputDataConfig,
config.getBucket(),
config.getPrefix(),
- (object) -> object.getLastUpdateTime() < timestamp
+ (object) -> object.getLastUpdateTimeMillis() < timestampMs
);
}
catch (Exception e) {
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
index b93128cc2fa..01ae094912c 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
@@ -67,7 +67,7 @@ public class GoogleTimestampVersionedDataFinder extends
GoogleDataSegmentPuller
if (pattern != null && !pattern.matcher(keyString).matches()) {
continue;
}
- final long latestModified = objectMetadata.getLastUpdateTime();
+ final long latestModified = objectMetadata.getLastUpdateTimeMillis();
if (latestModified >= mostRecent) {
mostRecent = latestModified;
latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS);
diff --git
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
index d92339f53c7..71562802207 100644
---
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
+++
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
@@ -124,7 +124,7 @@ public class GoogleStorageTest
}
@Test
- public void testGetMetadata() throws IOException
+ public void testGetMetadataMatch() throws IOException
{
EasyMock.expect(mockStorage.get(
EasyMock.eq(BUCKET),
@@ -140,7 +140,10 @@ public class GoogleStorageTest
EasyMock.replay(mockStorage, blob);
GoogleStorageObjectMetadata objectMetadata =
googleStorage.getMetadata(BUCKET, PATH);
- assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH,
SIZE, UPDATE_TIME.toEpochSecond()));
+ assertEquals(
+ objectMetadata,
+ new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE,
UPDATE_TIME.toEpochSecond() * 1000)
+ );
}
@@ -243,13 +246,13 @@ public class GoogleStorageTest
bucket1,
path1,
size1,
- updateTime1.toEpochSecond()
+ updateTime1.toEpochSecond() * 1000
);
GoogleStorageObjectMetadata objectMetadata2 = new
GoogleStorageObjectMetadata(
bucket2,
path2,
size2,
- updateTime2.toEpochSecond()
+ updateTime2.toEpochSecond() * 1000
);
GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH,
null, null);
diff --git
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
index b9417b7f7f0..2c65d386379 100644
---
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
+++
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
@@ -37,13 +37,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0);
- storageObject1.setLastUpdateTime(System.currentTimeMillis());
+ storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1);
- storageObject2.setLastUpdateTime(System.currentTimeMillis());
+ storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1);
- storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4);
- storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage =
ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1,
storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new
GoogleTimestampVersionedDataFinder(storage);
@@ -61,13 +61,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0);
- storageObject1.setLastUpdateTime(System.currentTimeMillis());
+ storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1);
- storageObject2.setLastUpdateTime(System.currentTimeMillis());
+ storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1);
- storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4);
- storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage =
ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1,
storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new
GoogleTimestampVersionedDataFinder(storage);
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
index 71945ad9230..26c2a9362a4 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
@@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig
@JsonProperty
private final long durationToRetain;
+ /**
+ * Config for Task logs auto-cleaner.
+ * All time-related parameters should be in milliseconds.
+ */
@JsonCreator
public TaskLogAutoCleanerConfig(
@JsonProperty("enabled") boolean enabled,
diff --git
a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
index d2a3d0e92fd..76dc7b8e92f 100644
--- a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
+++ b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
@@ -30,5 +30,11 @@ import java.io.IOException;
public interface TaskLogKiller
{
void killAll() throws IOException;
+
+ /**
+ * Removes logs older than the provided timestamp
+ * @param timestamp Timestamp in milliseconds
+ * @throws IOException
+ */
void killOlderThan(long timestamp) throws IOException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
