This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b1c1937e94f Change last update timestamp granularity of GCS objects
from seconds to milliseconds (#16083)
b1c1937e94f is described below
commit b1c1937e94fef7cada74e908b3fe7cc073c14d4d
Author: Vishesh Garg <[email protected]>
AuthorDate: Sat Mar 9 07:54:33 2024 +0530
Change last update timestamp granularity of GCS objects from seconds to
milliseconds (#16083)
The previously used GCS API client library returned last update time for
objects directly in milliseconds. The new library returns it in OffsetDateTime
format which was being converted to seconds and stored against the object. This
fix converts the time back to ms before storing it.
---
.../apache/druid/storage/google/GoogleStorage.java | 4 ++--
.../google/GoogleStorageObjectMetadata.java | 26 +++++++++++++---------
.../druid/storage/google/GoogleTaskLogs.java | 6 ++---
.../google/GoogleTimestampVersionedDataFinder.java | 2 +-
.../druid/storage/google/GoogleStorageTest.java | 11 +++++----
.../GoogleTimestampVersionedDataFinderTest.java | 16 ++++++-------
.../overlord/duty/TaskLogAutoCleanerConfig.java | 4 ++++
.../org/apache/druid/tasklogs/TaskLogKiller.java | 6 +++++
8 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
index 15cca5e08d5..57c2ac1843b 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
@@ -141,7 +141,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
- .toEpochSecond()
+ .toEpochSecond() * 1000
);
}
@@ -250,7 +250,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
- .toEpochSecond()
+ .toEpochSecond() * 1000
))
.collect(Collectors.toList());
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
index 87feb774a5d..ce4b4ca3064 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
@@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata
final String bucket;
final String name;
final Long size;
- Long lastUpdateTime;
+ Long lastUpdateTimeMillis;
- public GoogleStorageObjectMetadata(final String bucket, final String name,
final Long size, final Long lastUpdateTime)
+ public GoogleStorageObjectMetadata(
+ final String bucket,
+ final String name,
+ final Long size,
+ final Long lastUpdateTimeMillis
+ )
{
this.bucket = bucket;
this.name = name;
this.size = size;
- this.lastUpdateTime = lastUpdateTime;
+ this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
- public void setLastUpdateTime(Long lastUpdateTime)
+ public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis)
{
- this.lastUpdateTime = lastUpdateTime;
+ this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
@@ -57,9 +62,9 @@ public class GoogleStorageObjectMetadata
return size;
}
- public Long getLastUpdateTime()
+ public Long getLastUpdateTimeMillis()
{
- return lastUpdateTime;
+ return lastUpdateTimeMillis;
}
@Override
@@ -74,13 +79,14 @@ public class GoogleStorageObjectMetadata
GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o;
return Objects.equals(bucket, that.bucket)
&& Objects.equals(name, that.name)
- && Objects.equals(size, that.size);
+ && Objects.equals(size, that.size)
+ && Objects.equals(lastUpdateTimeMillis,
that.getLastUpdateTimeMillis());
}
@Override
public int hashCode()
{
- return Objects.hash(bucket, name, size);
+ return Objects.hash(bucket, name, size, lastUpdateTimeMillis);
}
@Override
@@ -90,7 +96,7 @@ public class GoogleStorageObjectMetadata
"bucket='" + bucket + '\'' +
", name='" + name + '\'' +
", size=" + size +
- ", lastUpdateTime=" + lastUpdateTime +
+ ", lastUpdateTimeMillis=" + lastUpdateTimeMillis +
'}';
}
}
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
index 4f7444f8ea9..a11694f4a2f 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
@@ -190,13 +190,13 @@ public class GoogleTaskLogs implements TaskLogs
}
@Override
- public void killOlderThan(long timestamp) throws IOException
+ public void killOlderThan(long timestampMs) throws IOException
{
LOG.info(
"Deleting all task logs from gs location [bucket: '%s' prefix: '%s']
older than %s.",
config.getBucket(),
config.getPrefix(),
- new Date(timestamp)
+ new Date(timestampMs)
);
try {
GoogleUtils.deleteObjectsInPath(
@@ -204,7 +204,7 @@ public class GoogleTaskLogs implements TaskLogs
inputDataConfig,
config.getBucket(),
config.getPrefix(),
- (object) -> object.getLastUpdateTime() < timestamp
+ (object) -> object.getLastUpdateTimeMillis() < timestampMs
);
}
catch (Exception e) {
diff --git
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
index b93128cc2fa..01ae094912c 100644
---
a/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
+++
b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
@@ -67,7 +67,7 @@ public class GoogleTimestampVersionedDataFinder extends
GoogleDataSegmentPuller
if (pattern != null && !pattern.matcher(keyString).matches()) {
continue;
}
- final long latestModified = objectMetadata.getLastUpdateTime();
+ final long latestModified = objectMetadata.getLastUpdateTimeMillis();
if (latestModified >= mostRecent) {
mostRecent = latestModified;
latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS);
diff --git
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
index f227d593ce6..0a6d346d1c4 100644
---
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
+++
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
@@ -143,7 +143,7 @@ public class GoogleStorageTest
}
@Test
- public void testGetMetadata() throws IOException
+ public void testGetMetadataMatch() throws IOException
{
EasyMock.expect(mockStorage.get(
EasyMock.eq(BUCKET),
@@ -159,7 +159,10 @@ public class GoogleStorageTest
EasyMock.replay(mockStorage, blob);
GoogleStorageObjectMetadata objectMetadata =
googleStorage.getMetadata(BUCKET, PATH);
- assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH,
SIZE, UPDATE_TIME.toEpochSecond()));
+ assertEquals(
+ objectMetadata,
+ new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE,
UPDATE_TIME.toEpochSecond() * 1000)
+ );
}
@@ -262,13 +265,13 @@ public class GoogleStorageTest
bucket1,
path1,
size1,
- updateTime1.toEpochSecond()
+ updateTime1.toEpochSecond() * 1000
);
GoogleStorageObjectMetadata objectMetadata2 = new
GoogleStorageObjectMetadata(
bucket2,
path2,
size2,
- updateTime2.toEpochSecond()
+ updateTime2.toEpochSecond() * 1000
);
GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH,
null, null);
diff --git
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
index b9417b7f7f0..2c65d386379 100644
---
a/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
+++
b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
@@ -37,13 +37,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0);
- storageObject1.setLastUpdateTime(System.currentTimeMillis());
+ storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1);
- storageObject2.setLastUpdateTime(System.currentTimeMillis());
+ storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1);
- storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4);
- storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage =
ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1,
storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new
GoogleTimestampVersionedDataFinder(storage);
@@ -61,13 +61,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0);
- storageObject1.setLastUpdateTime(System.currentTimeMillis());
+ storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1);
- storageObject2.setLastUpdateTime(System.currentTimeMillis());
+ storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1);
- storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 =
ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4);
- storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
+ storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage =
ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1,
storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new
GoogleTimestampVersionedDataFinder(storage);
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
index 71945ad9230..26c2a9362a4 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
@@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig
@JsonProperty
private final long durationToRetain;
+ /**
+ * Config for Task logs auto-cleaner.
+ * All time-related parameters should be in milliseconds.
+ */
@JsonCreator
public TaskLogAutoCleanerConfig(
@JsonProperty("enabled") boolean enabled,
diff --git
a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
index d2a3d0e92fd..76dc7b8e92f 100644
--- a/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
+++ b/processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
@@ -30,5 +30,11 @@ import java.io.IOException;
public interface TaskLogKiller
{
void killAll() throws IOException;
+
+ /**
+ * Removes logs older than the provided timestamp
+ * @param timestamp Timestamp in milliseconds
+ * @throws IOException
+ */
void killOlderThan(long timestamp) throws IOException;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
