This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new f9bdb3b236 Fix usage of maxColumnsToMerge in auto-compaction tuning
config (#12551)
f9bdb3b236 is described below
commit f9bdb3b236c01a32eecd98f13e42fce0e1e5c3d4
Author: superivaj <[email protected]>
AuthorDate: Fri May 20 18:53:08 2022 +0200
Fix usage of maxColumnsToMerge in auto-compaction tuning config (#12551)
Issue:
Even though `CompactionTuningConfig` allows a `maxColumnsToMerge` config
(to optimize memory usage, particulary for datasources with many
dimensions),
the corresponding client object `ClientCompactionTaskQueryTuningConfig`
(used by the coordinator duty `CompactSegments` to trigger auto-compaction)
does not contain this field. Thus, the value of `maxColumnsToMerge`
specified
in any datasource compaction config is ignored.
Changes:
- Add field `maxColumnsToMerge` in `ClientCompactionTaskQueryTuningConfig`
and `UserCompactionTaskQueryTuningConfig`
- Fix tests
---
.../task/ClientCompactionTaskQuerySerdeTest.java | 8 +++++---
.../apache/druid/testing/utils/CompactionUtil.java | 3 ++-
.../tests/coordinator/duty/ITAutoCompactionTest.java | 3 ++-
.../duty/ITAutoCompactionUpgradeTest.java | 3 ++-
.../ClientCompactionTaskQueryTuningConfig.java | 20 ++++++++++++++++++--
.../UserCompactionTaskQueryTuningConfig.java | 6 ++++--
.../coordinator/DataSourceCompactionConfigTest.java | 8 ++++++--
.../UserCompactionTaskQueryTuningConfigTest.java | 4 +++-
.../server/coordinator/duty/CompactSegmentsTest.java | 16 ++++++++++++++++
.../duty/NewestSegmentFirstIteratorTest.java | 8 ++++++++
.../duty/NewestSegmentFirstPolicyTest.java | 2 ++
11 files changed, 68 insertions(+), 13 deletions(-)
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java
index 29952ebc1c..011b35eb63 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/ClientCompactionTaskQuerySerdeTest.java
@@ -122,7 +122,8 @@ public class ClientCompactionTaskQuerySerdeTest
new Duration(3000L),
7,
1000,
- 100
+ 100,
+ 2
),
new ClientCompactionTaskGranularitySpec(Granularities.DAY,
Granularities.HOUR, true),
new
ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts",
"dim"))),
@@ -287,7 +288,7 @@ public class ClientCompactionTaskQuerySerdeTest
null,
null,
null,
- null,
+ 2,
null,
null
)
@@ -342,7 +343,8 @@ public class ClientCompactionTaskQuerySerdeTest
new Duration(3000L),
7,
1000,
- 100
+ 100,
+ 2
),
new ClientCompactionTaskGranularitySpec(Granularities.DAY,
Granularities.HOUR, true),
new
ClientCompactionTaskDimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("ts",
"dim"))),
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java
b/integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java
index accaec0146..a57a404b61 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/utils/CompactionUtil.java
@@ -67,7 +67,8 @@ public class CompactionUtil
null,
null,
null,
- 1
+ 1,
+ null
),
null,
null,
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
index 32d563f25b..edfe387365 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
@@ -1632,7 +1632,8 @@ public class ITAutoCompactionTest extends
AbstractIndexerTest
null,
null,
null,
- 1
+ 1,
+ null
),
granularitySpec,
dimensionsSpec,
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java
index 7c5afb1ffd..9c32cc5055 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionUpgradeTest.java
@@ -95,7 +95,8 @@ public class ITAutoCompactionUpgradeTest extends
AbstractIndexerTest
null,
null,
null,
- 1
+ 1,
+ null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null,
null),
null,
diff --git
a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java
b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java
index 374f61c512..7b1a7c5468 100644
---
a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java
+++
b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionTaskQueryTuningConfig.java
@@ -75,6 +75,8 @@ public class ClientCompactionTaskQueryTuningConfig
@Nullable
private final Integer totalNumMergeTasks;
@Nullable
+ private final Integer maxColumnsToMerge;
+ @Nullable
private final AppendableIndexSpec appendableIndexSpec;
public static ClientCompactionTaskQueryTuningConfig from(
@@ -103,6 +105,7 @@ public class ClientCompactionTaskQueryTuningConfig
null,
null,
null,
+ null,
null
);
} else {
@@ -128,7 +131,8 @@ public class ClientCompactionTaskQueryTuningConfig
userCompactionTaskQueryTuningConfig.getChatHandlerTimeout(),
userCompactionTaskQueryTuningConfig.getChatHandlerNumRetries(),
userCompactionTaskQueryTuningConfig.getMaxNumSegmentsToMerge(),
- userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks()
+ userCompactionTaskQueryTuningConfig.getTotalNumMergeTasks(),
+ userCompactionTaskQueryTuningConfig.getMaxColumnsToMerge()
);
}
}
@@ -153,7 +157,8 @@ public class ClientCompactionTaskQueryTuningConfig
@JsonProperty("chatHandlerTimeout") @Nullable Duration
chatHandlerTimeout,
@JsonProperty("chatHandlerNumRetries") @Nullable Integer
chatHandlerNumRetries,
@JsonProperty("maxNumSegmentsToMerge") @Nullable Integer
maxNumSegmentsToMerge,
- @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
+ @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
+ @JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
this.maxRowsPerSegment = maxRowsPerSegment;
@@ -175,6 +180,7 @@ public class ClientCompactionTaskQueryTuningConfig
this.chatHandlerNumRetries = chatHandlerNumRetries;
this.maxNumSegmentsToMerge = maxNumSegmentsToMerge;
this.totalNumMergeTasks = totalNumMergeTasks;
+ this.maxColumnsToMerge = maxColumnsToMerge;
}
@JsonProperty
@@ -318,6 +324,13 @@ public class ClientCompactionTaskQueryTuningConfig
return totalNumMergeTasks;
}
+ @JsonProperty
+ @Nullable
+ public Integer getMaxColumnsToMerge()
+ {
+ return maxColumnsToMerge;
+ }
+
@JsonProperty
@Nullable
public AppendableIndexSpec getAppendableIndexSpec()
@@ -353,6 +366,7 @@ public class ClientCompactionTaskQueryTuningConfig
Objects.equals(chatHandlerNumRetries, that.chatHandlerNumRetries) &&
Objects.equals(maxNumSegmentsToMerge, that.maxNumSegmentsToMerge) &&
Objects.equals(totalNumMergeTasks, that.totalNumMergeTasks) &&
+ Objects.equals(maxColumnsToMerge, that.maxColumnsToMerge) &&
Objects.equals(appendableIndexSpec, that.appendableIndexSpec);
}
@@ -378,6 +392,7 @@ public class ClientCompactionTaskQueryTuningConfig
chatHandlerNumRetries,
maxNumSegmentsToMerge,
totalNumMergeTasks,
+ maxColumnsToMerge,
appendableIndexSpec
);
}
@@ -404,6 +419,7 @@ public class ClientCompactionTaskQueryTuningConfig
", chatHandlerNumRetries=" + chatHandlerNumRetries +
", maxNumSegmentsToMerge=" + maxNumSegmentsToMerge +
", totalNumMergeTasks=" + totalNumMergeTasks +
+ ", maxColumnsToMerge=" + maxColumnsToMerge +
", appendableIndexSpec=" + appendableIndexSpec +
'}';
}
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java
b/server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java
index 07ab96c343..9c1bfb2006 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfig.java
@@ -53,7 +53,8 @@ public class UserCompactionTaskQueryTuningConfig extends
ClientCompactionTaskQue
@JsonProperty("chatHandlerTimeout") @Nullable Duration
chatHandlerTimeout,
@JsonProperty("chatHandlerNumRetries") @Nullable Integer
chatHandlerNumRetries,
@JsonProperty("maxNumSegmentsToMerge") @Nullable Integer
maxNumSegmentsToMerge,
- @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks
+ @JsonProperty("totalNumMergeTasks") @Nullable Integer totalNumMergeTasks,
+ @JsonProperty("maxColumnsToMerge") @Nullable Integer maxColumnsToMerge
)
{
super(
@@ -75,7 +76,8 @@ public class UserCompactionTaskQueryTuningConfig extends
ClientCompactionTaskQue
chatHandlerTimeout,
chatHandlerNumRetries,
maxNumSegmentsToMerge,
- totalNumMergeTasks
+ totalNumMergeTasks,
+ maxColumnsToMerge
);
}
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
index 81b3311cdd..a0f4cf9a61 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
@@ -141,6 +141,7 @@ public class DataSourceCompactionConfigTest extends
InitializedNullHandlingTest
null,
null,
null,
+ null,
null
),
null,
@@ -189,6 +190,7 @@ public class DataSourceCompactionConfigTest extends
InitializedNullHandlingTest
null,
null,
null,
+ null,
null
),
null,
@@ -242,7 +244,8 @@ public class DataSourceCompactionConfigTest extends
InitializedNullHandlingTest
new Duration(3000L),
7,
1000,
- 100
+ 100,
+ 2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);
@@ -282,7 +285,8 @@ public class DataSourceCompactionConfigTest extends
InitializedNullHandlingTest
new Duration(3000L),
7,
1000,
- 100
+ 100,
+ 2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java
index dc52f78422..01c889ad2c 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/UserCompactionTaskQueryTuningConfigTest.java
@@ -62,6 +62,7 @@ public class UserCompactionTaskQueryTuningConfigTest
null,
null,
null,
+ null,
null
);
final String json = OBJECT_MAPPER.writeValueAsString(config);
@@ -103,7 +104,8 @@ public class UserCompactionTaskQueryTuningConfigTest
new Duration(3000L),
7,
1000,
- 100
+ 100,
+ 2
);
final String json = OBJECT_MAPPER.writeValueAsString(tuningConfig);
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
index 08b32feff3..d201bb8ffb 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
@@ -736,6 +736,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -801,6 +802,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -860,6 +862,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -919,6 +922,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null,
null),
@@ -986,6 +990,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1048,6 +1053,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1109,6 +1115,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null,
true),
@@ -1211,6 +1218,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null,
null),
@@ -1337,6 +1345,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1399,6 +1408,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1465,6 +1475,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1612,6 +1623,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1709,6 +1721,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
new UserCompactionTaskGranularityConfig(Granularities.YEAR, null,
null),
@@ -1774,6 +1787,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -1836,6 +1850,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
@@ -2154,6 +2169,7 @@ public class CompactSegmentsTest
null,
null,
null,
+ null,
null
),
null,
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java
index cbcd1b906b..35021a56a2 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstIteratorTest.java
@@ -133,6 +133,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -177,6 +178,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -221,6 +223,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -265,6 +268,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -309,6 +313,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -353,6 +358,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -397,6 +403,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
@@ -441,6 +448,7 @@ public class NewestSegmentFirstIteratorTest
null,
null,
null,
+ null,
null
),
null,
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java
index 1b7570a972..2ae82a5f11 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/duty/NewestSegmentFirstPolicyTest.java
@@ -1538,6 +1538,7 @@ public class NewestSegmentFirstPolicyTest
null,
null,
null,
+ null,
null
),
null
@@ -1572,6 +1573,7 @@ public class NewestSegmentFirstPolicyTest
null,
null,
null,
+ null,
null
),
null
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
