This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch 27.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/27.0.0 by this push:
new c33035c636 add support for smartSegmentLoading (#14610) (#14681)
c33035c636 is described below
commit c33035c636c466f2e29115910e9350c6fff053db
Author: Kashif Faraz <[email protected]>
AuthorDate: Fri Jul 28 08:15:23 2023 +0530
add support for smartSegmentLoading (#14610) (#14681)
Co-authored-by: Vadim Ogievetsky <[email protected]>
---
.../coordinator-dynamic-config.mock.ts} | 26 ++-
.../coordinator-dynamic-config.tsx | 229 +++++++++++++--------
web-console/src/druid-models/mocks.ts | 1 +
3 files changed, 166 insertions(+), 90 deletions(-)
diff --git a/web-console/src/druid-models/mocks.ts
b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
similarity index 51%
copy from web-console/src/druid-models/mocks.ts
copy to
web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
index 38c924033d..015a5c8b59 100644
--- a/web-console/src/druid-models/mocks.ts
+++
b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
@@ -16,7 +16,25 @@
* limitations under the License.
*/
-export * from './async-query/async-query.mock';
-export * from './execution/execution-ingest-complete.mock';
-export * from './execution/execution-ingest-error.mock';
-export * from './stages/stages.mock';
+import type { CoordinatorDynamicConfig } from './coordinator-dynamic-config';
+
+export const DEFAULT_COORDINATOR_DYNAMIC_CONFIG: CoordinatorDynamicConfig = {
+ millisToWaitBeforeDeleting: 900000,
+ mergeBytesLimit: 524288000,
+ mergeSegmentsLimit: 100,
+ maxSegmentsToMove: 100,
+ replicantLifetime: 15,
+ replicationThrottleLimit: 500,
+ balancerComputeThreads: 1,
+ killDataSourceWhitelist: [],
+ killPendingSegmentsSkipList: [],
+ maxSegmentsInNodeLoadingQueue: 500,
+ decommissioningNodes: [],
+ decommissioningMaxPercentOfMaxSegmentsToMove: 70,
+ pauseCoordination: false,
+ replicateAfterLoadTimeout: false,
+ maxNonPrimaryReplicantsToLoad: 2147483647,
+ useRoundRobinSegmentAssignment: true,
+ smartSegmentLoading: true,
+ debugDimensions: null,
+};
diff --git
a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
index f8ca99c221..04af80cd13 100644
---
a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
+++
b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
@@ -20,6 +20,8 @@ import { Code } from '@blueprintjs/core';
import React from 'react';
import type { Field } from '../../components';
+import { ExternalLink } from '../../components';
+import { getLink } from '../../links';
export interface CoordinatorDynamicConfig {
maxSegmentsToMove?: number;
@@ -37,55 +39,66 @@ export interface CoordinatorDynamicConfig {
decommissioningMaxPercentOfMaxSegmentsToMove?: number;
pauseCoordination?: boolean;
maxNonPrimaryReplicantsToLoad?: number;
+ replicateAfterLoadTimeout?: boolean;
+ useRoundRobinSegmentAssignment?: boolean;
+ smartSegmentLoading?: boolean;
+
+ // Undocumented
+ debugDimensions?: any;
}
export const COORDINATOR_DYNAMIC_CONFIG_FIELDS:
Field<CoordinatorDynamicConfig>[] = [
{
- name: 'maxSegmentsToMove',
- type: 'number',
- defaultValue: 100,
- info: <>The maximum number of segments that can be moved at any given
time.</>,
- },
- {
- name: 'balancerComputeThreads',
- type: 'number',
- defaultValue: 1,
+ name: 'pauseCoordination',
+ type: 'boolean',
+ defaultValue: false,
info: (
<>
- Thread pool size for computing moving cost of segments in segment
balancing. Consider
- increasing this if you have a lot of segments and moving segments
starts to get stuck.
+ Boolean flag for whether or not the coordinator should execute its
various duties of
+ coordinating the cluster. Setting this to true essentially pauses all
coordination work
+ while allowing the API to remain up. Duties that are paused include
all classes that
+ implement the <Code>CoordinatorDuty</Code> interface. Such duties
include: Segment
+ balancing, Segment compaction, Submitting kill tasks for unused
segments (if enabled),
+ Logging of used segments in the cluster, Marking of newly unused or
overshadowed segments,
+ Matching and execution of load/drop rules for used segments, Unloading
segments that are no
+ longer marked as used from Historical servers. An example of when an
admin may want to pause
+ coordination would be if they are doing deep storage maintenance on
HDFS Name Nodes with
+ downtime and don't want the coordinator to be directing
Historical Nodes to hit the
+ Name Node with API requests until maintenance is done and the deep
store is declared healthy
+ for use again.
</>
),
},
+
+ // Start "smart" segment loading section
+
{
- name: 'killDataSourceWhitelist',
- label: 'Kill datasource whitelist',
- type: 'string-array',
- emptyValue: [],
+ name: 'smartSegmentLoading',
+ type: 'boolean',
+ defaultValue: true,
info: (
<>
- List of dataSources for which kill tasks are sent if property{' '}
- <Code>druid.coordinator.kill.on</Code> is true. This can be a list of
comma-separated
- dataSources or a JSON array.
+ Enables{' '}
+ <ExternalLink
href={`${getLink('DOCS')}/configuration#smart-segment-loading`}>
+ "smart" segment loading mode
+ </ExternalLink>{' '}
+ which dynamically computes the optimal values of several properties
that maximize
+ Coordinator performance.
</>
),
},
{
- name: 'killPendingSegmentsSkipList',
- type: 'string-array',
- emptyValue: [],
- info: (
- <>
- List of dataSources for which pendingSegments are NOT cleaned up if
property{' '}
- <Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This
can be a list of
- comma-separated dataSources or a JSON array.
- </>
- ),
+ name: 'maxSegmentsToMove',
+ type: 'number',
+ defaultValue: 100,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
+ info: <>The maximum number of segments that can be moved at any given
time.</>,
},
{
name: 'maxSegmentsInNodeLoadingQueue',
type: 'number',
defaultValue: 500,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
info: (
<>
The maximum number of segments that could be queued for loading to any
given server. This
@@ -99,45 +112,85 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS:
Field<CoordinatorDynamicConfig>[
),
},
{
- name: 'mergeBytesLimit',
- type: 'size-bytes',
- defaultValue: 524288000,
- info: <>The maximum total uncompressed size in bytes of segments to
merge.</>,
+ name: 'useRoundRobinSegmentAssignment',
+ type: 'boolean',
+ defaultValue: true,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
+ info: (
+ <>
+ Boolean flag for whether segments should be assigned to historicals in
a round robin
+ fashion. When disabled, segment assignment is done using the chosen
balancer strategy. When
+ enabled, this can speed up segment assignments leaving balancing to
move the segments to
+ their optimal locations (based on the balancer strategy) lazily.
+ </>
+ ),
},
{
- name: 'mergeSegmentsLimit',
+ name: 'replicationThrottleLimit',
type: 'number',
- defaultValue: 100,
- info: <>The maximum number of segments that can be in a single append
task.</>,
+ defaultValue: 500,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
+ info: <>The maximum number of segments that can be replicated at one
time.</>,
},
{
- name: 'millisToWaitBeforeDeleting',
+ name: 'replicantLifetime',
type: 'number',
- defaultValue: 900000,
+ defaultValue: 15,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
info: (
<>
- How long does the Coordinator need to be active before it can start
removing (marking
- unused) segments in metadata storage.
+ The maximum number of Coordinator runs for which a segment can wait in
the load queue of a
+ Historical before Druid raises an alert.
</>
),
},
{
- name: 'replicantLifetime',
+ name: 'maxNonPrimaryReplicantsToLoad',
type: 'number',
- defaultValue: 15,
+ defaultValue: 2147483647,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
info: (
<>
- The maximum number of Coordinator runs for a segment to be replicated
before we start
- alerting.
+ The maximum number of non-primary replicants to load in a single
Coordinator cycle. Once
+ this limit is hit, only primary replicants will be loaded for the
remainder of the cycle.
+ Tuning this value lower can help reduce the delay in loading primary
segments when the
+ cluster has a very large number of non-primary replicants to load
(such as when a single
+ historical drops out of the cluster leaving many under-replicated
segments).
</>
),
},
{
- name: 'replicationThrottleLimit',
+ name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
type: 'number',
- defaultValue: 500,
- info: <>The maximum number of segments that can be replicated at one
time.</>,
+ defaultValue: 70,
+ defined: cdc => (cdc.smartSegmentLoading === false ? true : undefined),
+ info: (
+ <>
+ <p>
+ Upper limit of segments the Coordinator can move from
decommissioning servers to active
+ non-decommissioning servers during a single run. This value is
relative to the total
+ maximum number of segments that can be moved at any given time based
upon the value of
+ <Code>maxSegmentsToMove</Code>.
+ </p>
+ <p>
+ If <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0,
the Coordinator does
+ not move segments to decommissioning servers, effectively putting
them in a type of
+ "maintenance" mode. In this case, decommissioning servers
do not participate in
+ balancing or assignment by load rules. The Coordinator still
considers segments on
+ decommissioning servers as candidates to replicate on active servers.
+ </p>
+ <p>
+ Decommissioning can stall if there are no available active servers
to move the segments
+ to. You can use the maximum percent of decommissioning segment
movements to prioritize
+ balancing or to decrease commissioning time to prevent active
servers from being
+ overloaded. The value must be between 0 and 100.
+ </p>
+ </>
+ ),
},
+
+ // End "smart" segment loading section
+
{
name: 'decommissioningNodes',
type: 'string-array',
@@ -152,72 +205,76 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS:
Field<CoordinatorDynamicConfig>[
),
},
{
- name: 'decommissioningMaxPercentOfMaxSegmentsToMove',
- type: 'number',
- defaultValue: 70,
+ name: 'killDataSourceWhitelist',
+ label: 'Kill datasource whitelist',
+ type: 'string-array',
+ emptyValue: [],
info: (
<>
- The maximum number of segments that may be moved away from
'decommissioning'
- services to non-decommissioning (that is, active) services during one
Coordinator run. This
- value is relative to the total maximum segment movements allowed
during one run which is
- determined by <Code>maxSegmentsToMove</Code>. If
- <Code>decommissioningMaxPercentOfMaxSegmentsToMove</Code> is 0,
segments will neither be
- moved from or to 'decommissioning' services, effectively
putting them in a sort of
- "maintenance" mode that will not participate in balancing or
assignment by load
- rules. Decommissioning can also become stalled if there are no
available active services to
- place the segments. By leveraging the maximum percent of
decommissioning segment movements,
- an operator can prevent active services from overload by prioritizing
balancing, or decrease
- decommissioning time instead. The value should be between 0 and 100.
+ List of dataSources for which kill tasks are sent if property{' '}
+ <Code>druid.coordinator.kill.on</Code> is true. This can be a list of
comma-separated
+ dataSources or a JSON array.
</>
),
},
{
- name: 'useRoundRobinSegmentAssignment',
- type: 'boolean',
- defaultValue: true,
+ name: 'killPendingSegmentsSkipList',
+ type: 'string-array',
+ emptyValue: [],
info: (
<>
- Boolean flag for whether segments should be assigned to historicals in
a round-robin
- fashion. If enabled, this can speed up initial segment loading leaving
segment balancing to
- make cost-based decisions and find the optimal location of a segment.
+ List of dataSources for which pendingSegments are NOT cleaned up if
property{' '}
+ <Code>druid.coordinator.kill.pendingSegments.on</Code> is true. This
can be a list of
+ comma-separated dataSources or a JSON array.
</>
),
},
{
- name: 'pauseCoordination',
- type: 'boolean',
- defaultValue: false,
+ name: 'balancerComputeThreads',
+ type: 'number',
+ defaultValue: 1,
info: (
<>
- Boolean flag for whether or not the coordinator should execute its
various duties of
- coordinating the cluster. Setting this to true essentially pauses all
coordination work
- while allowing the API to remain up.
+ Thread pool size for computing moving cost of segments during segment
balancing. Consider
+ increasing this if you have a lot of segments and moving segments
begins to stall.
</>
),
},
{
- name: 'replicateAfterLoadTimeout',
- type: 'boolean',
- defaultValue: false,
+ name: 'mergeBytesLimit',
+ type: 'size-bytes',
+ defaultValue: 524288000,
+ info: <>The maximum total uncompressed size in bytes of segments to
merge.</>,
+ },
+ {
+ name: 'mergeSegmentsLimit',
+ type: 'number',
+ defaultValue: 100,
+ info: <>The maximum number of segments that can be in a single append
task.</>,
+ },
+ {
+ name: 'millisToWaitBeforeDeleting',
+ type: 'number',
+ defaultValue: 900000,
info: (
<>
- Boolean flag for whether or not additional replication is needed for
segments that have
- failed to load due to the expiry of coordinator load timeout. If this
is set to true, the
- coordinator will attempt to replicate the failed segment on a
different historical server.
+ How long does the Coordinator need to be a leader before it can start
marking overshadowed
+ segments as unused in metadata storage.
</>
),
},
{
- name: 'maxNonPrimaryReplicantsToLoad',
- type: 'number',
- defaultValue: 2147483647,
+ name: 'replicateAfterLoadTimeout',
+ type: 'boolean',
+ defaultValue: false,
info: (
<>
- The maximum number of non-primary replicants to load in a single
Coordinator cycle. Once
- this limit is hit, only primary replicants will be loaded for the
remainder of the cycle.
- Tuning this value lower can help reduce the delay in loading primary
segments when the
- cluster has a very large number of non-primary replicants to load
(such as when a single
- historical drops out of the cluster leaving many under-replicated
segments).
+ Boolean flag for whether or not additional replication is needed for
segments that have
+ failed to load due to the expiry of
<Code>druid.coordinator.load.timeout</Code>. If this is
+ set to true, the coordinator will attempt to replicate the failed
segment on a different
+ historical server. This helps improve the segment availability if
there are a few slow
+ historicals in the cluster. However, the slow historical may still
load the segment later
+ and the coordinator may issue drop requests if the segment is
over-replicated.
</>
),
},
diff --git a/web-console/src/druid-models/mocks.ts
b/web-console/src/druid-models/mocks.ts
index 38c924033d..061aadbd8a 100644
--- a/web-console/src/druid-models/mocks.ts
+++ b/web-console/src/druid-models/mocks.ts
@@ -17,6 +17,7 @@
*/
export * from './async-query/async-query.mock';
+export * from './coordinator-dynamic-config/coordinator-dynamic-config.mock';
export * from './execution/execution-ingest-complete.mock';
export * from './execution/execution-ingest-error.mock';
export * from './stages/stages.mock';
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
