This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 85b8cf9f378 Web console: Fix concurrent tasks (#15649)
85b8cf9f378 is described below
commit 85b8cf9f3784bc7fc33fe6c08d68c1b6f0a44351
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Tue Jan 9 16:09:42 2024 -0800
Web console: Fix concurrent tasks (#15649)
* Improve handling of concurrent tasks option
* Update snapshots
---
web-console/src/components/auto-form/auto-form.tsx | 10 +-
.../compaction-config-dialog.spec.tsx.snap | 2640 ++++++++++----------
.../compaction-config-dialog.tsx | 27 +-
.../compaction-config/compaction-config.tsx | 9 -
.../druid-models/ingestion-spec/ingestion-spec.tsx | 5 +-
.../src/views/load-data-view/load-data-view.tsx | 47 +-
6 files changed, 1356 insertions(+), 1382 deletions(-)
diff --git a/web-console/src/components/auto-form/auto-form.tsx
b/web-console/src/components/auto-form/auto-form.tsx
index c63f7a7bc50..357046876af 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -72,10 +72,6 @@ export interface Field<M> {
hide?: Functor<M, boolean>;
hideInMore?: Functor<M, boolean>;
valueAdjustment?: (value: any) => any;
- /**
- * An optional callback to transform the value before it is set on the input
- */
- adjustValue?: (value: any) => any;
adjustment?: (model: Partial<M>, oldModel: Partial<M>) => Partial<M>;
issueWithValue?: (value: any) => string | undefined;
@@ -382,14 +378,12 @@ export class AutoForm<T extends Record<string, any>>
extends React.PureComponent
const disabled = AutoForm.evaluateFunctor(field.disabled, model, false);
const intent = required && modelValue == null ? AutoForm.REQUIRED_INTENT :
undefined;
- const adjustedValue = field.adjustValue ? field.adjustValue(shownValue) :
shownValue;
-
return (
<ButtonGroup large={large}>
<Button
intent={intent}
disabled={disabled}
- active={adjustedValue === false}
+ active={shownValue === false}
onClick={() => {
this.fieldChange(field, false);
if (onFinalize) onFinalize();
@@ -400,7 +394,7 @@ export class AutoForm<T extends Record<string, any>>
extends React.PureComponent
<Button
intent={intent}
disabled={disabled}
- active={adjustedValue === true}
+ active={shownValue === true}
onClick={() => {
this.fieldChange(field, true);
if (onFinalize) onFinalize();
diff --git
a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
index 11949933699..7bb67d84f97 100644
---
a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
+++
b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
@@ -15,351 +15,347 @@ exports[`CompactionConfigDialog matches snapshot with
compactionConfig (dynamic
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you
should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you
should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up
the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and
find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all
dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed
up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions
and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all
dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same
time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a
single task. If a single segment is larger than this number, it will be
processed by itself in a single task (input segments are never split across
tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single
subtask. This limit is to avoid task failures when the ingestion spec is too
long. There are two known limits on the max size of serialized ingestion spec,
i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single
task. If a single segment is larger than this number, it will be processed by
itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask.
This limit is to avoid task failures when the ingestion spec is too long. There
are two known limits on the max size of serialized ingestion spec, i.e., the
max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "dynamic",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "dynamic",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -408,351 +404,347 @@ exports[`CompactionConfigDialog matches snapshot with
compactionConfig (hashed p
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you
should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you
should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up
the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and
find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all
dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed
up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions
and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all
dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same
time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a
single task. If a single segment is larger than this number, it will be
processed by itself in a single task (input segments are never split across
tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single
subtask. This limit is to avoid task failures when the ingestion spec is too
long. There are two known limits on the max size of serialized ingestion spec,
i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single
task. If a single segment is larger than this number, it will be processed by
itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask.
This limit is to avoid task failures when the ingestion spec is too long. There
are two known limits on the max size of serialized ingestion spec, i.e., the
max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "hashed",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "hashed",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -801,351 +793,347 @@ exports[`CompactionConfigDialog matches snapshot with
compactionConfig (range pa
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you
should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you
should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up
the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and
find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all
dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed
up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions
and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all
dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same
time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a
single task. If a single segment is larger than this number, it will be
processed by itself in a single task (input segments are never split across
tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single
subtask. This limit is to avoid task failures when the ingestion spec is too
long. There are two known limits on the max size of serialized ingestion spec,
i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single
task. If a single segment is larger than this number, it will be processed by
itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask.
This limit is to avoid task failures when the ingestion spec is too long. There
are two known limits on the max size of serialized ingestion spec, i.e., the
max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "range",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "range",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -1194,351 +1182,347 @@ exports[`CompactionConfigDialog matches snapshot
without compactionConfig 1`] =
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you
should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly
recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you
should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up
the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and
find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all
dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the
requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will
have a row count close to the target assuming evenly distributed keys. Defaults
to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best
partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed
up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is
specified and 'intervals' is specified in the granularitySpec, the index task
can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions
and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all
dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a
number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to
show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same
time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a
single task. If a single segment is larger than this number, it will be
processed by itself in a single task (input segments are never split across
tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single
subtask. This limit is to avoid task failures when the ingestion spec is too
long. There are two known limits on the max size of serialized ingestion spec,
i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to
show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and
dimensions. Ingestion will run faster, but may choose sub-optimal partitions if
this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The
supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks
regardless of the available task slots. If this value is set to 1, the
supervisor task processes data ingestion on its own instead of spawning worker
tasks. If this value is set to too large, too many worker tasks can be created
which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase
when merging segments for publishing. This limit affects the total number of
columns present in a set of segments to merge. If the limit is exceeded,
segment merging occurs in multiple phases. Druid merges at least 2 segments per
phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single
task. If a single segment is larger than this number, it will be processed by
itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask.
This limit is to avoid task failures when the ingestion spec is too long. There
are two known limits on the max size of serialized ingestion spec, i.e., the
max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized
ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "dynamic",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "dynamic",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
diff --git
a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
index 8a3a232072e..ec2135e43aa 100644
---
a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
+++
b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Classes, Code, Dialog, Intent } from
'@blueprintjs/core';
+import { Button, Callout, Classes, Code, Dialog, Intent, Switch } from
'@blueprintjs/core';
import React, { useState } from 'react';
import type { FormJsonTabs } from '../../components';
@@ -26,7 +26,7 @@ import {
COMPACTION_CONFIG_FIELDS,
compactionConfigHasLegacyInputSegmentSizeBytesSet,
} from '../../druid-models';
-import { deepDelete, formatBytesCompact } from '../../utils';
+import { deepDelete, deepGet, deepSet, formatBytesCompact } from '../../utils';
import { CompactionHistoryDialog } from
'../compaction-history-dialog/compaction-history-dialog';
import './compaction-config-dialog.scss';
@@ -96,11 +96,24 @@ export const CompactionConfigDialog = React.memo(function
CompactionConfigDialog
/>
<div className="content">
{currentTab === 'form' ? (
- <AutoForm
- fields={COMPACTION_CONFIG_FIELDS}
- model={currentConfig}
- onChange={m => setCurrentConfig(m as CompactionConfig)}
- />
+ <>
+ <AutoForm
+ fields={COMPACTION_CONFIG_FIELDS}
+ model={currentConfig}
+ onChange={m => setCurrentConfig(m as CompactionConfig)}
+ />
+ <Switch
+ label="Allow concurrent compactions (experimental)"
+ checked={typeof deepGet(currentConfig,
'taskContext.taskLockType') === 'string'}
+ onChange={() => {
+ setCurrentConfig(
+ (typeof deepGet(currentConfig, 'taskContext.taskLockType')
=== 'string'
+ ? deepDelete(currentConfig, 'taskContext.taskLockType')
+ : deepSet(currentConfig, 'taskContext.taskLockType',
'REPLACE')) as any,
+ );
+ }}
+ />
+ </>
) : (
<JsonInput
value={currentConfig}
diff --git
a/web-console/src/druid-models/compaction-config/compaction-config.tsx
b/web-console/src/druid-models/compaction-config/compaction-config.tsx
index a074af7ae14..8c03b6a5786 100644
--- a/web-console/src/druid-models/compaction-config/compaction-config.tsx
+++ b/web-console/src/druid-models/compaction-config/compaction-config.tsx
@@ -354,13 +354,4 @@ export const COMPACTION_CONFIG_FIELDS:
Field<CompactionConfig>[] = [
</>
),
},
- {
- name: 'taskContext.taskLockType',
- type: 'boolean',
- label: 'Allow concurrent compactions (experimental)',
- defaultValue: undefined,
- valueAdjustment: v => (v ? 'REPLACE' : undefined),
- adjustValue: v => v === 'REPLACE',
- info: <p>Allows or forbids concurrent compactions.</p>,
- },
];
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 7dd6bd4e82f..3594f70c992 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -347,7 +347,10 @@ export function normalizeSpec(spec:
Partial<IngestionSpec>): IngestionSpec {
spec = deepSetIfUnset(spec, 'spec.tuningConfig.type', specType);
if (spec.context?.taskLockType !== undefined) {
- spec.context.taskLockType = spec.spec?.ioConfig.appendToExisting ?
'APPEND' : 'REPLACE';
+ spec.context.taskLockType =
+ isStreamingSpec(spec) || deepGet(spec, 'spec.ioConfig.appendToExisting')
+ ? 'APPEND'
+ : 'REPLACE';
}
return spec as IngestionSpec;
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index af39c2d0385..db7e8e0f093 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -3134,8 +3134,6 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
const { spec } = this.state;
const parallel = deepGet(spec,
'spec.tuningConfig.maxNumConcurrentSubTasks') > 1;
- const appendToExisting = spec.spec?.ioConfig.appendToExisting;
-
return (
<>
<div className="main">
@@ -3161,6 +3159,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
name: 'spec.ioConfig.appendToExisting',
label: 'Append to existing',
type: 'boolean',
+ defined: s => !isStreamingSpec(s),
defaultValue: false,
// appendToExisting can only be set on 'dynamic' portioning.
// We chose to show it always and instead have a specific
message, separate from this form, to notify the user of the issue.
@@ -3171,37 +3170,27 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
</>
),
},
- {
- name: 'context.taskLockType',
- type: 'boolean',
- label: `Allow concurrent ${
- appendToExisting ? 'append' : 'replace'
- } tasks (experimental)`,
- defaultValue: undefined,
- valueAdjustment: v => {
- if (!v) return undefined;
-
- if (isStreamingSpec(spec)) {
- return 'APPEND';
- } else {
- return appendToExisting ? 'APPEND' : 'REPLACE';
- }
- },
- adjustValue: v => {
- if (v === undefined) return false;
-
- if (isStreamingSpec(spec)) {
- return v === 'APPEND';
- }
-
- return v === (appendToExisting ? 'APPEND' : 'REPLACE');
- },
- info: <p>Allows or forbids concurrent tasks.</p>,
- },
]}
model={spec}
onChange={this.updateSpec}
/>
+ <Switch
+ label="Allow concurrent tasks (experimental)"
+ checked={typeof deepGet(spec, 'context.taskLockType') === 'string'}
+ onChange={() => {
+ this.updateSpec(
+ typeof deepGet(spec, 'context.taskLockType') === 'string'
+ ? deepDelete(spec, 'context.taskLockType')
+ : deepSet(
+ spec,
+ 'context.taskLockType',
+ isStreamingSpec(spec) || deepGet(spec,
'spec.ioConfig.appendToExisting')
+ ? 'APPEND'
+ : 'REPLACE',
+ ),
+ );
+ }}
+ />
</div>
<div className="other">
<H5>Parse error reporting</H5>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
