This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch 25.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/25.0.0 by this push:
new 348c9f64f9 [Backport] backport ux bug fixes to 25 (#13533)
348c9f64f9 is described below
commit 348c9f64f9ca12803b8dcefe596d1f9e58db41ca
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Fri Dec 9 19:03:55 2022 -0800
[Backport] backport ux bug fixes to 25 (#13533)
* Web console: add arrayOfDoublesSketch and other small fixes (#13486)
* add padding and keywords
* add arrayOfDoubles
* Update docs/development/extensions-core/datasketches-tuple.md
Co-authored-by: Charles Smith <[email protected]>
* Update docs/development/extensions-core/datasketches-tuple.md
Co-authored-by: Charles Smith <[email protected]>
* Update docs/development/extensions-core/datasketches-tuple.md
Co-authored-by: Charles Smith <[email protected]>
* Update docs/development/extensions-core/datasketches-tuple.md
Co-authored-by: Charles Smith <[email protected]>
* Update docs/development/extensions-core/datasketches-tuple.md
Co-authored-by: Charles Smith <[email protected]>
* partiton int
* fix docs
Co-authored-by: Charles Smith <[email protected]>
* Web console: improve compaction status display (#13523)
* improve compaction status display
* even more accurate
* fix snapshot
* MSQ: Improve TooManyBuckets error message, improve error docs. (#13525)
1) Edited the TooManyBuckets error message to mention PARTITIONED BY
instead of segmentGranularity.
2) Added error-code-specific anchors in the docs.
3) Add information to various error codes in the docs about common
causes and solutions.
* update error anchors (#13527)
* update snapshot
Co-authored-by: Charles Smith <[email protected]>
Co-authored-by: Gian Merlino <[email protected]>
---
.../extensions-core/datasketches-tuple.md | 43 +++++-
docs/multi-stage-query/concepts.md | 3 +-
docs/multi-stage-query/known-issues.md | 8 +-
docs/multi-stage-query/reference.md | 70 +++++-----
.../msq/indexing/error/TooManyBucketsFault.java | 2 +-
web-console/lib/keywords.js | 3 +
web-console/script/create-sql-docs.js | 4 +-
web-console/src/bootstrap/react-table-defaults.tsx | 4 +-
.../segment-timeline/segment-timeline.tsx | 2 +-
.../compaction-dialog/compaction-dialog.scss | 5 +
.../compaction-dialog/compaction-dialog.tsx | 27 +++-
.../compaction-config/compaction-config.tsx | 29 +++-
.../compaction-status/compaction-status.spec.ts | 146 +++++++++++++++------
.../compaction-status/compaction-status.ts | 23 +++-
.../ingest-query-pattern/ingest-query-pattern.ts | 3 +-
.../src/druid-models/metric-spec/metric-spec.tsx | 43 ++++++
.../workbench-query/workbench-query.ts | 8 +-
.../src/helpers/execution/sql-task-execution.ts | 9 +-
web-console/src/links.ts | 2 +-
web-console/src/react-table/react-table-extra.scss | 4 +
.../__snapshots__/datasources-view.spec.tsx.snap | 2 +-
.../views/datasources-view/datasources-view.tsx | 2 +-
.../sql-data-loader-view/sql-data-loader-view.tsx | 2 +
.../connect-external-data-dialog.tsx | 18 ++-
.../execution-error-pane.spec.tsx.snap | 2 +-
.../execution-error-pane/execution-error-pane.tsx | 4 +-
.../input-source-step/example-inputs.ts | 3 +
.../input-source-step/input-source-step.tsx | 12 +-
.../src/views/workbench-view/workbench-view.tsx | 9 +-
29 files changed, 372 insertions(+), 120 deletions(-)
diff --git a/docs/development/extensions-core/datasketches-tuple.md
b/docs/development/extensions-core/datasketches-tuple.md
index fc4f74d5c8..c9a05b5ab1 100644
--- a/docs/development/extensions-core/datasketches-tuple.md
+++ b/docs/development/extensions-core/datasketches-tuple.md
@@ -39,19 +39,52 @@ druid.extensions.loadList=["druid-datasketches"]
"name" : <output_name>,
"fieldName" : <metric_name>,
"nominalEntries": <number>,
- "numberOfValues" : <number>,
- "metricColumns" : <array of strings>
+ "metricColumns" : <array of strings>,
+ "numberOfValues" : <number>
}
```
|property|description|required?|
|--------|-----------|---------|
|type|This String should always be "arrayOfDoublesSketch"|yes|
-|name|A String for the output (result) name of the calculation.|yes|
+|name|String representing the output column to store sketch values.|yes|
|fieldName|A String for the name of the input field.|yes|
|nominalEntries|Parameter that determines the accuracy and size of the sketch.
Higher k means higher accuracy but more space to store sketches. Must be a
power of 2. See the [Theta sketch
accuracy](https://datasketches.apache.org/docs/Theta/ThetaErrorTable) for
details. |no, defaults to 16384|
-|numberOfValues|Number of values associated with each distinct key. |no,
defaults to 1|
-|metricColumns|If building sketches from raw data, an array of names of the
input columns containing numeric values to be associated with each distinct
key.|no, defaults to empty array|
+|metricColumns|When building sketches from raw data, an array input column
that contain numeric values to associate with each distinct key. If not
provided, assumes `fieldName` is an `arrayOfDoublesSketch`|no, if not provided
`fieldName` is assumed to be an arrayOfDoublesSketch|
+|numberOfValues|Number of values associated with each distinct key. |no,
defaults to the length of `metricColumns` if provided and 1 otherwise|
+
+You can use the `arrayOfDoublesSketch` aggregator to:
+
+- Build a sketch from raw data. In this case, set `metricColumns` to an array.
+- Build a sketch from an existing `ArrayOfDoubles` sketch . In this case,
leave `metricColumns` unset and set the `fieldName` to an `ArrayOfDoubles`
sketch with `numberOfValues` doubles. At ingestion time, you must base64 encode
`ArrayOfDoubles` sketches at ingestion time.
+
+#### Example on top of raw data
+
+Compute a theta of unique users. For each user store the `added` and `deleted`
scores. The new sketch column will be called `users_theta`.
+
+```json
+{
+ "type": "arrayOfDoublesSketch",
+ "name": "users_theta",
+ "fieldName": "user",
+ "nominalEntries": 16384,
+ "metricColumns": ["added", "deleted"],
+}
+```
+
+#### Example ingesting a precomputed sketch column
+
+Ingest a sketch column called `user_sketches` that has a base64 encoded value
of two doubles in its array and store it in a column called `users_theta`.
+
+```json
+{
+ "type": "arrayOfDoublesSketch",
+ "name": "users_theta",
+ "fieldName": "user_sketches",
+ "nominalEntries": 16384,
+ "numberOfValues": 2,
+}
+```
### Post Aggregators
diff --git a/docs/multi-stage-query/concepts.md
b/docs/multi-stage-query/concepts.md
index 44e5ea43d4..da0e774152 100644
--- a/docs/multi-stage-query/concepts.md
+++ b/docs/multi-stage-query/concepts.md
@@ -233,7 +233,8 @@ happens:
The [`maxNumTasks`](./reference.md#context-parameters) query parameter
determines the maximum number of tasks your
query will use, including the one `query_controller` task. Generally, queries
perform better with more workers. The
lowest possible value of `maxNumTasks` is two (one worker and one controller).
Do not set this higher than the number of
-free slots available in your cluster; doing so will result in a
[TaskStartTimeout](reference.md#error-codes) error.
+free slots available in your cluster; doing so will result in a
[TaskStartTimeout](reference.md#error_TaskStartTimeout)
+error.
When [reading external data](#extern), EXTERN can read multiple files in
parallel across
different worker tasks. However, EXTERN does not split individual files across
multiple worker tasks. If you have a
diff --git a/docs/multi-stage-query/known-issues.md
b/docs/multi-stage-query/known-issues.md
index c76ab57aa7..648d3c297b 100644
--- a/docs/multi-stage-query/known-issues.md
+++ b/docs/multi-stage-query/known-issues.md
@@ -33,16 +33,18 @@ sidebar_label: Known issues
- Worker task stage outputs are stored in the working directory given by
`druid.indexer.task.baseDir`. Stages that
generate a large amount of output data may exhaust all available disk space.
In this case, the query fails with
-an [UnknownError](./reference.md#error-codes) with a message including "No
space left on device".
+an [UnknownError](./reference.md#error_UnknownError) with a message including
"No space left on device".
## SELECT
- SELECT from a Druid datasource does not include unpublished real-time data.
- GROUPING SETS and UNION ALL are not implemented. Queries using these
features return a
- [QueryNotSupported](reference.md#error-codes) error.
+ [QueryNotSupported](reference.md#error_QueryNotSupported) error.
-- For some COUNT DISTINCT queries, you'll encounter a
[QueryNotSupported](reference.md#error-codes) error that includes `Must not
have 'subtotalsSpec'` as one of its causes. This is caused by the planner
attempting to use GROUPING SETs, which are not implemented.
+- For some COUNT DISTINCT queries, you'll encounter a
[QueryNotSupported](reference.md#error_QueryNotSupported) error
+ that includes `Must not have 'subtotalsSpec'` as one of its causes. This is
caused by the planner attempting to use
+ GROUPING SETs, which are not implemented.
- The numeric varieties of the EARLIEST and LATEST aggregators do not work
properly. Attempting to use the numeric
varieties of these aggregators lead to an error like
diff --git a/docs/multi-stage-query/reference.md
b/docs/multi-stage-query/reference.md
index a4bcbfc27b..ae9bc106ca 100644
--- a/docs/multi-stage-query/reference.md
+++ b/docs/multi-stage-query/reference.md
@@ -249,14 +249,14 @@ The following table lists query limits:
| Limit | Value | Error if exceeded |
|---|---|---|
-| Size of an individual row written to a frame. Row size when written to a
frame may differ from the original row size. | 1 MB | `RowTooLarge` |
-| Number of segment-granular time chunks encountered during ingestion. | 5,000
| `TooManyBuckets` |
-| Number of input files/segments per worker. | 10,000 | `TooManyInputFiles` |
-| Number of output partitions for any one stage. Number of segments generated
during ingestion. |25,000 | `TooManyPartitions` |
-| Number of output columns for any one stage. | 2,000 | `TooManyColumns` |
-| Number of cluster by columns that can appear in a stage | 1,500 |
`TooManyClusteredByColumns` |
-| Number of workers for any one stage. | Hard limit is 1,000. Memory-dependent
soft limit may be lower. | `TooManyWorkers` |
-| Maximum memory occupied by broadcasted tables. | 30% of each [processor
memory bundle](concepts.md#memory-usage). | `BroadcastTablesTooLarge` |
+| Size of an individual row written to a frame. Row size when written to a
frame may differ from the original row size. | 1 MB |
[`RowTooLarge`](#error_RowTooLarge) |
+| Number of segment-granular time chunks encountered during ingestion. | 5,000
| [`TooManyBuckets`](#error_TooManyBuckets) |
+| Number of input files/segments per worker. | 10,000 |
[`TooManyInputFiles`](#error_TooManyInputFiles) |
+| Number of output partitions for any one stage. Number of segments generated
during ingestion. |25,000 | [`TooManyPartitions`](#error_TooManyPartitions) |
+| Number of output columns for any one stage. | 2,000 |
[`TooManyColumns`](#error_TooManyColumns) |
+| Number of cluster by columns that can appear in a stage | 1,500 |
[`TooManyClusteredByColumns`](#error_TooManyClusteredByColumns) |
+| Number of workers for any one stage. | Hard limit is 1,000. Memory-dependent
soft limit may be lower. | [`TooManyWorkers`](#error_TooManyWorkers) |
+| Maximum memory occupied by broadcasted tables. | 30% of each [processor
memory bundle](concepts.md#memory-usage). |
[`BroadcastTablesTooLarge`](#error_BroadcastTablesTooLarge) |
<a name="errors"></a>
@@ -266,30 +266,30 @@ The following table describes error codes you may
encounter in the `multiStageQu
| Code | Meaning | Additional fields |
|---|---|---|
-| `BroadcastTablesTooLarge` | The size of the broadcast tables used in the
right hand side of the join exceeded the memory reserved for them in a worker
task.<br /><br />Try increasing the peon memory or reducing the size of the
broadcast tables. | `maxBroadcastTablesSize`: Memory reserved for the broadcast
tables, measured in bytes. |
-| `Canceled` | The query was canceled. Common reasons for cancellation:<br
/><br /><ul><li>User-initiated shutdown of the controller task via the
`/druid/indexer/v1/task/{taskId}/shutdown` API.</li><li>Restart or failure of
the server process that was running the controller task.</li></ul>| |
-| `CannotParseExternalData` | A worker task could not parse data from an
external datasource. | `errorMessage`: More details on why parsing failed. |
-| `ColumnNameRestricted` | The query uses a restricted column name. |
`columnName`: The restricted column name. |
-| `ColumnTypeNotSupported` | The column type is not supported. This can be
because:<br /> <br /><ul><li>Support for writing or reading from a particular
column type is not supported.</li><li>The query attempted to use a column type
that is not supported by the frame format. This occurs with ARRAY types, which
are not yet implemented for frames.</li></ul> | `columnName`: The column name
with an unsupported type.<br /> <br />`columnType`: The unknown column type. |
-| `InsertCannotAllocateSegment` | The controller task could not allocate a new
segment ID due to conflict with existing segments or pending segments. Common
reasons for such conflicts:<br /> <br /><ul><li>Attempting to mix different
granularities in the same intervals of the same datasource.</li><li>Prior
ingestions that used non-extendable shard specs.</li></ul>| `dataSource`<br />
<br />`interval`: The interval for the attempted new segment allocation. |
-| `InsertCannotBeEmpty` | An INSERT or REPLACE query did not generate any
output rows in a situation where output rows are required for success. This can
happen for INSERT or REPLACE queries with `PARTITIONED BY` set to something
other than `ALL` or `ALL TIME`. | `dataSource` |
-| `InsertCannotOrderByDescending` | An INSERT query contained a `CLUSTERED BY`
expression in descending order. Druid's segment generation code only supports
ascending order. | `columnName` |
-| `InsertCannotReplaceExistingSegment` | A REPLACE query cannot proceed
because an existing segment partially overlaps those bounds, and the portion
within the bounds is not fully overshadowed by query results. <br /> <br
/>There are two ways to address this without modifying your
query:<ul><li>Shrink the OVERLAP filter to match the query
results.</li><li>Expand the OVERLAP filter to fully contain the existing
segment.</li></ul>| `segmentId`: The existing segment <br />
-| `InsertLockPreempted` | An INSERT or REPLACE query was canceled by a
higher-priority ingestion job, such as a real-time ingestion task. | |
-| `InsertTimeNull` | An INSERT or REPLACE query encountered a null timestamp
in the `__time` field.<br /><br />This can happen due to using an expression
like `TIME_PARSE(timestamp) AS __time` with a timestamp that cannot be parsed.
(TIME_PARSE returns null when it cannot parse a timestamp.) In this case, try
parsing your timestamps using a different function or pattern.<br /><br />If
your timestamps may genuinely be null, consider using COALESCE to provide a
default value. One option is [...]
-| `InsertTimeOutOfBounds` | A REPLACE query generated a timestamp outside the
bounds of the TIMESTAMP parameter for your OVERWRITE WHERE clause.<br /> <br
/>To avoid this error, verify that the you specified is valid. | `interval`:
time chunk interval corresponding to the out-of-bounds timestamp |
-| `InvalidNullByte` | A string column included a null byte. Null bytes in
strings are not permitted. | `column`: The column that included the null byte |
-| `QueryNotSupported` | QueryKit could not translate the provided native query
to a multi-stage query.<br /> <br />This can happen if the query uses features
that aren't supported, like GROUPING SETS. | |
-| `RowTooLarge` | The query tried to process a row that was too large to write
to a single frame. See the [Limits](#limits) table for the specific limit on
frame size. Note that the effective maximum row size is smaller than the
maximum frame size due to alignment considerations during frame writing. |
`maxFrameSize`: The limit on the frame size. |
-| `TaskStartTimeout` | Unable to launch all the worker tasks in time. <br />
<br />There might be insufficient available slots to start all the worker tasks
simultaneously.<br /> <br /> Try splitting up the query into smaller chunks
with lesser `maxNumTasks` number. Another option is to increase capacity. |
`numTasks`: The number of tasks attempted to launch. |
-| `TooManyBuckets` | Exceeded the number of partition buckets for a stage.
Partition buckets are only used for `segmentGranularity` during INSERT queries.
The most common reason for this error is that your `segmentGranularity` is too
narrow relative to the data. See the [Limits](#limits) table for the specific
limit. | `maxBuckets`: The limit on buckets. |
-| `TooManyInputFiles` | Exceeded the number of input files/segments per
worker. See the [Limits](#limits) table for the specific limit. |
`numInputFiles`: The total number of input files/segments for the stage.<br
/><br />`maxInputFiles`: The maximum number of input files/segments per worker
per stage.<br /><br />`minNumWorker`: The minimum number of workers required
for a successful run. |
-| `TooManyPartitions` | Exceeded the number of partitions for a stage. The
most common reason for this is that the final stage of an INSERT or REPLACE
query generated too many segments. See the [Limits](#limits) table for the
specific limit. | `maxPartitions`: The limit on partitions which was exceeded |
-| `TooManyClusteredByColumns` | Exceeded the number of cluster by columns
for a stage. See the [Limits](#limits) table for the specific limit. |
`numColumns`: The number of columns requested.<br /><br />`maxColumns`: The
limit on columns which was exceeded.`stage`: The stage number exceeding the
limit<br /><br /> |
-| `TooManyColumns` | Exceeded the number of columns for a stage. See the
[Limits](#limits) table for the specific limit. | `numColumns`: The number of
columns requested.<br /><br />`maxColumns`: The limit on columns which was
exceeded. |
-| `TooManyWarnings` | Exceeded the allowed number of warnings of a particular
type. | `rootErrorCode`: The error code corresponding to the exception that
exceeded the required limit. <br /><br />`maxWarnings`: Maximum number of
warnings that are allowed for the corresponding `rootErrorCode`. |
-| `TooManyWorkers` | Exceeded the supported number of workers running
simultaneously. See the [Limits](#limits) table for the specific limit. |
`workers`: The number of simultaneously running workers that exceeded a hard or
soft limit. This may be larger than the number of workers in any one stage if
multiple stages are running simultaneously. <br /><br />`maxWorkers`: The hard
or soft limit on workers that was exceeded. |
-| `NotEnoughMemory` | Insufficient memory to launch a stage. | `serverMemory`:
The amount of memory available to a single process.<br /><br />`serverWorkers`:
The number of workers running in a single process.<br /><br />`serverThreads`:
The number of threads in a single process. |
-| `WorkerFailed` | A worker task failed unexpectedly. | `errorMsg`<br /><br
/>`workerTaskId`: The ID of the worker task. |
-| `WorkerRpcFailed` | A remote procedure call to a worker task failed and
could not recover. | `workerTaskId`: the id of the worker task |
-| `UnknownError` | All other errors. | `message` |
+| <a name="error_BroadcastTablesTooLarge">`BroadcastTablesTooLarge`</a> | The
size of the broadcast tables used in the right hand side of the join exceeded
the memory reserved for them in a worker task.<br /><br />Try increasing the
peon memory or reducing the size of the broadcast tables. |
`maxBroadcastTablesSize`: Memory reserved for the broadcast tables, measured in
bytes. |
+| <a name="error_Canceled">`Canceled`</a> | The query was canceled. Common
reasons for cancellation:<br /><br /><ul><li>User-initiated shutdown of the
controller task via the `/druid/indexer/v1/task/{taskId}/shutdown`
API.</li><li>Restart or failure of the server process that was running the
controller task.</li></ul>| |
+| <a name="error_CannotParseExternalData">`CannotParseExternalData`</a> | A
worker task could not parse data from an external datasource. | `errorMessage`:
More details on why parsing failed. |
+| <a name="error_ColumnNameRestricted">`ColumnNameRestricted`</a> | The query
uses a restricted column name. | `columnName`: The restricted column name. |
+| <a name="error_ColumnTypeNotSupported">`ColumnTypeNotSupported`</a> | The
column type is not supported. This can be because:<br /> <br /><ul><li>Support
for writing or reading from a particular column type is not
supported.</li><li>The query attempted to use a column type that is not
supported by the frame format. This occurs with ARRAY types, which are not yet
implemented for frames.</li></ul> | `columnName`: The column name with an
unsupported type.<br /> <br />`columnType`: The unkn [...]
+| <a
name="error_InsertCannotAllocateSegment">`InsertCannotAllocateSegment`</a> |
The controller task could not allocate a new segment ID due to conflict with
existing segments or pending segments. Common reasons for such conflicts:<br />
<br /><ul><li>Attempting to mix different granularities in the same intervals
of the same datasource.</li><li>Prior ingestions that used non-extendable shard
specs.</li></ul>| `dataSource`<br /> <br />`interval`: The interval for the
attempted new segme [...]
+| <a name="error_InsertCannotBeEmpty">`InsertCannotBeEmpty`</a> | An INSERT or
REPLACE query did not generate any output rows in a situation where output rows
are required for success. This can happen for INSERT or REPLACE queries with
`PARTITIONED BY` set to something other than `ALL` or `ALL TIME`. |
`dataSource` |
+| <a
name="error_InsertCannotOrderByDescending">`InsertCannotOrderByDescending`</a>
| An INSERT query contained a `CLUSTERED BY` expression in descending order.
Druid's segment generation code only supports ascending order. | `columnName` |
+| <a
name="error_InsertCannotReplaceExistingSegment">`InsertCannotReplaceExistingSegment`</a>
| A REPLACE query cannot proceed because an existing segment partially
overlaps those bounds, and the portion within the bounds is not fully
overshadowed by query results. <br /> <br />There are two ways to address this
without modifying your query:<ul><li>Shrink the OVERLAP filter to match the
query results.</li><li>Expand the OVERLAP filter to fully contain the existing
segment.</li></ul>| `se [...]
+| <a name="error_InsertLockPreempted">`InsertLockPreempted`</a> | An INSERT or
REPLACE query was canceled by a higher-priority ingestion job, such as a
real-time ingestion task. | |
+| <a name="error_InsertTimeNull">`InsertTimeNull`</a> | An INSERT or REPLACE
query encountered a null timestamp in the `__time` field.<br /><br />This can
happen due to using an expression like `TIME_PARSE(timestamp) AS __time` with a
timestamp that cannot be parsed. (TIME_PARSE returns null when it cannot parse
a timestamp.) In this case, try parsing your timestamps using a different
function or pattern.<br /><br />If your timestamps may genuinely be null,
consider using COALESCE to pro [...]
+| <a name="error_InsertTimeOutOfBounds">`InsertTimeOutOfBounds`</a> | A
REPLACE query generated a timestamp outside the bounds of the TIMESTAMP
parameter for your OVERWRITE WHERE clause.<br /> <br />To avoid this error,
verify that the you specified is valid. | `interval`: time chunk interval
corresponding to the out-of-bounds timestamp |
+| <a name="error_InvalidNullByte">`InvalidNullByte`</a> | A string column
included a null byte. Null bytes in strings are not permitted. | `column`: The
column that included the null byte |
+| <a name="error_QueryNotSupported">`QueryNotSupported`</a> | QueryKit could
not translate the provided native query to a multi-stage query.<br /> <br
/>This can happen if the query uses features that aren't supported, like
GROUPING SETS. | |
+| <a name="error_RowTooLarge">`RowTooLarge`</a> | The query tried to process a
row that was too large to write to a single frame. See the [Limits](#limits)
table for specific limits on frame size. Note that the effective maximum row
size is smaller than the maximum frame size due to alignment considerations
during frame writing. | `maxFrameSize`: The limit on the frame size. |
+| <a name="error_TaskStartTimeout">`TaskStartTimeout`</a> | Unable to launch
all the worker tasks in time. <br /> <br />There might be insufficient
available slots to start all the worker tasks simultaneously.<br /> <br /> Try
splitting up the query into smaller chunks with lesser `maxNumTasks` number.
Another option is to increase capacity. | `numTasks`: The number of tasks
attempted to launch. |
+| <a name="error_TooManyBuckets">`TooManyBuckets`</a> | Exceeded the maximum
number of partition buckets for a stage (5,000 partition buckets).<br />< br
/>Partition buckets are created for each [`PARTITIONED BY`](#partitioned-by)
time chunk for INSERT and REPLACE queries. The most common reason for this
error is that your `PARTITIONED BY` is too narrow relative to your data. |
`maxBuckets`: The limit on partition buckets. |
+| <a name="error_TooManyInputFiles">`TooManyInputFiles`</a> | Exceeded the
maximum number of input files or segments per worker (10,000 files or
segments).<br /><br />If you encounter this limit, consider adding more
workers, or breaking up your query into smaller queries that process fewer
files or segments per query. | `numInputFiles`: The total number of input
files/segments for the stage.<br /><br />`maxInputFiles`: The maximum number of
input files/segments per worker per stage.<br [...]
+| <a name="error_TooManyPartitions">`TooManyPartitions`</a> | Exceeded the
maximum number of partitions for a stage (25,000 partitions).<br /><br />This
can occur with INSERT or REPLACE statements that generate large numbers of
segments, since each segment is associated with a partition. If you encounter
this limit, consider breaking up your INSERT or REPLACE statement into smaller
statements that process less data per statement. | `maxPartitions`: The limit
on partitions which was exceeded |
+| <a name="error_TooManyClusteredByColumns">`TooManyClusteredByColumns`</a> |
Exceeded the maximum number of clustering columns for a stage (1,500
columns).<br /><br />This can occur with `CLUSTERED BY`, `ORDER BY`, or `GROUP
BY` with a large number of columns. | `numColumns`: The number of columns
requested.<br /><br />`maxColumns`: The limit on columns which was
exceeded.`stage`: The stage number exceeding the limit<br /><br /> |
+| <a name="error_TooManyColumns">`TooManyColumns`</a> | Exceeded the maximum
number of columns for a stage (2,000 columns). | `numColumns`: The number of
columns requested.<br /><br />`maxColumns`: The limit on columns which was
exceeded. |
+| <a name="error_TooManyWarnings">`TooManyWarnings`</a> | Exceeded the maximum
allowed number of warnings of a particular type. | `rootErrorCode`: The error
code corresponding to the exception that exceeded the required limit. <br /><br
/>`maxWarnings`: Maximum number of warnings that are allowed for the
corresponding `rootErrorCode`. |
+| <a name="error_TooManyWorkers">`TooManyWorkers`</a> | Exceeded the maximum
number of simultaneously-running workers. See the [Limits](#limits) table for
more details. | `workers`: The number of simultaneously running workers that
exceeded a hard or soft limit. This may be larger than the number of workers in
any one stage if multiple stages are running simultaneously. <br /><br
/>`maxWorkers`: The hard or soft limit on workers that was exceeded. If this is
lower than the hard limit (1, [...]
+| <a name="error_NotEnoughMemory">`NotEnoughMemory`</a> | Insufficient memory
to launch a stage. | `serverMemory`: The amount of memory available to a single
process.<br /><br />`serverWorkers`: The number of workers running in a single
process.<br /><br />`serverThreads`: The number of threads in a single process.
|
+| <a name="error_WorkerFailed">`WorkerFailed`</a> | A worker task failed
unexpectedly. | `errorMsg`<br /><br />`workerTaskId`: The ID of the worker
task. |
+| <a name="error_WorkerRpcFailed">`WorkerRpcFailed`</a> | A remote procedure
call to a worker task failed and could not recover. | `workerTaskId`: the id of
the worker task |
+| <a name="error_UnknownError">`UnknownError`</a> | All other errors. |
`message` |
diff --git
a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TooManyBucketsFault.java
b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TooManyBucketsFault.java
index 8af20d0919..fdad421e64 100644
---
a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TooManyBucketsFault.java
+++
b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TooManyBucketsFault.java
@@ -41,7 +41,7 @@ public class TooManyBucketsFault extends BaseMSQFault
super(
CODE,
"Too many partition buckets (max = %,d); try breaking your query up
into smaller queries or "
- + "using a wider segmentGranularity",
+ + "using a wider PARTITIONED BY",
maxBuckets
);
this.maxBuckets = maxBuckets;
diff --git a/web-console/lib/keywords.js b/web-console/lib/keywords.js
index e34b2daf45..bc81153dd7 100644
--- a/web-console/lib/keywords.js
+++ b/web-console/lib/keywords.js
@@ -61,6 +61,9 @@ exports.SQL_KEYWORDS = [
'REPLACE INTO',
'OVERWRITE',
'RETURNING',
+ 'OVER',
+ 'PARTITION BY',
+ 'WINDOW',
];
exports.SQL_EXPRESSION_PARTS = [
diff --git a/web-console/script/create-sql-docs.js
b/web-console/script/create-sql-docs.js
index 6af65006f8..13ed438915 100755
--- a/web-console/script/create-sql-docs.js
+++ b/web-console/script/create-sql-docs.js
@@ -52,9 +52,7 @@ function convertMarkdownToHtml(markdown) {
// Concert to markdown
markdown = snarkdown(markdown);
- return markdown
- .replace(/<br \/>/g, '<br /><br />') // Double up the <br>s
- .replace(/<a[^>]*>(.*?)<\/a>/g, '$1'); // Remove links
+ return markdown.replace(/<a[^>]*>(.*?)<\/a>/g, '$1'); // Remove links
}
const readDoc = async () => {
diff --git a/web-console/src/bootstrap/react-table-defaults.tsx
b/web-console/src/bootstrap/react-table-defaults.tsx
index 4c31928064..139a13bcd5 100644
--- a/web-console/src/bootstrap/react-table-defaults.tsx
+++ b/web-console/src/bootstrap/react-table-defaults.tsx
@@ -53,12 +53,12 @@ export function bootstrapReactTable() {
.map((row: any) => row[column.id]);
const previewCount = countBy(previewValues);
return (
- <span>
+ <div className="default-aggregated">
{Object.keys(previewCount)
.sort()
.map(v => `${v} (${previewCount[v]})`)
.join(', ')}
- </span>
+ </div>
);
},
defaultPageSize: 20,
diff --git a/web-console/src/components/segment-timeline/segment-timeline.tsx
b/web-console/src/components/segment-timeline/segment-timeline.tsx
index c138e82dff..f8cef06189 100644
--- a/web-console/src/components/segment-timeline/segment-timeline.tsx
+++ b/web-console/src/components/segment-timeline/segment-timeline.tsx
@@ -278,7 +278,7 @@ ORDER BY "start" DESC`;
intervals = await queryDruidSql({
query: SegmentTimeline.getSqlQuery(startDate, endDate),
});
- datasources = uniq(intervals.map(r => r.datasource));
+ datasources = uniq(intervals.map(r => r.datasource).sort());
} else if (capabilities.hasCoordinatorAccess()) {
const startIso = startDate.toISOString();
diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
index e3ca37b14e..499df985c9 100644
--- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
+++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.scss
@@ -23,6 +23,11 @@
height: 80vh;
}
+ .legacy-callout {
+ width: auto;
+ margin: 10px 15px 0;
+ }
+
.form-json-selector {
margin: 15px;
}
diff --git a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
index d63501b1b0..3b5456e7d0 100644
--- a/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
+++ b/web-console/src/dialogs/compaction-dialog/compaction-dialog.tsx
@@ -16,11 +16,16 @@
* limitations under the License.
*/
-import { Button, Classes, Dialog, Intent } from '@blueprintjs/core';
+import { Button, Callout, Classes, Code, Dialog, Intent } from
'@blueprintjs/core';
import React, { useState } from 'react';
import { AutoForm, FormJsonSelector, FormJsonTabs, JsonInput } from
'../../components';
-import { COMPACTION_CONFIG_FIELDS, CompactionConfig } from
'../../druid-models';
+import {
+ COMPACTION_CONFIG_FIELDS,
+ CompactionConfig,
+ compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../../druid-models';
+import { deepDelete, formatBytesCompact } from '../../utils';
import './compaction-dialog.scss';
@@ -55,13 +60,29 @@ export const CompactionDialog = React.memo(function
CompactionDialog(props: Comp
canOutsideClickClose={false}
title={`Compaction config: ${datasource}`}
>
+ {compactionConfigHasLegacyInputSegmentSizeBytesSet(currentConfig) && (
+ <Callout className="legacy-callout" intent={Intent.WARNING}>
+ <p>
+ You current config sets the legacy
<Code>inputSegmentSizeBytes</Code> to{' '}
+
<Code>{formatBytesCompact(currentConfig.inputSegmentSizeBytes!)}</Code> it is
+ recommended to unset this property.
+ </p>
+ <p>
+ <Button
+ intent={Intent.WARNING}
+ text="Remove legacy setting"
+ onClick={() => setCurrentConfig(deepDelete(currentConfig,
'inputSegmentSizeBytes'))}
+ />
+ </p>
+ </Callout>
+ )}
<FormJsonSelector tab={currentTab} onChange={setCurrentTab} />
<div className="content">
{currentTab === 'form' ? (
<AutoForm
fields={COMPACTION_CONFIG_FIELDS}
model={currentConfig}
- onChange={m => setCurrentConfig(m)}
+ onChange={m => setCurrentConfig(m as CompactionConfig)}
/>
) : (
<JsonInput
diff --git
a/web-console/src/druid-models/compaction-config/compaction-config.tsx
b/web-console/src/druid-models/compaction-config/compaction-config.tsx
index 1958f2b263..b03b66ea0e 100644
--- a/web-console/src/druid-models/compaction-config/compaction-config.tsx
+++ b/web-console/src/druid-models/compaction-config/compaction-config.tsx
@@ -22,7 +22,26 @@ import React from 'react';
import { Field } from '../../components';
import { deepGet, deepSet, oneOf } from '../../utils';
-export type CompactionConfig = Record<string, any>;
+export interface CompactionConfig {
+ dataSource: string;
+ skipOffsetFromLatest?: string;
+ tuningConfig?: any;
+ [key: string]: any;
+
+ // Deprecated:
+ inputSegmentSizeBytes?: number;
+}
+
+export const NOOP_INPUT_SEGMENT_SIZE_BYTES = 100000000000000;
+
+export function compactionConfigHasLegacyInputSegmentSizeBytesSet(
+ config: CompactionConfig,
+): boolean {
+ return (
+ typeof config.inputSegmentSizeBytes === 'number' &&
+ config.inputSegmentSizeBytes < NOOP_INPUT_SEGMENT_SIZE_BYTES
+ );
+}
export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
{
@@ -182,7 +201,7 @@ export const COMPACTION_CONFIG_FIELDS:
Field<CompactionConfig>[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim',
'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
- required: (t: CompactionConfig) =>
+ required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@@ -205,7 +224,7 @@ export const COMPACTION_CONFIG_FIELDS:
Field<CompactionConfig>[] = [
defined: t =>
oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'single_dim',
'range') &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
- required: (t: CompactionConfig) =>
+ required: t =>
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
info: (
@@ -277,7 +296,7 @@ export const COMPACTION_CONFIG_FIELDS:
Field<CompactionConfig>[] = [
defaultValue: 1073741824,
min: 1000000,
hideInMore: true,
- adjustment: (t: CompactionConfig) => deepSet(t,
'tuningConfig.splitHintSpec.type', 'maxSize'),
+ adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of bytes of input segments to process in a single task.
If a single segment
@@ -293,7 +312,7 @@ export const COMPACTION_CONFIG_FIELDS:
Field<CompactionConfig>[] = [
defaultValue: 1000,
min: 1,
hideInMore: true,
- adjustment: (t: CompactionConfig) => deepSet(t,
'tuningConfig.splitHintSpec.type', 'maxSize'),
+ adjustment: t => deepSet(t, 'tuningConfig.splitHintSpec.type', 'maxSize'),
info: (
<>
Maximum number of input segments to process in a single subtask. This
limit is to avoid task
diff --git
a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
index 9d1254090b..abe660e5c2 100644
--- a/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
+++ b/web-console/src/druid-models/compaction-status/compaction-status.spec.ts
@@ -21,7 +21,13 @@ import { CompactionConfig } from
'../compaction-config/compaction-config';
import { CompactionStatus, formatCompactionInfo, zeroCompactionStatus } from
'./compaction-status';
describe('compaction status', () => {
- const BASIC_CONFIG: CompactionConfig = {};
+ const BASIC_CONFIG: CompactionConfig = {
+ dataSource: 'tbl',
+ };
+ const LEGACY_CONFIG: CompactionConfig = {
+ dataSource: 'tbl',
+ inputSegmentSizeBytes: 1e6,
+ };
const ZERO_STATUS: CompactionStatus = {
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
@@ -36,52 +42,112 @@ describe('compaction status', () => {
intervalCountSkipped: 0,
};
- it('zeroCompactionStatus', () => {
- expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
-
- expect(
- zeroCompactionStatus({
- dataSource: 'tbl',
- scheduleStatus: 'RUNNING',
- bytesAwaitingCompaction: 1,
- bytesCompacted: 0,
- bytesSkipped: 0,
- segmentCountAwaitingCompaction: 0,
- segmentCountCompacted: 0,
- segmentCountSkipped: 0,
- intervalCountAwaitingCompaction: 0,
- intervalCountCompacted: 0,
- intervalCountSkipped: 0,
- }),
- ).toEqual(false);
- });
-
- it('formatCompactionConfigAndStatus', () => {
- expect(formatCompactionInfo({})).toEqual('Not enabled');
-
- expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting
first run');
-
- expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not
enabled');
+ describe('zeroCompactionStatus', () => {
+ it('works with zero', () => {
+ expect(zeroCompactionStatus(ZERO_STATUS)).toEqual(true);
+ });
- expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS
})).toEqual('Running');
-
- expect(
- formatCompactionInfo({
- config: BASIC_CONFIG,
- status: {
+ it('works with non-zero', () => {
+ expect(
+ zeroCompactionStatus({
dataSource: 'tbl',
scheduleStatus: 'RUNNING',
- bytesAwaitingCompaction: 0,
- bytesCompacted: 100,
+ bytesAwaitingCompaction: 1,
+ bytesCompacted: 0,
bytesSkipped: 0,
segmentCountAwaitingCompaction: 0,
- segmentCountCompacted: 10,
+ segmentCountCompacted: 0,
segmentCountSkipped: 0,
intervalCountAwaitingCompaction: 0,
- intervalCountCompacted: 10,
+ intervalCountCompacted: 0,
intervalCountSkipped: 0,
- },
- }),
- ).toEqual('Fully compacted');
+ }),
+ ).toEqual(false);
+ });
+ });
+
+ describe('formatCompactionConfigAndStatus', () => {
+ it('works with nothing', () => {
+ expect(formatCompactionInfo({})).toEqual('Not enabled');
+ });
+
+ it('works when there is no status', () => {
+ expect(formatCompactionInfo({ config: BASIC_CONFIG })).toEqual('Awaiting
first run');
+ });
+
+ it('works when here is no config', () => {
+ expect(formatCompactionInfo({ status: ZERO_STATUS })).toEqual('Not
enabled');
+ });
+
+ it('works with config and zero status', () => {
+ expect(formatCompactionInfo({ config: BASIC_CONFIG, status: ZERO_STATUS
})).toEqual(
+ 'Running',
+ );
+ });
+
+ it('works when fully compacted', () => {
+ expect(
+ formatCompactionInfo({
+ config: BASIC_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 100,
+ bytesSkipped: 0,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 10,
+ segmentCountSkipped: 0,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 10,
+ intervalCountSkipped: 0,
+ },
+ }),
+ ).toEqual('Fully compacted');
+ });
+
+ it('works when fully compacted and some segments skipped', () => {
+ expect(
+ formatCompactionInfo({
+ config: BASIC_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 0,
+ bytesSkipped: 3776979,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 24,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 24,
+ },
+ }),
+ ).toEqual('Fully compacted (except the last P1D of data, 24 segments
skipped)');
+ });
+
+ it('works when fully compacted and some segments skipped (with legacy
config)', () => {
+ expect(
+ formatCompactionInfo({
+ config: LEGACY_CONFIG,
+ status: {
+ dataSource: 'tbl',
+ scheduleStatus: 'RUNNING',
+ bytesAwaitingCompaction: 0,
+ bytesCompacted: 0,
+ bytesSkipped: 3776979,
+ segmentCountAwaitingCompaction: 0,
+ segmentCountCompacted: 0,
+ segmentCountSkipped: 24,
+ intervalCountAwaitingCompaction: 0,
+ intervalCountCompacted: 0,
+ intervalCountSkipped: 24,
+ },
+ }),
+ ).toEqual(
+ 'Fully compacted (except the last P1D of data and segments larger than
1.00MB, 24 segments skipped)',
+ );
+ });
});
});
diff --git
a/web-console/src/druid-models/compaction-status/compaction-status.ts
b/web-console/src/druid-models/compaction-status/compaction-status.ts
index d17f2c44fd..c19cb26b6c 100644
--- a/web-console/src/druid-models/compaction-status/compaction-status.ts
+++ b/web-console/src/druid-models/compaction-status/compaction-status.ts
@@ -16,7 +16,11 @@
* limitations under the License.
*/
-import { CompactionConfig } from '../compaction-config/compaction-config';
+import { formatBytesCompact, pluralIfNeeded } from '../../utils';
+import {
+ CompactionConfig,
+ compactionConfigHasLegacyInputSegmentSizeBytesSet,
+} from '../compaction-config/compaction-config';
function capitalizeFirst(str: string): string {
return str.slice(0, 1).toUpperCase() + str.slice(1).toLowerCase();
@@ -59,8 +63,21 @@ export function formatCompactionInfo(compaction:
CompactionInfo) {
const { config, status } = compaction;
if (config) {
if (status) {
- if (status.bytesAwaitingCompaction === 0 &&
!zeroCompactionStatus(status)) {
- return 'Fully compacted';
+ if (
+ status.bytesAwaitingCompaction === 0 &&
+ status.segmentCountAwaitingCompaction === 0 &&
+ status.intervalCountAwaitingCompaction === 0 &&
+ !zeroCompactionStatus(status)
+ ) {
+ if (status.segmentCountSkipped) {
+ return `Fully compacted (except the last
${config.skipOffsetFromLatest || 'P1D'} of data${
+ compactionConfigHasLegacyInputSegmentSizeBytesSet(config)
+ ? ` and segments larger than
${formatBytesCompact(config.inputSegmentSizeBytes!)}`
+ : ''
+ }, ${pluralIfNeeded(status.segmentCountSkipped, 'segment')}
skipped)`;
+ } else {
+ return 'Fully compacted';
+ }
} else {
return capitalizeFirst(status.scheduleStatus);
}
diff --git
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
index f4dee926b6..7bdcaae50a 100644
--- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
+++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
@@ -63,6 +63,7 @@ export function externalConfigToIngestQueryPattern(
config: ExternalConfig,
isArrays: boolean[],
timeExpression: SqlExpression | undefined,
+ partitionedByHint: string | undefined,
): IngestQueryPattern {
return {
destinationTableName:
guessDataSourceNameFromInputSource(config.inputSource) || 'data',
@@ -71,7 +72,7 @@ export function externalConfigToIngestQueryPattern(
mainExternalConfig: config,
filters: [],
dimensions: externalConfigToInitDimensions(config, isArrays,
timeExpression),
- partitionedBy: timeExpression ? 'day' : 'all',
+ partitionedBy: partitionedByHint || (timeExpression ? 'day' : 'all'),
clusteredBy: [],
};
}
diff --git a/web-console/src/druid-models/metric-spec/metric-spec.tsx
b/web-console/src/druid-models/metric-spec/metric-spec.tsx
index 6b3290272e..4295310486 100644
--- a/web-console/src/druid-models/metric-spec/metric-spec.tsx
+++ b/web-console/src/druid-models/metric-spec/metric-spec.tsx
@@ -78,6 +78,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
// Should the first / last aggregators become usable at ingestion time,
reverse the changes made in:
// https://github.com/apache/druid/pull/10794
'thetaSketch',
+ 'arrayOfDoublesSketch',
{
group: 'HLLSketch',
suggestions: ['HLLSketchBuild', 'HLLSketchMerge'],
@@ -104,6 +105,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
'doubleMax',
'floatMax',
'thetaSketch',
+ 'arrayOfDoublesSketch',
'HLLSketchBuild',
'HLLSketchMerge',
'quantilesDoublesSketch',
@@ -178,6 +180,47 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
</>
),
},
+ // arrayOfDoublesSketch
+ {
+ name: 'nominalEntries',
+ type: 'number',
+ defined: typeIs('arrayOfDoublesSketch'),
+ defaultValue: 16384,
+ info: (
+ <>
+ <p>
+ Parameter that determines the accuracy and size of the sketch.
Higher k means higher
+ accuracy but more space to store sketches.
+ </p>
+ <p>Must be a power of 2.</p>
+ <p>
+ See the{' '}
+ <ExternalLink
href="https://datasketches.apache.org/docs/Theta/ThetaErrorTable";>
+ Theta sketch accuracy
+ </ExternalLink>{' '}
+ for details.
+ </p>
+ </>
+ ),
+ },
+ {
+ name: 'metricColumns',
+ type: 'string-array',
+ defined: typeIs('arrayOfDoublesSketch'),
+ info: (
+ <>
+ If building sketches from raw data, an array of names of the input
columns containing
+ numeric values to be associated with each distinct key.
+ </>
+ ),
+ },
+ {
+ name: 'numberOfValues',
+ type: 'number',
+ defined: typeIs('arrayOfDoublesSketch'),
+ placeholder: 'metricColumns length or 1',
+ info: <>Number of values associated with each distinct key.</>,
+ },
// HLLSketchBuild & HLLSketchMerge
{
name: 'lgK',
diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts
b/web-console/src/druid-models/workbench-query/workbench-query.ts
index 36c71cb07a..b6a1f74aa1 100644
--- a/web-console/src/druid-models/workbench-query/workbench-query.ts
+++ b/web-console/src/druid-models/workbench-query/workbench-query.ts
@@ -82,13 +82,19 @@ export class WorkbenchQuery {
externalConfig: ExternalConfig,
isArrays: boolean[],
timeExpression: SqlExpression | undefined,
+ partitionedByHint: string | undefined,
): WorkbenchQuery {
return new WorkbenchQuery({
queryContext: {},
queryParts: [
WorkbenchQueryPart.fromQueryString(
ingestQueryPatternToQuery(
- externalConfigToIngestQueryPattern(externalConfig, isArrays,
timeExpression),
+ externalConfigToIngestQueryPattern(
+ externalConfig,
+ isArrays,
+ timeExpression,
+ partitionedByHint,
+ ),
).toString(),
),
],
diff --git a/web-console/src/helpers/execution/sql-task-execution.ts
b/web-console/src/helpers/execution/sql-task-execution.ts
index e7f7250c53..358eee25c5 100644
--- a/web-console/src/helpers/execution/sql-task-execution.ts
+++ b/web-console/src/helpers/execution/sql-task-execution.ts
@@ -124,9 +124,14 @@ export async function reattachTaskExecution(
option: ReattachTaskQueryOptions,
): Promise<Execution | IntermediateQueryState<Execution>> {
const { id, cancelToken, preserveOnTermination } = option;
- let execution = await getTaskExecution(id, undefined, cancelToken);
+ let execution: Execution;
- execution = await updateExecutionWithDatasourceExistsIfNeeded(execution,
cancelToken);
+ try {
+ execution = await getTaskExecution(id, undefined, cancelToken);
+ execution = await updateExecutionWithDatasourceExistsIfNeeded(execution,
cancelToken);
+ } catch (e) {
+ throw new Error(`Reattaching to query failed due to: ${e.message}`);
+ }
if (execution.isFullyComplete()) return execution;
diff --git a/web-console/src/links.ts b/web-console/src/links.ts
index 5c391d838e..6f61c29d05 100644
--- a/web-console/src/links.ts
+++ b/web-console/src/links.ts
@@ -84,7 +84,7 @@ export function getLink(linkName: LinkNames): string {
case 'DOCS_API':
return `${links.docsHref}/operations/api-reference.html`;
case 'DOCS_MSQ_ERROR':
- return `${links.docsHref}/multi-stage-query/concepts.html#error-codes`;
+ return `${links.docsHref}/multi-stage-query/reference.html`;
case 'COMMUNITY':
return links.communityHref;
case 'SLACK':
diff --git a/web-console/src/react-table/react-table-extra.scss
b/web-console/src/react-table/react-table-extra.scss
index d87c25c84d..bdeecf5e96 100644
--- a/web-console/src/react-table/react-table-extra.scss
+++ b/web-console/src/react-table/react-table-extra.scss
@@ -45,4 +45,8 @@
}
}
}
+
+ .default-aggregated {
+ padding: 10px 5px;
+ }
}
diff --git
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
index c48713eff2..9bb9c16fef 100644
---
a/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
+++
b/web-console/src/views/datasources-view/__snapshots__/datasources-view.spec.tsx.snap
@@ -283,7 +283,7 @@ exports[`DatasourcesView matches snapshot 1`] = `
"filterable": false,
"id": "compactionStatus",
"show": true,
- "width": 150,
+ "width": 180,
},
Object {
"Cell": [Function],
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx
b/web-console/src/views/datasources-view/datasources-view.tsx
index c38a42b639..38b85388d2 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -1324,7 +1324,7 @@ ORDER BY 1`;
id: 'compactionStatus',
accessor: row => Boolean(row.compaction?.status),
filterable: false,
- width: 150,
+ width: 180,
Cell: ({ original }) => {
const { datasource, compaction } = original as Datasource;
return (
diff --git
a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 23c81d02d2..2b3126372a 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -151,6 +151,7 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
{ inputSource, inputFormat, signature },
isArrays,
timeExpression,
+ undefined,
),
).toString(),
queryContext: {
@@ -167,6 +168,7 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
{ inputSource, inputFormat, signature },
isArrays,
timeExpression,
+ undefined,
),
).toString(),
});
diff --git
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
index d857d37dbf..e043cf1340 100644
---
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
+++
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
@@ -20,7 +20,7 @@ import { Classes, Dialog } from '@blueprintjs/core';
import { SqlExpression } from 'druid-query-toolkit';
import React, { useState } from 'react';
-import { ExternalConfig } from '../../../druid-models';
+import { ExternalConfig, InputFormat, InputSource } from
'../../../druid-models';
import { InputFormatStep } from '../input-format-step/input-format-step';
import { InputSourceStep } from '../input-source-step/input-source-step';
@@ -32,20 +32,27 @@ export interface ConnectExternalDataDialogProps {
config: ExternalConfig,
isArrays: boolean[],
timeExpression: SqlExpression | undefined,
+ partitionedByHint: string | undefined,
): void;
onClose(): void;
}
+interface ExternalConfigStep {
+ inputSource?: InputSource;
+ inputFormat?: InputFormat;
+ partitionedByHint?: string;
+}
+
export const ConnectExternalDataDialog = React.memo(function
ConnectExternalDataDialog(
props: ConnectExternalDataDialogProps,
) {
const { initExternalConfig, onClose, onSetExternalConfig } = props;
- const [externalConfigStep, setExternalConfigStep] =
useState<Partial<ExternalConfig>>(
+ const [externalConfigStep, setExternalConfigStep] =
useState<ExternalConfigStep>(
initExternalConfig || {},
);
- const { inputSource, inputFormat } = externalConfigStep;
+ const { inputSource, inputFormat, partitionedByHint } = externalConfigStep;
return (
<Dialog
@@ -65,6 +72,7 @@ export const ConnectExternalDataDialog = React.memo(function
ConnectExternalData
{ inputSource, inputFormat, signature },
isArrays,
timeExpression,
+ partitionedByHint,
);
onClose();
}}
@@ -76,8 +84,8 @@ export const ConnectExternalDataDialog = React.memo(function
ConnectExternalData
<InputSourceStep
initInputSource={inputSource}
mode="sampler"
- onSet={(inputSource, inputFormat) => {
- setExternalConfigStep({ inputSource, inputFormat });
+ onSet={(inputSource, inputFormat, partitionedByHint) => {
+ setExternalConfigStep({ inputSource, inputFormat,
partitionedByHint });
}}
/>
)}
diff --git
a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
index 24573eea25..ecd700f396 100644
---
a/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
+++
b/web-console/src/views/workbench-view/execution-error-pane/__snapshots__/execution-error-pane.spec.tsx.snap
@@ -9,7 +9,7 @@ exports[`ExecutionErrorPane matches snapshot 1`] = `
className="error-message-text"
>
<Memo(ExternalLink)
-
href="https://druid.apache.org/docs/25.0.0/multi-stage-query/concepts.html#error-codes";
+
href="https://druid.apache.org/docs/25.0.0/multi-stage-query/reference.html#error_TooManyWarnings";
>
TooManyWarnings
</Memo(ExternalLink)>
diff --git
a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx
b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx
index b0368534cc..967cea3b0e 100644
---
a/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx
+++
b/web-console/src/views/workbench-view/execution-error-pane/execution-error-pane.tsx
@@ -46,7 +46,9 @@ export const ExecutionErrorPane = React.memo(function
ExecutionErrorPane(
<p className="error-message-text">
{error.errorCode && (
<>
- <ExternalLink
href={getLink('DOCS_MSQ_ERROR')}>{error.errorCode}</ExternalLink>
+ <ExternalLink
href={`${getLink('DOCS_MSQ_ERROR')}#error_${error.errorCode}`}>
+ {error.errorCode}
+ </ExternalLink>
{': '}
</>
)}
diff --git
a/web-console/src/views/workbench-view/input-source-step/example-inputs.ts
b/web-console/src/views/workbench-view/input-source-step/example-inputs.ts
index a74f1754b1..a6ad104c7f 100644
--- a/web-console/src/views/workbench-view/input-source-step/example-inputs.ts
+++ b/web-console/src/views/workbench-view/input-source-step/example-inputs.ts
@@ -23,6 +23,7 @@ export interface ExampleInput {
description: string;
inputSource: InputSource;
inputFormat?: InputFormat;
+ partitionedByHint?: string;
}
const TRIPS_INPUT_FORMAT: InputFormat = {
@@ -122,6 +123,7 @@ export const EXAMPLE_INPUTS: ExampleInput[] = [
],
},
inputFormat: TRIPS_INPUT_FORMAT,
+ partitionedByHint: 'month',
},
{
name: 'NYC Taxi cabs (all files)',
@@ -206,6 +208,7 @@ export const EXAMPLE_INPUTS: ExampleInput[] = [
],
},
inputFormat: TRIPS_INPUT_FORMAT,
+ partitionedByHint: 'month',
},
{
name: 'FlightCarrierOnTime (1 month)',
diff --git
a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
index f144e8f975..9ea55fd0d1 100644
---
a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
+++
b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
@@ -71,7 +71,11 @@ const ROWS_TO_SAMPLE = 50;
export interface InputSourceStepProps {
initInputSource: Partial<InputSource> | undefined;
mode: 'sampler' | 'msq';
- onSet(inputSource: InputSource, inputFormat: InputFormat): void;
+ onSet(
+ inputSource: InputSource,
+ inputFormat: InputFormat,
+ partitionedByHint: string | undefined,
+ ): void;
}
export const InputSourceStep = React.memo(function InputSourceStep(props:
InputSourceStepProps) {
@@ -169,7 +173,11 @@ export const InputSourceStep = React.memo(function
InputSourceStep(props: InputS
useEffect(() => {
const guessedInputFormat = guessedInputFormatState.data;
if (!guessedInputFormat) return;
- onSet(exampleInput?.inputSource || (inputSource as any),
guessedInputFormat);
+ onSet(
+ exampleInput?.inputSource || (inputSource as any),
+ guessedInputFormat,
+ exampleInput?.partitionedByHint,
+ );
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [guessedInputFormatState]);
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx
b/web-console/src/views/workbench-view/workbench-view.tsx
index 5d601d2fb6..56af602e43 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -324,9 +324,14 @@ export class WorkbenchView extends
React.PureComponent<WorkbenchViewProps, Workb
return (
<ConnectExternalDataDialog
- onSetExternalConfig={(externalConfig, isArrays, timeExpression) => {
+ onSetExternalConfig={(externalConfig, isArrays, timeExpression,
partitionedByHint) => {
this.handleNewTab(
- WorkbenchQuery.fromInitExternalConfig(externalConfig, isArrays,
timeExpression),
+ WorkbenchQuery.fromInitExternalConfig(
+ externalConfig,
+ isArrays,
+ timeExpression,
+ partitionedByHint,
+ ),
'Ext ' +
guessDataSourceNameFromInputSource(externalConfig.inputSource),
);
}}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
