This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a23170e1586 Web console: support SET syntax in queries (#17966)
a23170e1586 is described below
commit a23170e15867382d4f42d6ad17237d2286cc69cd
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Thu May 1 11:14:27 2025 -0700
Web console: support SET syntax in queries (#17966)
* fix explain and semicolon suggestion
* move Edit context
* update for CodeQL feedback
* removed left over old comment
---
licenses.yaml | 2 +-
web-console/package-lock.json | 26 +--
web-console/package.json | 2 +-
.../array-ingest-mode-switch.spec.tsx.snap} | 2 +-
.../array-ingest-mode-switch.spec.tsx} | 8 +-
.../array-ingest-mode-switch.tsx} | 18 +-
web-console/src/components/index.ts | 2 +-
.../array-ingest-mode/array-ingest-mode.ts} | 13 +-
.../druid-models/dimension-spec/dimension-spec.ts | 5 +-
.../external-config/external-config.ts | 6 +-
web-console/src/druid-models/index.ts | 1 +
.../ingest-query-pattern.spec.ts | 10 +
.../ingest-query-pattern/ingest-query-pattern.ts | 30 ++-
.../ingestion-spec/ingestion-spec.spec.ts | 10 +-
.../druid-models/ingestion-spec/ingestion-spec.tsx | 25 ++-
.../druid-models/query-context/query-context.tsx | 3 +-
.../workbench-query/workbench-query.ts | 19 +-
.../__snapshots__/spec-conversion.spec.ts.snap | 29 +++
web-console/src/helpers/spec-conversion.spec.ts | 6 +-
web-console/src/helpers/spec-conversion.ts | 37 ++--
web-console/src/utils/druid-query.spec.ts | 12 +-
web-console/src/utils/druid-query.ts | 17 +-
web-console/src/utils/explain.spec.ts | 145 ++++++++++++++
.../explain.ts} | 29 ++-
web-console/src/utils/index.tsx | 1 +
web-console/src/utils/sql.spec.ts | 220 +++++++++++++++++++++
web-console/src/utils/sql.ts | 55 ++++--
.../src/views/load-data-view/load-data-view.tsx | 46 ++---
.../schema-step/schema-step.tsx | 14 +-
.../sql-data-loader-view/sql-data-loader-view.tsx | 20 --
.../connect-external-data-dialog.tsx | 9 +-
.../explain-dialog/explain-dialog.tsx | 10 +-
.../input-format-step/input-format-step.tsx | 26 ++-
.../views/workbench-view/run-panel/run-panel.tsx | 180 +++++++++--------
34 files changed, 747 insertions(+), 291 deletions(-)
diff --git a/licenses.yaml b/licenses.yaml
index 33c09795182..9509f442d24 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5762,7 +5762,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Imply Data
-version: 1.0.2
+version: 1.1.1
---
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index 27eef016a1d..006e331466c 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -30,7 +30,7 @@
"d3-shape": "^3.2.0",
"d3-time-format": "^4.1.0",
"date-fns": "^2.28.0",
- "druid-query-toolkit": "^1.0.2",
+ "druid-query-toolkit": "^1.1.1",
"echarts": "^5.5.1",
"file-saver": "^2.0.5",
"hjson": "^3.2.2",
@@ -7067,9 +7067,9 @@
}
},
"node_modules/druid-query-toolkit": {
- "version": "1.0.2",
- "resolved":
"https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-1.0.2.tgz";,
- "integrity":
"sha512-TXu8io3oF04g0xhiGeXBB3xsGt2kJtp+95jrSja1a5Db8NFpkso6UPbfc6vlok24bky1aUKObTPJVVKOwSaBIw==",
+ "version": "1.1.1",
+ "resolved":
"https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-1.1.1.tgz";,
+ "integrity":
"sha512-+DtPaCf7WPitr/G1YKUWsYGy4vrDzybQUqbFpyGMKIkqwcmTdCS10qfb9+eGKUu/3EIx4FVBoXFeRcMCdJ73Ow==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.5.2"
@@ -9273,9 +9273,9 @@
"dev": true
},
"node_modules/http-proxy-middleware": {
- "version": "2.0.7",
- "resolved":
"https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz";,
- "integrity":
"sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==",
+ "version": "2.0.9",
+ "resolved":
"https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz";,
+ "integrity":
"sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -23311,9 +23311,9 @@
}
},
"druid-query-toolkit": {
- "version": "1.0.2",
- "resolved":
"https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-1.0.2.tgz";,
- "integrity":
"sha512-TXu8io3oF04g0xhiGeXBB3xsGt2kJtp+95jrSja1a5Db8NFpkso6UPbfc6vlok24bky1aUKObTPJVVKOwSaBIw==",
+ "version": "1.1.1",
+ "resolved":
"https://registry.npmjs.org/druid-query-toolkit/-/druid-query-toolkit-1.1.1.tgz";,
+ "integrity":
"sha512-+DtPaCf7WPitr/G1YKUWsYGy4vrDzybQUqbFpyGMKIkqwcmTdCS10qfb9+eGKUu/3EIx4FVBoXFeRcMCdJ73Ow==",
"requires": {
"tslib": "^2.5.2"
}
@@ -24922,9 +24922,9 @@
}
},
"http-proxy-middleware": {
- "version": "2.0.7",
- "resolved":
"https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz";,
- "integrity":
"sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==",
+ "version": "2.0.9",
+ "resolved":
"https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz";,
+ "integrity":
"sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==",
"dev": true,
"requires": {
"@types/http-proxy": "^1.17.8",
diff --git a/web-console/package.json b/web-console/package.json
index 038c42962ef..ac3f4891b17 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -71,7 +71,7 @@
"d3-shape": "^3.2.0",
"d3-time-format": "^4.1.0",
"date-fns": "^2.28.0",
- "druid-query-toolkit": "^1.0.2",
+ "druid-query-toolkit": "^1.1.1",
"echarts": "^5.5.1",
"file-saver": "^2.0.5",
"hjson": "^3.2.2",
diff --git
a/web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
b/web-console/src/components/array-ingest-mode-switch/__snapshots__/array-ingest-mode-switch.spec.tsx.snap
similarity index 94%
rename from
web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
rename to
web-console/src/components/array-ingest-mode-switch/__snapshots__/array-ingest-mode-switch.spec.tsx.snap
index 278de27a359..12c7ddf5b9f 100644
---
a/web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
+++
b/web-console/src/components/array-ingest-mode-switch/__snapshots__/array-ingest-mode-switch.spec.tsx.snap
@@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`ArrayModeSwitch matches snapshot 1`] = `
+exports[`ArrayIngestModeSwitch matches snapshot 1`] = `
<div
class="bp5-form-group form-group-with-info"
>
diff --git
a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
b/web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.spec.tsx
similarity index 81%
copy from
web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
copy to
web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.spec.tsx
index a9ac2320c2d..db2737e7ebe 100644
--- a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
+++
b/web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.spec.tsx
@@ -18,11 +18,13 @@
import { render } from '@testing-library/react';
-import { ArrayModeSwitch } from './array-mode-switch';
+import { ArrayIngestModeSwitch } from './array-ingest-mode-switch';
-describe('ArrayModeSwitch', () => {
+describe('ArrayIngestModeSwitch', () => {
it('matches snapshot', () => {
- const arrayInput = <ArrayModeSwitch arrayMode="multi-values"
changeArrayMode={() => {}} />;
+ const arrayInput = (
+ <ArrayIngestModeSwitch arrayIngestMode="mvd" changeArrayIngestMode={()
=> {}} />
+ );
const { container } = render(arrayInput);
expect(container.firstChild).toMatchSnapshot();
diff --git a/web-console/src/components/array-mode-switch/array-mode-switch.tsx
b/web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.tsx
similarity index 79%
rename from web-console/src/components/array-mode-switch/array-mode-switch.tsx
rename to
web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.tsx
index 4d9de1274b9..5f7cb4d9c93 100644
--- a/web-console/src/components/array-mode-switch/array-mode-switch.tsx
+++
b/web-console/src/components/array-ingest-mode-switch/array-ingest-mode-switch.tsx
@@ -19,19 +19,21 @@
import { Switch } from '@blueprintjs/core';
import React from 'react';
-import type { ArrayMode } from '../../druid-models';
+import type { ArrayIngestMode } from '../../druid-models';
import { getLink } from '../../links';
import { ExternalLink } from '../external-link/external-link';
import { FormGroupWithInfo } from
'../form-group-with-info/form-group-with-info';
import { PopoverText } from '../popover-text/popover-text';
-export interface ArrayModeSwitchProps {
- arrayMode: ArrayMode;
- changeArrayMode(newArrayMode: ArrayMode): void;
+export interface ArrayIngestModeSwitchProps {
+ arrayIngestMode: ArrayIngestMode;
+ changeArrayIngestMode(arrayIngestMode: ArrayIngestMode): void;
}
-export const ArrayModeSwitch = React.memo(function ArrayModeSwitch(props:
ArrayModeSwitchProps) {
- const { arrayMode, changeArrayMode } = props;
+export const ArrayIngestModeSwitch = React.memo(function ArrayIngestModeSwitch(
+ props: ArrayIngestModeSwitchProps,
+) {
+ const { arrayIngestMode, changeArrayIngestMode } = props;
return (
<FormGroupWithInfo
@@ -54,8 +56,8 @@ export const ArrayModeSwitch = React.memo(function
ArrayModeSwitch(props: ArrayM
<Switch
label="Store ARRAYs as MVDs"
className="legacy-switch"
- checked={arrayMode === 'multi-values'}
- onChange={() => changeArrayMode(arrayMode === 'arrays' ?
'multi-values' : 'arrays')}
+ checked={arrayIngestMode === 'mvd'}
+ onChange={() => changeArrayIngestMode(arrayIngestMode === 'array' ?
'mvd' : 'array')}
/>
</FormGroupWithInfo>
);
diff --git a/web-console/src/components/index.ts
b/web-console/src/components/index.ts
index 60f35c51191..88c0981645c 100644
--- a/web-console/src/components/index.ts
+++ b/web-console/src/components/index.ts
@@ -18,8 +18,8 @@
export * from './action-cell/action-cell';
export * from './action-icon/action-icon';
+export * from './array-ingest-mode-switch/array-ingest-mode-switch';
export * from './array-input/array-input';
-export * from './array-mode-switch/array-mode-switch';
export * from './auto-form/auto-form';
export * from './braced-text/braced-text';
export * from './center-message/center-message';
diff --git
a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
b/web-console/src/druid-models/array-ingest-mode/array-ingest-mode.ts
similarity index 68%
copy from
web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
copy to web-console/src/druid-models/array-ingest-mode/array-ingest-mode.ts
index a9ac2320c2d..beef8521c8c 100644
--- a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
+++ b/web-console/src/druid-models/array-ingest-mode/array-ingest-mode.ts
@@ -16,15 +16,6 @@
* limitations under the License.
*/
-import { render } from '@testing-library/react';
+export type ArrayIngestMode = 'array' | 'mvd';
-import { ArrayModeSwitch } from './array-mode-switch';
-
-describe('ArrayModeSwitch', () => {
- it('matches snapshot', () => {
- const arrayInput = <ArrayModeSwitch arrayMode="multi-values"
changeArrayMode={() => {}} />;
-
- const { container } = render(arrayInput);
- expect(container.firstChild).toMatchSnapshot();
- });
-});
+export const DEFAULT_ARRAY_INGEST_MODE: ArrayIngestMode = 'array';
diff --git a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
index 3037cbe0838..85933bdeb54 100644
--- a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
+++ b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
@@ -20,6 +20,7 @@ import type { Field } from '../../components';
import { filterMap, typeIsKnown } from '../../utils';
import type { SampleResponse, TimeColumnAction } from '../../utils/sampler';
import { getHeaderNamesFromSampleResponse } from '../../utils/sampler';
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
import { guessColumnTypeFromSampleResponse } from
'../ingestion-spec/ingestion-spec';
import { TIME_COLUMN } from '../timestamp-spec/timestamp-spec';
@@ -165,7 +166,7 @@ export function getDimensionSpecs(
sampleResponse: SampleResponse,
columnTypeHints: Record<string, string>,
guessNumericStringsAsNumbers: boolean,
- forceMvdInsteadOfArray: boolean,
+ arrayIngestMode: ArrayIngestMode,
hasRollup: boolean,
timeColumnAction: TimeColumnAction,
): (string | DimensionSpec)[] {
@@ -182,7 +183,7 @@ export function getDimensionSpecs(
);
let columnType = columnTypeHint || guessedColumnType;
- if (forceMvdInsteadOfArray) {
+ if (arrayIngestMode === 'mvd') {
if (columnType.startsWith('ARRAY')) {
columnType = MADE_UP_MV_COLUMN_TYPE;
}
diff --git a/web-console/src/druid-models/external-config/external-config.ts
b/web-console/src/druid-models/external-config/external-config.ts
index 6fa74eb038b..dc4dfbcf496 100644
--- a/web-console/src/druid-models/external-config/external-config.ts
+++ b/web-console/src/druid-models/external-config/external-config.ts
@@ -32,7 +32,7 @@ import {
import * as JSONBig from 'json-bigint-native';
import { nonEmptyArray } from '../../utils';
-import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
import type { InputFormat } from '../input-format/input-format';
import type { InputSource } from '../input-source/input-source';
@@ -129,7 +129,7 @@ export function externalConfigToTableExpression(config:
ExternalConfig): SqlExpr
export function externalConfigToInitDimensions(
config: ExternalConfig,
timeExpression: SqlExpression | undefined,
- arrayMode: ArrayMode,
+ arrayIngestMode: ArrayIngestMode,
): SqlExpression[] {
return (timeExpression ? [timeExpression.setAlias('__time')] : [])
.concat(
@@ -137,7 +137,7 @@ export function externalConfigToInitDimensions(
const columnName = columnDeclaration.getColumnName();
if (timeExpression && timeExpression.containsColumnName(columnName))
return;
return C(columnName).applyIf(
- arrayMode === 'multi-values' &&
columnDeclaration.columnType.isArray(),
+ arrayIngestMode === 'mvd' && columnDeclaration.columnType.isArray(),
ex => F('ARRAY_TO_MV', ex).as(columnName),
);
}),
diff --git a/web-console/src/druid-models/index.ts
b/web-console/src/druid-models/index.ts
index 2e23cf7ee2c..6031c8ef624 100644
--- a/web-console/src/druid-models/index.ts
+++ b/web-console/src/druid-models/index.ts
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+export * from './array-ingest-mode/array-ingest-mode';
export * from './async-query/async-query';
export * from './compaction-config/compaction-config';
export * from './compaction-dynamic-config/compaction-dynamic-config';
diff --git
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
index cf108c2b389..00c6a2bb54b 100644
---
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
+++
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
@@ -23,6 +23,9 @@ import { fitIngestQueryPattern, ingestQueryPatternToQuery }
from './ingest-query
describe('ingest-query-pattern', () => {
it('works with no group by', () => {
const query = SqlQuery.parse(sane`
+ SET arrayIngestMode = 'array';
+ SET finalizeAggregations = FALSE;
+ SET groupByEnableMultiValueUnnesting = FALSE;
INSERT INTO "kttm-2019"
WITH "ext" AS (
SELECT *
@@ -48,6 +51,7 @@ describe('ingest-query-pattern', () => {
expect(insertQueryPattern).toMatchInlineSnapshot(`
{
+ "arrayIngestMode": "array",
"clusteredBy": [
3,
],
@@ -60,6 +64,7 @@ describe('ingest-query-pattern', () => {
"browser_version",
],
"filters": [],
+ "forceSegmentSortByTime": true,
"mainExternalConfig": {
"inputFormat": {
"type": "json",
@@ -117,6 +122,9 @@ describe('ingest-query-pattern', () => {
it('works with group by', () => {
const query = SqlQuery.parse(sane`
+ SET arrayIngestMode = 'array';
+ SET finalizeAggregations = FALSE;
+ SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "inline_data" OVERWRITE ALL
WITH "ext" AS (
SELECT *
@@ -143,6 +151,7 @@ describe('ingest-query-pattern', () => {
expect(insertQueryPattern).toMatchInlineSnapshot(`
{
+ "arrayIngestMode": "array",
"clusteredBy": [],
"destinationTableName": "inline_data",
"dimensions": [
@@ -153,6 +162,7 @@ describe('ingest-query-pattern', () => {
""floats"",
],
"filters": [],
+ "forceSegmentSortByTime": true,
"mainExternalConfig": {
"inputFormat": {
"type": "json",
diff --git
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
index d2009ad2e5e..00e4f7c608c 100644
--- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
+++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
@@ -29,13 +29,13 @@ import {
} from 'druid-query-toolkit';
import { filterMap, oneOf } from '../../utils';
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
import type { ExternalConfig } from '../external-config/external-config';
import {
externalConfigToInitDimensions,
externalConfigToTableExpression,
fitExternalConfigPattern,
} from '../external-config/external-config';
-import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
import { guessDataSourceNameFromInputSource } from
'../ingestion-spec/ingestion-spec';
export type IngestMode = 'insert' | 'replace';
@@ -50,6 +50,7 @@ function removeTopLevelArrayToMvOrUndefined(dimension:
SqlExpression): SqlExpres
export interface IngestQueryPattern {
destinationTableName: string;
mode: IngestMode;
+ arrayIngestMode: ArrayIngestMode;
overwriteWhere?: SqlExpression;
mainExternalName: string;
mainExternalConfig: ExternalConfig;
@@ -58,23 +59,26 @@ export interface IngestQueryPattern {
metrics?: readonly SqlExpression[];
partitionedBy: string;
clusteredBy: readonly number[];
+ forceSegmentSortByTime: boolean;
}
export function externalConfigToIngestQueryPattern(
config: ExternalConfig,
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
- arrayMode: ArrayMode,
+ arrayIngestMode: ArrayIngestMode,
): IngestQueryPattern {
return {
destinationTableName:
guessDataSourceNameFromInputSource(config.inputSource) || 'data',
mode: 'replace',
+ arrayIngestMode,
mainExternalName: 'ext',
mainExternalConfig: config,
filters: [],
- dimensions: externalConfigToInitDimensions(config, timeExpression,
arrayMode),
+ dimensions: externalConfigToInitDimensions(config, timeExpression,
arrayIngestMode),
partitionedBy: partitionedByHint || (timeExpression ? 'day' : 'all'),
clusteredBy: [],
+ forceSegmentSortByTime: true,
};
}
@@ -147,6 +151,10 @@ export function fitIngestQueryPattern(query: SqlQuery):
IngestQueryPattern {
);
}
+ const queryContext = query.getContext();
+ const arrayIngestMode = queryContext['arrayIngestMode'] ?? 'array';
+ const forceSegmentSortByTime = queryContext['forceSegmentSortByTime'] ??
true;
+
let destinationTableName: string;
let mode: IngestMode;
let overwriteWhere: SqlExpression | undefined;
@@ -229,6 +237,7 @@ export function fitIngestQueryPattern(query: SqlQuery):
IngestQueryPattern {
return {
destinationTableName,
mode,
+ arrayIngestMode,
overwriteWhere,
mainExternalName,
mainExternalConfig,
@@ -237,6 +246,7 @@ export function fitIngestQueryPattern(query: SqlQuery):
IngestQueryPattern {
metrics,
partitionedBy,
clusteredBy,
+ forceSegmentSortByTime,
};
}
@@ -248,6 +258,7 @@ export function ingestQueryPatternToQuery(
const {
destinationTableName,
mode,
+ arrayIngestMode,
overwriteWhere,
mainExternalName,
mainExternalConfig,
@@ -256,8 +267,21 @@ export function ingestQueryPatternToQuery(
metrics,
partitionedBy,
clusteredBy,
+ forceSegmentSortByTime,
} = ingestQueryPattern;
+
+ const queryContext: Record<string, any> = {
+ arrayIngestMode,
+ finalizeAggregations: false,
+ groupByEnableMultiValueUnnesting: false,
+ };
+
+ if (!forceSegmentSortByTime) {
+ queryContext.forceSegmentSortByTime = forceSegmentSortByTime;
+ }
+
return SqlQuery.from(T(mainExternalName))
+ .changeContext(queryContext)
.applyIf(!preview, q =>
mode === 'insert'
? q.changeInsertIntoTable(destinationTableName)
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
index 1371001e980..5b4a43aaaa3 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
@@ -864,7 +864,7 @@ describe('spec utils', () => {
JSON_SAMPLE,
false,
'fixed',
- 'arrays',
+ 'array',
true,
);
expect(updateSpec.spec).toMatchInlineSnapshot(`
@@ -947,7 +947,7 @@ describe('spec utils', () => {
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
- 'arrays',
+ 'array',
true,
);
expect(updateSpec.spec).toMatchInlineSnapshot(`
@@ -1025,7 +1025,7 @@ describe('spec utils', () => {
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
- 'multi-values',
+ 'mvd',
true,
);
expect(updateSpec.spec).toMatchInlineSnapshot(`
@@ -1103,7 +1103,7 @@ describe('spec utils', () => {
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
- 'arrays',
+ 'array',
false,
);
expect(updatedSpec.spec).toMatchInlineSnapshot(`
@@ -1172,7 +1172,7 @@ describe('spec utils', () => {
JSON_SAMPLE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
'fixed',
- 'multi-values',
+ 'mvd',
false,
);
expect(updatedSpec.spec).toMatchInlineSnapshot(`
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index f0f0e7e281f..0100cc34c04 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -43,6 +43,8 @@ import {
typeIsKnown,
} from '../../utils';
import type { SampleResponse } from '../../utils/sampler';
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
+import { DEFAULT_ARRAY_INGEST_MODE } from
'../array-ingest-mode/array-ingest-mode';
import type { DimensionSpec, DimensionsSpec } from
'../dimension-spec/dimension-spec';
import {
getDimensionSpecColumnType,
@@ -297,11 +299,8 @@ export interface DataSchema {
export type SchemaMode = 'fixed' | 'string-only-discovery' |
'type-aware-discovery';
-export type ArrayMode = 'arrays' | 'multi-values';
-
export const DEFAULT_FORCE_SEGMENT_SORT_BY_TIME = true;
export const DEFAULT_SCHEMA_MODE: SchemaMode = 'fixed';
-export const DEFAULT_ARRAY_MODE: ArrayMode = 'arrays';
export function getForceSegmentSortByTime(spec: Partial<IngestionSpec>):
boolean {
return (
@@ -320,17 +319,17 @@ export function getSchemaMode(spec:
Partial<IngestionSpec>): SchemaMode {
return Array.isArray(dimensions) && dimensions.length === 0 ?
'string-only-discovery' : 'fixed';
}
-export function getArrayMode(
+export function getArrayIngestMode(
spec: Partial<IngestionSpec>,
- whenUnclear: ArrayMode = 'arrays',
-): ArrayMode {
+ whenUnclear: ArrayIngestMode = DEFAULT_ARRAY_INGEST_MODE,
+): ArrayIngestMode {
const schemaMode = getSchemaMode(spec);
switch (schemaMode) {
case 'type-aware-discovery':
- return 'arrays';
+ return 'array';
case 'string-only-discovery':
- return 'multi-values';
+ return 'mvd';
default: {
const dimensions: (DimensionSpec | string)[] = deepGet(
@@ -344,7 +343,7 @@ export function getArrayMode(
typeof d === 'object' && d.type === 'auto' &&
String(d.castToType).startsWith('ARRAY'),
)
) {
- return 'arrays';
+ return 'array';
}
if (
@@ -355,7 +354,7 @@ export function getArrayMode(
typeof d.multiValueHandling === 'string',
)
) {
- return 'multi-values';
+ return 'mvd';
}
return whenUnclear;
@@ -363,7 +362,7 @@ export function getArrayMode(
}
}
-export function showArrayModeToggle(spec: Partial<IngestionSpec>): boolean {
+export function showArrayIngestModeToggle(spec: Partial<IngestionSpec>):
boolean {
const schemaMode = getSchemaMode(spec);
if (schemaMode !== 'fixed') return false;
@@ -2771,7 +2770,7 @@ export function updateSchemaWithSample(
sampleResponse: SampleResponse,
forceSegmentSortByTime: boolean,
schemaMode: SchemaMode,
- arrayMode: ArrayMode,
+ arrayIngestMode: ArrayIngestMode,
rollup: boolean,
forcePartitionInitialization = false,
): Partial<IngestionSpec> {
@@ -2817,7 +2816,7 @@ export function updateSchemaWithSample(
sampleResponse,
columnTypeHints,
guessNumericStringsAsNumbers,
- arrayMode === 'multi-values',
+ arrayIngestMode,
rollup,
forceSegmentSortByTime ?? DEFAULT_FORCE_SEGMENT_SORT_BY_TIME ?
'ignore' : 'preserve',
),
diff --git a/web-console/src/druid-models/query-context/query-context.tsx
b/web-console/src/druid-models/query-context/query-context.tsx
index b702c233457..ae6ffb27e1c 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -16,8 +16,9 @@
* limitations under the License.
*/
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
+
export type SelectDestination = 'taskReport' | 'durableStorage';
-export type ArrayIngestMode = 'array' | 'mvd';
export type TaskAssignment = 'auto' | 'max';
export type SqlJoinAlgorithm = 'broadcast' | 'sortMerge';
diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts
b/web-console/src/druid-models/workbench-query/workbench-query.ts
index 6ac6bba3c2a..44ec5e70f0a 100644
--- a/web-console/src/druid-models/workbench-query/workbench-query.ts
+++ b/web-console/src/druid-models/workbench-query/workbench-query.ts
@@ -30,6 +30,7 @@ import {
SqlOrderByClause,
SqlOrderByExpression,
SqlQuery,
+ SqlSetStatement,
} from 'druid-query-toolkit';
import Hjson from 'hjson';
import * as JSONBig from 'json-bigint-native';
@@ -37,6 +38,7 @@ import { v4 as uuidv4 } from 'uuid';
import type { RowColumn } from '../../utils';
import { caseInsensitiveEquals, deleteKeys } from '../../utils';
+import type { ArrayIngestMode } from '../array-ingest-mode/array-ingest-mode';
import type { DruidEngine } from '../druid-engine/druid-engine';
import { validDruidEngine } from '../druid-engine/druid-engine';
import type { LastExecution } from '../execution/execution';
@@ -46,7 +48,6 @@ import {
externalConfigToIngestQueryPattern,
ingestQueryPatternToQuery,
} from '../ingest-query-pattern/ingest-query-pattern';
-import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
import type { QueryContext } from '../query-context/query-context';
const ISSUE_MARKER = '--:ISSUE:';
@@ -93,10 +94,8 @@ export class WorkbenchQuery {
externalConfig: ExternalConfig,
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
- arrayMode: ArrayMode,
+ arrayMode: ArrayIngestMode,
): WorkbenchQuery {
- const queryContext: QueryContext = {};
- if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
return new WorkbenchQuery({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@@ -106,7 +105,7 @@ export class WorkbenchQuery {
arrayMode,
),
).toString(),
- queryContext,
+ queryContext: {},
});
}
@@ -286,6 +285,16 @@ export class WorkbenchQuery {
return new WorkbenchQuery({ ...this.valueOf(), queryString });
}
+ public changeQueryStringContext(queryContext: QueryContext): WorkbenchQuery {
+ const { queryString } = this;
+ return
this.changeQueryString(SqlSetStatement.setContextInText(queryString,
queryContext));
+ }
+
+ public getQueryStringContext(): QueryContext {
+ const { queryString } = this;
+ return SqlSetStatement.getContextFromText(queryString);
+ }
+
public changeQueryContext(queryContext: QueryContext): WorkbenchQuery {
return new WorkbenchQuery({ ...this.valueOf(), queryContext });
}
diff --git a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
index 11f2a388ad2..235e36da062 100644
--- a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
+++ b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
@@ -2,6 +2,9 @@
exports[`spec conversion converts index_hadoop spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "newSource" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -30,6 +33,9 @@ PARTITIONED BY HOUR
exports[`spec conversion converts index_parallel spec (with rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "wikipedia_rollup" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -70,6 +76,11 @@ PARTITIONED BY HOUR
exports[`spec conversion converts index_parallel spec (without rollup) 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET maxNumTasks = 5;
+SET maxParseExceptions = 3;
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -111,6 +122,11 @@ CLUSTERED BY "isRobot"
exports[`spec conversion converts with issue when there is a __time transform
1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET maxNumTasks = 5;
+SET maxParseExceptions = 3;
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -151,6 +167,11 @@ CLUSTERED BY "isRobot"
exports[`spec conversion converts with issue when there is a dimension
transform and strange filter 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET maxNumTasks = 5;
+SET maxParseExceptions = 3;
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -192,6 +213,11 @@ CLUSTERED BY "isRobot"
exports[`spec conversion converts with when the __time column is used as the
__time column 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET maxNumTasks = 5;
+SET maxParseExceptions = 3;
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "wikipedia" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
@@ -211,6 +237,9 @@ CLUSTERED BY "isRobot"
exports[`spec conversion works with ARRAY mode 1`] = `
-- This SQL query was auto generated from an ingestion spec
+SET arrayIngestMode = 'array';
+SET finalizeAggregations = FALSE;
+SET groupByEnableMultiValueUnnesting = FALSE;
REPLACE INTO "lol" OVERWRITE ALL
WITH "source" AS (SELECT * FROM TABLE(
EXTERN(
diff --git a/web-console/src/helpers/spec-conversion.spec.ts
b/web-console/src/helpers/spec-conversion.spec.ts
index 9271196dafa..32580a4b487 100644
--- a/web-console/src/helpers/spec-conversion.spec.ts
+++ b/web-console/src/helpers/spec-conversion.spec.ts
@@ -123,8 +123,6 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
- maxParseExceptions: 3,
- maxNumTasks: 5,
indexSpec: {
dimensionCompression: 'lzf',
},
@@ -653,8 +651,6 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
- expect(converted.queryContext).toEqual({
- arrayIngestMode: 'array',
- });
+ expect(converted.queryContext).toEqual({});
});
});
diff --git a/web-console/src/helpers/spec-conversion.ts
b/web-console/src/helpers/spec-conversion.ts
index 9fc95d3b60f..84ccb7bcb38 100644
--- a/web-console/src/helpers/spec-conversion.ts
+++ b/web-console/src/helpers/spec-conversion.ts
@@ -38,7 +38,8 @@ import type {
Transform,
} from '../druid-models';
import {
- getArrayMode,
+ DEFAULT_ARRAY_INGEST_MODE,
+ getArrayIngestMode,
inflateDimensionSpec,
NO_SUCH_COLUMN,
TIME_COLUMN,
@@ -82,24 +83,32 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
const context: QueryContext = {};
- if (getArrayMode(spec, 'multi-values') === 'arrays') {
- context.arrayIngestMode = 'array';
+ const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
+ if (indexSpec) {
+ context.indexSpec = indexSpec;
}
+ const lines: string[] = [`-- This SQL query was auto generated from an
ingestion spec`];
+
+ lines.push(`SET arrayIngestMode = ${L(getArrayIngestMode(spec,
DEFAULT_ARRAY_INGEST_MODE))};`);
+
const forceSegmentSortByTime = deepGet(
spec,
'spec.dataSchema.dimensionsSpec.forceSegmentSortByTime',
);
if (typeof forceSegmentSortByTime !== 'undefined') {
- context.forceSegmentSortByTime = forceSegmentSortByTime;
+ lines.push(`SET forceSegmentSortByTime = ${L(forceSegmentSortByTime)};`);
}
- const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
- if (indexSpec) {
- context.indexSpec = indexSpec;
+ const maxNumConcurrentSubTasks = deepGet(spec,
'spec.tuningConfig.maxNumConcurrentSubTasks');
+ if (maxNumConcurrentSubTasks > 1) {
+ lines.push(`SET maxNumTasks = ${maxNumConcurrentSubTasks + 1};`);
}
- const lines: string[] = [];
+ const maxParseExceptions = deepGet(spec,
'spec.tuningConfig.maxParseExceptions');
+ if (typeof maxParseExceptions === 'number') {
+ lines.push(`SET maxParseExceptions = ${maxParseExceptions};`);
+ }
const rollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup') ??
true;
@@ -212,17 +221,7 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
);
}
- lines.push(`-- This SQL query was auto generated from an ingestion spec`);
-
- const maxNumConcurrentSubTasks = deepGet(spec,
'spec.tuningConfig.maxNumConcurrentSubTasks');
- if (maxNumConcurrentSubTasks > 1) {
- context.maxNumTasks = maxNumConcurrentSubTasks + 1;
- }
-
- const maxParseExceptions = deepGet(spec,
'spec.tuningConfig.maxParseExceptions');
- if (typeof maxParseExceptions === 'number') {
- context.maxParseExceptions = maxParseExceptions;
- }
+ lines.push(`SET finalizeAggregations = FALSE;`, `SET
groupByEnableMultiValueUnnesting = FALSE;`);
const dataSource = deepGet(spec, 'spec.dataSchema.dataSource');
if (typeof dataSource !== 'string') throw new
Error(`spec.dataSchema.dataSource is not a string`);
diff --git a/web-console/src/utils/druid-query.spec.ts
b/web-console/src/utils/druid-query.spec.ts
index a940b8ac7bb..bf0f6558f1b 100644
--- a/web-console/src/utils/druid-query.spec.ts
+++ b/web-console/src/utils/druid-query.spec.ts
@@ -214,15 +214,13 @@ describe('DruidQuery', () => {
);
});
- it('removes trailing semicolon (;)', () => {
- const sql = `SELECT page FROM wikipedia WHERE channel =
'#ar.wikipedia';`;
+ it('adds missing semicolon (;)', () => {
+ const sql = `SET x = 1 SELECT * FROM wikipedia`;
const suggestion = DruidError.getSuggestion(
- `Received an unexpected token [;] (line [1], column [59]), acceptable
options:`,
- );
- expect(suggestion!.label).toEqual(`Remove trailing semicolon (;)`);
- expect(suggestion!.fn(sql)).toEqual(
- `SELECT page FROM wikipedia WHERE channel = '#ar.wikipedia'`,
+ `Received an unexpected token [SELECT] (line [1], column [11]),
acceptable options: [<EOF>, ";"]`,
);
+ expect(suggestion!.label).toEqual(`Add semicolon (;) after SET
statement`);
+ expect(suggestion!.fn(sql)).toEqual(`SET x = 1; SELECT * FROM
wikipedia`);
});
it('does nothing there there is nothing to do', () => {
diff --git a/web-console/src/utils/druid-query.ts
b/web-console/src/utils/druid-query.ts
index d83cf5eb9b6..72813889034 100644
--- a/web-console/src/utils/druid-query.ts
+++ b/web-console/src/utils/druid-query.ts
@@ -236,20 +236,23 @@ export class DruidError extends Error {
};
}
- // Semicolon (;) at the end. https://bit.ly/1n1yfkJ
- // ex: SELECT 1;
- // ex: Received an unexpected token [;] (line [1], column [9]), acceptable
options:
+ // Missing (;) after SET statement
+ // ex: SET sqlTimeZone = 'America/Los_Angeles' SELECT * FROM "kttm_simple"
+ // ex: Received an unexpected token [SELECT] (line [1], column [41]),
acceptable options: [<EOF>, <QUOTED_STRING>, ";", "UESCAPE"]
const matchSemicolon =
- /Received an unexpected token \[;] \(line \[(\d+)], column
\[(\d+)]\),/i.exec(errorMessage);
+ /Received an unexpected token \[(?:SET|SELECT)] \(line \[(\d+)], column
\[(\d+)]\), acceptable options: \[[^;]*";"/i.exec(
+ errorMessage,
+ );
if (matchSemicolon) {
const line = Number(matchSemicolon[1]);
const column = Number(matchSemicolon[2]);
return {
- label: `Remove trailing semicolon (;)`,
+ label: `Add semicolon (;) after SET statement`,
fn: str => {
const index = DruidError.positionToIndex(str, line, column);
- if (str[index] !== ';') return;
- return str.slice(0, index) + str.slice(index + 1);
+ const prefix = str.slice(0, index).trimEnd();
+ if (prefix.endsWith(';')) return;
+ return prefix + ';' + str.slice(prefix.length);
},
};
}
diff --git a/web-console/src/utils/explain.spec.ts
b/web-console/src/utils/explain.spec.ts
new file mode 100644
index 00000000000..f23a412c10d
--- /dev/null
+++ b/web-console/src/utils/explain.spec.ts
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { sane } from 'druid-query-toolkit';
+
+import {
+ wrapInExplainAsParsedIfNeeded,
+ wrapInExplainAsStringIfNeeded,
+ wrapInExplainIfNeeded,
+} from './explain';
+
+describe('explain utils', () => {
+ describe('wrapInExplain*', () => {
+ const queries: [string, string][] = [
+ [
+ sane`
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ sane`
+ EXPLAIN PLAN FOR
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ ],
+ [
+ sane`
+ WITH "wikipedia" AS (select * from wikipedia2)
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ sane`
+ EXPLAIN PLAN FOR
+ WITH "wikipedia" AS (select * from wikipedia2)
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ ],
+ [
+ sane`
+ INSERT INTO "table"
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ sane`
+ EXPLAIN PLAN FOR
+ INSERT INTO "table"
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ ],
+ [
+ sane`
+ REPLACE INTO "table" OVERWRITE ALL
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ sane`
+ EXPLAIN PLAN FOR
+ REPLACE INTO "table" OVERWRITE ALL
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ ],
+ [
+ sane`
+ REPLACE INTO "table" OVERWRITE ALL
+ WITH "wikipedia" AS (select * from wikipedia2)
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ sane`
+ EXPLAIN PLAN FOR
+ REPLACE INTO "table" OVERWRITE ALL
+ WITH "wikipedia" AS (select * from wikipedia2)
+ SELECT *
+ FROM (
+ select * from wikipedia
+ )
+ `,
+ ],
+ ];
+
+ const sets = "-- A select comment\nSET a = 1;\nset b = 'EXPLAIN PLAN
FOR';\n";
+
+ it('works with queries', () => {
+ for (const [before, after] of queries) {
+ expect(wrapInExplainIfNeeded(before)).toEqual(after);
+ expect(wrapInExplainAsParsedIfNeeded(before)).toEqual(after);
+ expect(wrapInExplainAsStringIfNeeded(before)).toEqual(after);
+
+ const beforeWithSets = sets + before;
+ const afterWithSets = sets + after;
+ expect(wrapInExplainIfNeeded(beforeWithSets)).toEqual(afterWithSets);
+
expect(wrapInExplainAsParsedIfNeeded(beforeWithSets)).toEqual(afterWithSets);
+
expect(wrapInExplainAsStringIfNeeded(beforeWithSets)).toEqual(afterWithSets);
+
+ const beforeUnparsable = before + '~';
+ const afterUnparsable = after + '~';
+
expect(wrapInExplainIfNeeded(beforeUnparsable)).toEqual(afterUnparsable);
+
expect(wrapInExplainAsStringIfNeeded(beforeUnparsable)).toEqual(afterUnparsable);
+
+ const beforeUnparsableWithSets = beforeWithSets + '~';
+ const afterUnparsableWithSets = afterWithSets + '~';
+
expect(wrapInExplainIfNeeded(beforeUnparsableWithSets)).toEqual(afterUnparsableWithSets);
+
expect(wrapInExplainAsStringIfNeeded(beforeUnparsableWithSets)).toEqual(
+ afterUnparsableWithSets,
+ );
+ }
+ });
+ });
+});
diff --git
a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
b/web-console/src/utils/explain.ts
similarity index 50%
rename from
web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
rename to web-console/src/utils/explain.ts
index a9ac2320c2d..bb98409c35b 100644
--- a/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
+++ b/web-console/src/utils/explain.ts
@@ -16,15 +16,26 @@
* limitations under the License.
*/
-import { render } from '@testing-library/react';
+import { SqlQuery, SqlSetStatement } from 'druid-query-toolkit';
-import { ArrayModeSwitch } from './array-mode-switch';
+export function wrapInExplainIfNeeded(query: string): string {
+ try {
+ return wrapInExplainAsParsedIfNeeded(query);
+ } catch {
+ return wrapInExplainAsStringIfNeeded(query);
+ }
+}
-describe('ArrayModeSwitch', () => {
- it('matches snapshot', () => {
- const arrayInput = <ArrayModeSwitch arrayMode="multi-values"
changeArrayMode={() => {}} />;
+export function wrapInExplainAsParsedIfNeeded(query: string): string {
+ const parsed = SqlQuery.parse(query);
+ if (parsed.explain) return query;
+ return parsed.makeExplain().toString();
+}
- const { container } = render(arrayInput);
- expect(container.firstChild).toMatchSnapshot();
- });
-});
+export function wrapInExplainAsStringIfNeeded(query: string): string {
+ const [setPart, queryPart] = SqlSetStatement.partitionSetStatements(query,
true);
+ if (/^\s*EXPLAIN\sPLAN\sFOR/im.test(queryPart)) return query;
+
+ // Only replace the first occurrence
+ return setPart + queryPart.replace(/REPLACE|INSERT|SELECT|WITH/i, 'EXPLAIN
PLAN FOR\n$&');
+}
diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx
index 760e79b6f8d..a4dee7a62fc 100644
--- a/web-console/src/utils/index.tsx
+++ b/web-console/src/utils/index.tsx
@@ -25,6 +25,7 @@ export * from './download';
export * from './download-query-detail-archive';
export * from './druid-lookup';
export * from './druid-query';
+export * from './explain';
export * from './formatter';
export * from './general';
export * from './local-storage-backed-visibility';
diff --git a/web-console/src/utils/sql.spec.ts
b/web-console/src/utils/sql.spec.ts
index 57e7bce95fe..85d01816409 100644
--- a/web-console/src/utils/sql.spec.ts
+++ b/web-console/src/utils/sql.spec.ts
@@ -619,5 +619,225 @@ describe('sql', () => {
]
`);
});
+
+ it('works with SET statements', () => {
+ const text = sane`
+ SET timeout = 100;
+ SET timeout = 50;
+ SELECT * FROM wikipedia
+ `;
+
+ const found = findAllSqlQueriesInText(text);
+
+ expect(found).toMatchInlineSnapshot(`
+ [
+ {
+ "endOffset": 60,
+ "endRowColumn": {
+ "column": 23,
+ "row": 2,
+ },
+ "index": 0,
+ "sql": "SET timeout = 100;
+ SET timeout = 50;
+ SELECT * FROM wikipedia",
+ "startOffset": 0,
+ "startRowColumn": {
+ "column": 0,
+ "row": 0,
+ },
+ },
+ ]
+ `);
+ });
+
+ it('works with multiple SET statement queries', () => {
+ const text = sane`
+ SET timeout = 100;
+ SELECT * FROM wikipedia
+
+
+ SET timeout = 50;
+ SET sqlTimeZone = 'Etc/UTC';
+ SELECT * FROM wikipedia
+ `;
+
+ const found = findAllSqlQueriesInText(text);
+
+ expect(found).toMatchInlineSnapshot(`
+ [
+ {
+ "endOffset": 42,
+ "endRowColumn": {
+ "column": 23,
+ "row": 1,
+ },
+ "index": 0,
+ "sql": "SET timeout = 100;
+ SELECT * FROM wikipedia",
+ "startOffset": 0,
+ "startRowColumn": {
+ "column": 0,
+ "row": 0,
+ },
+ },
+ {
+ "endOffset": 115,
+ "endRowColumn": {
+ "column": 23,
+ "row": 6,
+ },
+ "index": 1,
+ "sql": "SET timeout = 50;
+ SET sqlTimeZone = 'Etc/UTC';
+ SELECT * FROM wikipedia",
+ "startOffset": 45,
+ "startRowColumn": {
+ "column": 0,
+ "row": 4,
+ },
+ },
+ ]
+ `);
+ });
+
+ it('test', () => {
+ const text = sane`
+ SET finalizeAggregations = FALSE;
+ SET groupByEnableMultiValueUnnesting = FALSE;
+ REPLACE INTO "kttm-v2-2019-08-25" OVERWRITE ALL
+ SELECT
+ TIME_PARSE("timestamp") AS "__time",
+ "agent_category",
+ "agent_type",
+ "browser",
+ "browser_version",
+ "city",
+ "continent",
+ "country",
+ "version",
+ "event_type",
+ "event_subtype",
+ "loaded_image",
+ "adblock_list",
+ "forwarded_for",
+ ARRAY_TO_MV("language") AS "language",
+ "number",
+ "os",
+ "path",
+ "platform",
+ "referrer",
+ "referrer_host",
+ "region",
+ "remote_address",
+ "screen",
+ "session",
+ "session_length",
+ "timezone",
+ "timezone_offset",
+ "window"
+ FROM "ext"
+ PARTITIONED BY DAY
+ `;
+
+ const found = findAllSqlQueriesInText(text);
+
+ expect(found).toMatchInlineSnapshot(`
+ [
+ {
+ "endOffset": 655,
+ "endRowColumn": {
+ "column": 18,
+ "row": 34,
+ },
+ "index": 0,
+ "sql": "SET finalizeAggregations = FALSE;
+ SET groupByEnableMultiValueUnnesting = FALSE;
+ REPLACE INTO "kttm-v2-2019-08-25" OVERWRITE ALL
+ SELECT
+ TIME_PARSE("timestamp") AS "__time",
+ "agent_category",
+ "agent_type",
+ "browser",
+ "browser_version",
+ "city",
+ "continent",
+ "country",
+ "version",
+ "event_type",
+ "event_subtype",
+ "loaded_image",
+ "adblock_list",
+ "forwarded_for",
+ ARRAY_TO_MV("language") AS "language",
+ "number",
+ "os",
+ "path",
+ "platform",
+ "referrer",
+ "referrer_host",
+ "region",
+ "remote_address",
+ "screen",
+ "session",
+ "session_length",
+ "timezone",
+ "timezone_offset",
+ "window"
+ FROM "ext"
+ PARTITIONED BY DAY",
+ "startOffset": 0,
+ "startRowColumn": {
+ "column": 0,
+ "row": 0,
+ },
+ },
+ {
+ "endOffset": 636,
+ "endRowColumn": {
+ "column": 10,
+ "row": 33,
+ },
+ "index": 1,
+ "sql": "SELECT
+ TIME_PARSE("timestamp") AS "__time",
+ "agent_category",
+ "agent_type",
+ "browser",
+ "browser_version",
+ "city",
+ "continent",
+ "country",
+ "version",
+ "event_type",
+ "event_subtype",
+ "loaded_image",
+ "adblock_list",
+ "forwarded_for",
+ ARRAY_TO_MV("language") AS "language",
+ "number",
+ "os",
+ "path",
+ "platform",
+ "referrer",
+ "referrer_host",
+ "region",
+ "remote_address",
+ "screen",
+ "session",
+ "session_length",
+ "timezone",
+ "timezone_offset",
+ "window"
+ FROM "ext"",
+ "startOffset": 128,
+ "startRowColumn": {
+ "column": 0,
+ "row": 3,
+ },
+ },
+ ]
+ `);
+ });
});
});
diff --git a/web-console/src/utils/sql.ts b/web-console/src/utils/sql.ts
index 6c168002078..c7768dc62a9 100644
--- a/web-console/src/utils/sql.ts
+++ b/web-console/src/utils/sql.ts
@@ -23,6 +23,7 @@ import {
SqlFunction,
SqlLiteral,
SqlQuery,
+ SqlSetStatement,
SqlStar,
} from 'druid-query-toolkit';
@@ -136,26 +137,44 @@ export function findAllSqlQueriesInText(text: string):
QuerySlice[] {
let remainingText = text;
let offset = 0;
let m: RegExpExecArray | null = null;
- do {
- m = /SELECT|WITH|INSERT|REPLACE|EXPLAIN/i.exec(remainingText);
- if (m) {
- const sql = findSqlQueryPrefix(remainingText.slice(m.index));
- const advanceBy = m.index + m[0].length; // Skip the initial word
- if (sql) {
- const endIndex = m.index + sql.length;
- found.push({
- index: found.length,
- startOffset: offset + m.index,
- startRowColumn: offsetToRowColumn(text, offset + m.index)!,
- endOffset: offset + endIndex,
- endRowColumn: offsetToRowColumn(text, offset + endIndex)!,
- sql: cleanSqlQueryPrefix(sql),
- });
+ // Have an upper bound to how many queries we might extract from a long text
+ for (let i = 0; i < 1e3; i++) {
+ m = /SET|SELECT|WITH|INSERT|REPLACE|EXPLAIN/i.exec(remainingText);
+ if (!m) break;
+
+ const sql = findSqlQueryPrefix(remainingText.slice(m.index));
+ if (sql) {
+ const endIndex = m.index + sql.length;
+ found.push({
+ index: found.length,
+ startOffset: offset + m.index,
+ startRowColumn: offsetToRowColumn(text, offset + m.index)!,
+ endOffset: offset + endIndex,
+ endRowColumn: offsetToRowColumn(text, offset + endIndex)!,
+ sql: cleanSqlQueryPrefix(sql),
+ });
+ }
+
+ let advanceBy = m.index;
+ const initialWord = m[0].toUpperCase();
+ if (sql && initialWord === 'SET') {
+ const startOfSetStatements = remainingText.slice(m.index);
+ const [setPart, queryPart] = SqlSetStatement.partitionSetStatements(
+ startOfSetStatements,
+ true,
+ );
+ advanceBy += setPart.length;
+ const nextWord = /SELECT|WITH|INSERT|REPLACE/i.exec(queryPart);
+ if (nextWord) {
+ advanceBy += nextWord[0].length;
}
- remainingText = remainingText.slice(advanceBy);
- offset += advanceBy;
+ } else {
+ advanceBy += initialWord.length; // Skip the initial word only
}
- } while (m);
+
+ remainingText = remainingText.slice(advanceBy);
+ offset += advanceBy;
+ }
return found;
}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 74c23027aaa..0df1689a73f 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -46,7 +46,7 @@ import type { JSX } from 'react';
import React from 'react';
import {
- ArrayModeSwitch,
+ ArrayIngestModeSwitch,
AutoForm,
CenterMessage,
ClearableInput,
@@ -58,7 +58,7 @@ import {
} from '../../components';
import { AlertDialog, AsyncActionDialog, DiffDialog } from '../../dialogs';
import type {
- ArrayMode,
+ ArrayIngestMode,
DimensionSpec,
DruidFilter,
FlattenField,
@@ -83,7 +83,7 @@ import {
computeFlattenPathsForData,
CONSTANT_TIMESTAMP_SPEC,
CONSTANT_TIMESTAMP_SPEC_FIELDS,
- DEFAULT_ARRAY_MODE,
+ DEFAULT_ARRAY_INGEST_MODE,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
DEFAULT_SCHEMA_MODE,
DIMENSION_SPEC_FIELDS,
@@ -92,7 +92,7 @@ import {
FILTER_FIELDS,
FILTERS_FIELDS,
FLATTEN_FIELD_FIELDS,
- getArrayMode,
+ getArrayIngestMode,
getDimensionSpecName,
getFlattenSpec,
getForceSegmentSortByTime,
@@ -132,7 +132,7 @@ import {
possibleDruidFormatForValues,
PRIMARY_PARTITION_RELATED_FORM_FIELDS,
removeTimestampTransform,
- showArrayModeToggle,
+ showArrayIngestModeToggle,
splitFilter,
STREAMING_INPUT_FORMAT_FIELDS,
TIME_COLUMN,
@@ -323,7 +323,7 @@ function initializeSchemaWithSampleIfNeeded(
sample,
DEFAULT_FORCE_SEGMENT_SORT_BY_TIME,
DEFAULT_SCHEMA_MODE,
- DEFAULT_ARRAY_MODE,
+ DEFAULT_ARRAY_INGEST_MODE,
getRollup(spec, false),
);
}
@@ -410,7 +410,7 @@ export interface LoadDataViewState {
newRollup?: boolean;
newForceSegmentSortByTime?: boolean;
newSchemaMode?: SchemaMode;
- newArrayMode?: ArrayMode;
+ newArrayIngestMode?: ArrayIngestMode;
// welcome
overlordModules?: string[];
@@ -2387,7 +2387,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
);
const forceSegmentSortByTime = getForceSegmentSortByTime(spec);
const schemaMode = getSchemaMode(spec);
- const arrayMode = getArrayMode(spec);
+ const arrayMode = getArrayIngestMode(spec);
let mainFill: JSX.Element | string;
if (schemaQueryState.isInit()) {
@@ -2496,12 +2496,12 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
label="Explicitly specify schema"
/>
</FormGroupWithInfo>
- {showArrayModeToggle(spec) && (
- <ArrayModeSwitch
- arrayMode={arrayMode}
- changeArrayMode={newArrayMode => {
+ {showArrayIngestModeToggle(spec) && (
+ <ArrayIngestModeSwitch
+ arrayIngestMode={arrayMode}
+ changeArrayIngestMode={newArrayIngestMode => {
this.setState({
- newArrayMode,
+ newArrayIngestMode: newArrayIngestMode,
});
}}
/>
@@ -2635,7 +2635,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
{this.renderChangeForceSegmentSortByTime()}
{this.renderChangeRollupAction()}
{this.renderChangeSchemaModeAction()}
- {this.renderChangeArrayModeAction()}
+ {this.renderChangeArrayIngestModeAction()}
</div>
{this.renderNextBar({
disabled: !schemaQueryState.data,
@@ -2739,7 +2739,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
sampleResponse,
newForceSegmentSortByTime,
getSchemaMode(spec),
- getArrayMode(spec),
+ getArrayIngestMode(spec),
getRollup(spec),
true,
),
@@ -2777,7 +2777,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
sampleResponse,
getForceSegmentSortByTime(spec),
getSchemaMode(spec),
- getArrayMode(spec),
+ getArrayIngestMode(spec),
newRollup,
true,
),
@@ -2814,7 +2814,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
sampleResponse,
getForceSegmentSortByTime(spec),
newSchemaMode,
- getArrayMode(spec),
+ getArrayIngestMode(spec),
getRollup(spec),
),
);
@@ -2867,10 +2867,10 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
);
}
- renderChangeArrayModeAction() {
- const { newArrayMode, spec, cacheRows } = this.state;
- if (!newArrayMode || !cacheRows) return;
- const multiValues = newArrayMode === 'multi-values';
+ renderChangeArrayIngestModeAction() {
+ const { newArrayIngestMode, spec, cacheRows } = this.state;
+ if (!newArrayIngestMode || !cacheRows) return;
+ const multiValues = newArrayIngestMode === 'mvd';
return (
<AsyncActionDialog
@@ -2886,7 +2886,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
sampleResponse,
getForceSegmentSortByTime(spec),
getSchemaMode(spec),
- newArrayMode,
+ newArrayIngestMode,
getRollup(spec),
),
);
@@ -2895,7 +2895,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
successText={`Array mode changed to ${multiValues ? 'multi-values' :
'arrays'}.`}
failText="Could not change array mode"
intent={Intent.WARNING}
- onClose={() => this.setState({ newArrayMode: undefined })}
+ onClose={() => this.setState({ newArrayIngestMode: undefined })}
>
<p>
{multiValues
diff --git
a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
index f3ee9fa2adc..dd5bdd89da5 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
@@ -255,8 +255,6 @@ interface EditorColumn {
export interface SchemaStepProps {
queryString: string;
onQueryStringChange(queryString: string): void;
- forceSegmentSortByTime: boolean;
- changeForceSegmentSortByTime(forceSegmentSortByTime: boolean): void;
enableAnalyze: boolean;
goToQuery: () => void;
onBack(): void;
@@ -268,8 +266,6 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
const {
queryString,
onQueryStringChange,
- forceSegmentSortByTime,
- changeForceSegmentSortByTime,
enableAnalyze,
goToQuery,
onBack,
@@ -674,7 +670,9 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
<MenuItem
key={i}
text={outputName}
- disabled={outputName === TIME_COLUMN &&
forceSegmentSortByTime}
+ disabled={
+ outputName === TIME_COLUMN &&
ingestQueryPattern.forceSegmentSortByTime
+ }
onClick={() =>
updatePattern({
...ingestQueryPattern,
@@ -690,8 +688,10 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
<MenuBoolean
icon={IconNames.GEOTIME}
text="Force segment sort by time"
- value={forceSegmentSortByTime}
- onValueChange={v =>
changeForceSegmentSortByTime(Boolean(v))}
+ value={ingestQueryPattern.forceSegmentSortByTime}
+ onValueChange={v =>
+ updatePattern({ ...ingestQueryPattern,
forceSegmentSortByTime: Boolean(v) })
+ }
optionsText={ENABLED_DISABLED_OPTIONS_TEXT}
optionsLabelElement={{ false: EXPERIMENTAL_ICON }}
/>
diff --git
a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 84e3901a566..d692c229441 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -33,7 +33,6 @@ import {
DEFAULT_SERVER_QUERY_CONTEXT,
Execution,
externalConfigToIngestQueryPattern,
- getQueryContextKey,
ingestQueryPatternToQuery,
} from '../../druid-models';
import type { Capabilities } from '../../helpers';
@@ -52,11 +51,6 @@ import { TitleFrame } from './title-frame/title-frame';
import './sql-data-loader-view.scss';
-const INITIAL_QUERY_CONTEXT: QueryContext = {
- finalizeAggregations: false,
- groupByEnableMultiValueUnnesting: false,
-};
-
interface LoaderContent extends QueryWithContext {
id?: string;
}
@@ -148,17 +142,6 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
<SchemaStep
queryString={content.queryString}
onQueryStringChange={queryString => setContent({ ...content,
queryString })}
- forceSegmentSortByTime={getQueryContextKey(
- 'forceSegmentSortByTime',
- content.queryContext || {},
- serverQueryContext,
- )}
- changeForceSegmentSortByTime={forceSegmentSortByTime =>
- setContent({
- ...content,
- queryContext: { ...content?.queryContext, forceSegmentSortByTime
},
- })
- }
enableAnalyze={false}
goToQuery={() => goToQuery(content)}
onBack={() => setContent(undefined)}
@@ -220,8 +203,6 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
initInputFormat={inputFormat}
doneButton={false}
onSet={({ inputSource, inputFormat, signature, timeExpression,
arrayMode }) => {
- const queryContext: QueryContext = { ...INITIAL_QUERY_CONTEXT };
- if (arrayMode === 'arrays') queryContext.arrayIngestMode =
'array';
setContent({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@@ -231,7 +212,6 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
arrayMode,
),
).toString(),
- queryContext,
});
}}
altText="Skip the wizard and continue with custom SQL"
diff --git
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
index 79515810262..eaccb498be9 100644
---
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
+++
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
@@ -20,7 +20,12 @@ import { Classes, Dialog } from '@blueprintjs/core';
import type { SqlExpression } from 'druid-query-toolkit';
import React, { useState } from 'react';
-import type { ArrayMode, ExternalConfig, InputFormat, InputSource } from
'../../../druid-models';
+import type {
+ ArrayIngestMode,
+ ExternalConfig,
+ InputFormat,
+ InputSource,
+} from '../../../druid-models';
import { InputFormatStep } from '../input-format-step/input-format-step';
import { InputSourceStep } from '../input-source-step/input-source-step';
@@ -32,7 +37,7 @@ export interface ConnectExternalDataDialogProps {
config: ExternalConfig,
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
- arrayMode: ArrayMode,
+ arrayMode: ArrayIngestMode,
): void;
onClose(): void;
}
diff --git
a/web-console/src/views/workbench-view/explain-dialog/explain-dialog.tsx
b/web-console/src/views/workbench-view/explain-dialog/explain-dialog.tsx
index 9b81edd6d83..d6eed657d09 100644
--- a/web-console/src/views/workbench-view/explain-dialog/explain-dialog.tsx
+++ b/web-console/src/views/workbench-view/explain-dialog/explain-dialog.tsx
@@ -46,19 +46,11 @@ import {
nonEmptyArray,
queryDruidSql,
queryDruidSqlDart,
+ wrapInExplainIfNeeded,
} from '../../../utils';
import './explain-dialog.scss';
-function isExplainQuery(query: string): boolean {
- return /^\s*EXPLAIN\sPLAN\sFOR/im.test(query);
-}
-
-function wrapInExplainIfNeeded(query: string): string {
- if (isExplainQuery(query)) return query;
- return `EXPLAIN PLAN FOR ${query}`;
-}
-
export interface QueryContextEngine extends QueryWithContext {
engine: DruidEngine;
}
diff --git
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
index 781a0196680..cd9e961ec0c 100644
---
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
+++
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
@@ -22,12 +22,18 @@ import type { SqlExpression } from 'druid-query-toolkit';
import { C, SqlColumnDeclaration, SqlType } from 'druid-query-toolkit';
import React, { useState } from 'react';
-import { ArrayModeSwitch, AutoForm, CenterMessage, LearnMore, Loader } from
'../../../components';
-import type { ArrayMode, InputFormat, InputSource } from
'../../../druid-models';
+import {
+ ArrayIngestModeSwitch,
+ AutoForm,
+ CenterMessage,
+ LearnMore,
+ Loader,
+} from '../../../components';
+import type { ArrayIngestMode, InputFormat, InputSource } from
'../../../druid-models';
import {
BATCH_INPUT_FORMAT_FIELDS,
chooseByBestTimestamp,
- DEFAULT_ARRAY_MODE,
+ DEFAULT_ARRAY_INGEST_MODE,
DETECTION_TIMESTAMP_SPEC,
getPossibleSystemFieldsForInputSource,
guessColumnTypeFromSampleResponse,
@@ -55,7 +61,7 @@ export interface InputSourceFormatAndMore {
inputFormat: InputFormat;
signature: SqlColumnDeclaration[];
timeExpression: SqlExpression | undefined;
- arrayMode: ArrayMode;
+ arrayMode: ArrayIngestMode;
}
interface InputSourceAndFormat {
@@ -94,7 +100,8 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
InputSourceAndFormat | undefined
>(isValidInputFormat(initInputFormat) ? inputSourceAndFormat : undefined);
const [selectTimestamp, setSelectTimestamp] = useState(true);
- const [arrayMode, setArrayMode] = useState<ArrayMode>(DEFAULT_ARRAY_MODE);
+ const [arrayIngestMode, setArrayIngestMode] =
+ useState<ArrayIngestMode>(DEFAULT_ARRAY_INGEST_MODE);
const [previewState] = useQueryManager<InputSourceAndFormat,
SampleResponse>({
query: inputSourceAndFormatToSample,
@@ -189,7 +196,7 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
),
),
timeExpression: selectTimestamp ?
possibleTimeExpression?.timeExpression : undefined,
- arrayMode,
+ arrayMode: arrayIngestMode,
}
: undefined;
@@ -282,7 +289,12 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
)}
</div>
<div className="bottom-controls">
- {hasArrays && <ArrayModeSwitch arrayMode={arrayMode}
changeArrayMode={setArrayMode} />}
+ {hasArrays && (
+ <ArrayIngestModeSwitch
+ arrayIngestMode={arrayIngestMode}
+ changeArrayIngestMode={setArrayIngestMode}
+ />
+ )}
{possibleTimeExpression && (
<FormGroup>
<Callout>
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
index 9ca4f300402..1c2d0b75108 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
@@ -207,35 +207,35 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
const emptyQuery = query.isEmptyQuery();
const ingestMode = query.isIngestQuery();
- const queryContext = query.queryContext;
- const numContextKeys = Object.keys(queryContext).length;
+ const queryContext = query.getQueryStringContext();
const queryParameters = query.queryParameters;
+ const effectiveDefaultContext = { ...defaultQueryContext,
...query.queryContext };
// Extract the context parts that have UI
const sqlTimeZone = queryContext.sqlTimeZone;
- const useCache = getQueryContextKey('useCache', queryContext,
defaultQueryContext);
+ const useCache = getQueryContextKey('useCache', queryContext,
effectiveDefaultContext);
const useApproximateTopN = getQueryContextKey(
'useApproximateTopN',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const useApproximateCountDistinct = getQueryContextKey(
'useApproximateCountDistinct',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const arrayIngestMode = queryContext.arrayIngestMode;
const maxParseExceptions = getQueryContextKey(
'maxParseExceptions',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const failOnEmptyInsert = getQueryContextKey(
'failOnEmptyInsert',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const finalizeAggregations = queryContext.finalizeAggregations;
const waitUntilSegmentsLoad = queryContext.waitUntilSegmentsLoad;
@@ -243,20 +243,20 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
const sqlJoinAlgorithm = getQueryContextKey(
'sqlJoinAlgorithm',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const selectDestination = getQueryContextKey(
'selectDestination',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
const durableShuffleStorage = getQueryContextKey(
'durableShuffleStorage',
queryContext,
- defaultQueryContext,
+ effectiveDefaultContext,
);
- const indexSpec: IndexSpec | undefined = deepGet(queryContext, 'indexSpec');
+ const indexSpec: IndexSpec | undefined = deepGet(query.queryContext,
'indexSpec');
const handleRun = useCallback(() => {
if (!onRun) return;
@@ -297,6 +297,10 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
onQueryChange(query.changeQueryContext(removeUndefinedValues(queryContext)));
}
+ function changeQueryStringContext(queryContext: QueryContext) {
+
onQueryChange(query.changeQueryStringContext(removeUndefinedValues(queryContext)));
+ }
+
const overloadWarning =
query.unlimited &&
(queryEngine === 'sql-native' ||
@@ -360,41 +364,22 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
/>
);
})}
-
<MenuDivider />
</>
)}
- {show('edit-query-context') && (
- <MenuItem
- icon={IconNames.PROPERTIES}
- text="Edit query context..."
- onClick={() => setEditContextDialogOpen(true)}
- label={pluralIfNeeded(numContextKeys, 'key')}
- />
- )}
- {show('define-parameters') && (
- <MenuItem
- icon={IconNames.HELP}
- text="Define parameters..."
- onClick={() => setEditParametersDialogOpen(true)}
- label={
- queryParameters ? pluralIfNeeded(queryParameters.length,
'parameter') : ''
- }
- />
- )}
{show('timezone') && (
<MenuItem
icon={IconNames.GLOBE_NETWORK}
text="Timezone"
- label={sqlTimeZone ?? defaultQueryContext.sqlTimeZone}
+ label={sqlTimeZone ?? effectiveDefaultContext.sqlTimeZone}
>
<MenuDivider title="Timezone type" />
<TimezoneMenuItems
sqlTimeZone={sqlTimeZone}
setSqlTimeZone={sqlTimeZone =>
- changeQueryContext({ ...queryContext, sqlTimeZone })
+ changeQueryStringContext({ ...queryContext,
sqlTimeZone })
}
- defaultSqlTimeZone={defaultQueryContext.sqlTimeZone}
+ defaultSqlTimeZone={effectiveDefaultContext.sqlTimeZone}
/>
<MenuItem
icon={IconNames.BLANK}
@@ -409,7 +394,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
text="Approximate COUNT(DISTINCT)"
value={useApproximateCountDistinct}
onValueChange={useApproximateCountDistinct =>
- changeQueryContext({
+ changeQueryStringContext({
...queryContext,
useApproximateCountDistinct,
})
@@ -423,7 +408,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
text="Approximate TopN"
value={useApproximateTopN}
onValueChange={useApproximateTopN =>
- changeQueryContext({
+ changeQueryStringContext({
...queryContext,
useApproximateTopN,
})
@@ -446,7 +431,9 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
icon={tickIcon(sqlJoinAlgorithm === o)}
text={SQL_JOIN_ALGORITHM_LABEL[o]}
shouldDismissPopover={false}
- onClick={() => changeQueryContext({ ...queryContext,
sqlJoinAlgorithm: o })}
+ onClick={() =>
+ changeQueryStringContext({ ...queryContext,
sqlJoinAlgorithm: o })
+ }
/>
))}
</MenuItem>
@@ -457,13 +444,49 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
icon={IconNames.BRING_DATA}
text="INSERT / REPLACE / EXTERN specific context"
>
+ <MenuItem
+ text="Array ingest mode"
+ label={
+ arrayIngestMode
+ ? ARRAY_INGEST_MODE_LABEL[arrayIngestMode]
+ : '(server default)'
+ }
+ >
+ {([undefined, 'array', 'mvd'] as (ArrayIngestMode |
undefined)[]).map(
+ (m, i) => (
+ <MenuItem
+ key={i}
+ icon={tickIcon(m === arrayIngestMode)}
+ text={
+ m
+ ? ARRAY_INGEST_MODE_DESCRIPTION[m]
+ : `(server default${
+ effectiveDefaultContext.arrayIngestMode
+ ? `:
${effectiveDefaultContext.arrayIngestMode}`
+ : ''
+ })`
+ }
+ onClick={() =>
+ changeQueryStringContext({ ...queryContext,
arrayIngestMode: m })
+ }
+ />
+ ),
+ )}
+ <MenuDivider />
+ <MenuItem
+ icon={IconNames.HELP}
+ text="Documentation"
+
href={`${getLink('DOCS')}/querying/arrays#arrayingestmode`}
+ target="_blank"
+ />
+ </MenuItem>
<MenuBoolean
text="Fail on empty insert"
value={failOnEmptyInsert}
showUndefined
undefinedEffectiveValue={false}
onValueChange={failOnEmptyInsert =>
- changeQueryContext({ ...queryContext,
failOnEmptyInsert })
+ changeQueryStringContext({ ...queryContext,
failOnEmptyInsert })
}
optionsText={ENABLED_DISABLED_OPTIONS_TEXT}
/>
@@ -473,7 +496,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
showUndefined
undefinedEffectiveValue={ingestMode}
onValueChange={waitUntilSegmentsLoad =>
- changeQueryContext({ ...queryContext,
waitUntilSegmentsLoad })
+ changeQueryStringContext({ ...queryContext,
waitUntilSegmentsLoad })
}
optionsText={ENABLED_DISABLED_OPTIONS_TEXT}
/>
@@ -484,7 +507,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
icon={tickIcon(v === maxParseExceptions)}
text={v === -1 ? '∞ (-1)' : String(v)}
onClick={() =>
- changeQueryContext({ ...queryContext,
maxParseExceptions: v })
+ changeQueryStringContext({ ...queryContext,
maxParseExceptions: v })
}
shouldDismissPopover={false}
/>
@@ -509,7 +532,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={finalizeAggregations =>
- changeQueryContext({ ...queryContext,
finalizeAggregations })
+ changeQueryStringContext({ ...queryContext,
finalizeAggregations })
}
optionsText={ENABLED_DISABLED_OPTIONS_TEXT}
/>
@@ -522,7 +545,10 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
showUndefined
undefinedEffectiveValue={!ingestMode}
onValueChange={groupByEnableMultiValueUnnesting =>
- changeQueryContext({ ...queryContext,
groupByEnableMultiValueUnnesting })
+ changeQueryStringContext({
+ ...queryContext,
+ groupByEnableMultiValueUnnesting,
+ })
}
optionsText={ENABLED_DISABLED_OPTIONS_TEXT}
/>
@@ -534,7 +560,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
text="Use cache"
value={useCache}
onValueChange={useCache =>
- changeQueryContext({
+ changeQueryStringContext({
...queryContext,
useCache,
populateCache: useCache,
@@ -563,7 +589,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
text="Durable shuffle storage"
value={durableShuffleStorage}
onValueChange={durableShuffleStorage =>
- changeQueryContext({
+ changeQueryStringContext({
...queryContext,
durableShuffleStorage,
})
@@ -588,7 +614,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
text={SELECT_DESTINATION_LABEL[o]}
shouldDismissPopover={false}
onClick={() =>
- changeQueryContext({ ...queryContext,
selectDestination: o })
+ changeQueryStringContext({ ...queryContext,
selectDestination: o })
}
/>
))}
@@ -607,6 +633,25 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
/>
</MenuItem>
)}
+ {(show('edit-query-context') || show('define-parameters')) &&
<MenuDivider />}
+ {show('edit-query-context') && (
+ <MenuItem
+ icon={IconNames.PROPERTIES}
+ text="Edit query context..."
+ onClick={() => setEditContextDialogOpen(true)}
+
label={pluralIfNeeded(Object.keys(query.queryContext).length, 'key')}
+ />
+ )}
+ {show('define-parameters') && (
+ <MenuItem
+ icon={IconNames.HELP}
+ text="Define parameters..."
+ onClick={() => setEditParametersDialogOpen(true)}
+ label={
+ queryParameters ? pluralIfNeeded(queryParameters.length,
'parameter') : ''
+ }
+ />
+ )}
</Menu>
}
>
@@ -624,53 +669,14 @@ export const RunPanel = React.memo(function
RunPanel(props: RunPanelProps) {
<MaxTasksButton
clusterCapacity={clusterCapacity}
queryContext={queryContext}
- changeQueryContext={changeQueryContext}
- defaultQueryContext={defaultQueryContext}
+ changeQueryContext={changeQueryStringContext}
+ defaultQueryContext={effectiveDefaultContext}
menuHeader={maxTasksMenuHeader}
maxTasksLabelFn={maxTasksLabelFn}
maxTasksOptions={maxTasksOptions}
fullClusterCapacityLabelFn={fullClusterCapacityLabelFn}
/>
)}
- {ingestMode && (
- <Popover
- position={Position.BOTTOM_LEFT}
- content={
- <Menu>
- {([undefined, 'array', 'mvd'] as (ArrayIngestMode |
undefined)[]).map((m, i) => (
- <MenuItem
- key={i}
- icon={tickIcon(m === arrayIngestMode)}
- text={
- m
- ? ARRAY_INGEST_MODE_DESCRIPTION[m]
- : `(server default${
- defaultQueryContext.arrayIngestMode
- ? `: ${defaultQueryContext.arrayIngestMode}`
- : ''
- })`
- }
- onClick={() => changeQueryContext({ ...queryContext,
arrayIngestMode: m })}
- />
- ))}
- <MenuDivider />
- <MenuItem
- icon={IconNames.HELP}
- text="Documentation"
- href={`${getLink('DOCS')}/querying/arrays#arrayingestmode`}
- target="_blank"
- />
- </Menu>
- }
- >
- <Button
- text={`Array ingest mode: ${
- arrayIngestMode ? ARRAY_INGEST_MODE_LABEL[arrayIngestMode] :
'(server default)'
- }`}
- rightIcon={IconNames.CARET_DOWN}
- />
- </Popover>
- )}
</ButtonGroup>
)}
{moreMenu && (
@@ -680,7 +686,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
)}
{editContextDialogOpen && (
<EditContextDialog
- initQueryContext={queryContext}
+ initQueryContext={query.queryContext}
onQueryContextChange={changeQueryContext}
onClose={() => {
setEditContextDialogOpen(false);
@@ -708,7 +714,7 @@ export const RunPanel = React.memo(function RunPanel(props:
RunPanelProps) {
{indexSpecDialogSpec && (
<IndexSpecDialog
onClose={() => setIndexSpecDialogSpec(undefined)}
- onSave={indexSpec => changeQueryContext({ ...queryContext, indexSpec
})}
+ onSave={indexSpec => changeQueryContext({ ...query.queryContext,
indexSpec })}
indexSpec={indexSpecDialogSpec}
/>
)}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
