This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new fcd65c98010 Web console: use arrayIngestMode: array (#15588)
fcd65c98010 is described below
commit fcd65c98010c8660fd6ccfa3eaa3d9908fbe572f
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Wed Jan 31 20:19:29 2024 -0800
Web console: use arrayIngestMode: array (#15588)
* Adapt to new array mode
* Feedback fixes
* fixing type detection and highlighting
* goodies
* add docs
* feedback fixes
* finish array work
* update snapshots
* typo fix
* color fixes
* small fix
* make MVDs default for now
* better sqlStringifyArrays default
* fix spec converter
* fix tests
---
licenses.yaml | 2 +-
web-console/assets/delta.png | Bin 6527 -> 26090 bytes
web-console/package-lock.json | 14 +-
web-console/package.json | 2 +-
web-console/script/create-sql-docs.js | 12 +-
web-console/src/blueprint-overrides/_index.scss | 1 +
.../_special-switch-modes.scss} | 28 +-
.../__snapshots__/array-mode-swtich.spec.tsx.snap | 47 ++++
.../array-mode-switch/array-mode-switch.tsx | 62 ++++
.../array-mode-switch/array-mode-swtich.spec.tsx} | 8 +-
web-console/src/components/index.ts | 1 +
.../__snapshots__/warning-checklist.spec.tsx.snap | 6 +-
.../warning-checklist/warning-checklist.tsx | 2 +-
.../druid-models/dimension-spec/dimension-spec.ts | 128 ++++++++-
.../external-config/external-config.ts | 10 +-
.../ingest-query-pattern.spec.ts | 139 ++++++++-
.../ingest-query-pattern/ingest-query-pattern.ts | 23 +-
.../ingestion-spec/ingestion-spec.spec.ts | 311 ++++++++++++++++++---
.../druid-models/ingestion-spec/ingestion-spec.tsx | 171 +++++++++--
.../src/druid-models/metric-spec/metric-spec.tsx | 4 +-
.../druid-models/query-context/query-context.tsx | 1 +
web-console/src/druid-models/stages/stages.ts | 2 +-
.../druid-models/timestamp-spec/timestamp-spec.tsx | 2 +-
.../workbench-query/workbench-query.spec.ts | 5 +
.../workbench-query/workbench-query.ts | 13 +-
.../__snapshots__/spec-conversion.spec.ts.snap | 23 ++
web-console/src/helpers/spec-conversion.spec.ts | 79 ++++++
web-console/src/helpers/spec-conversion.ts | 148 +++++-----
web-console/src/utils/index.tsx | 1 +
web-console/src/utils/null-mode-detection.ts | 154 ++++++++++
web-console/src/utils/object-change.ts | 8 +-
web-console/src/utils/sample-query.tsx | 30 +-
.../__snapshots__/home-view.spec.tsx.snap | 36 ++-
web-console/src/views/home-view/home-view.tsx | 2 +-
.../home-view/status-card/status-card.spec.tsx | 4 +-
.../views/home-view/status-card/status-card.tsx | 48 +++-
.../src/views/load-data-view/load-data-view.tsx | 116 +++++---
.../load-data-view/schema-table/schema-table.scss | 28 +-
.../load-data-view/schema-table/schema-table.tsx | 14 +-
.../column-editor/column-editor.tsx | 270 ++++++++++++------
.../schema-step/preview-table/preview-table.scss | 16 ++
.../schema-step/preview-table/preview-table.tsx | 19 +-
.../schema-step/schema-step.tsx | 26 +-
.../sql-data-loader-view/sql-data-loader-view.tsx | 19 +-
.../connect-external-data-dialog.tsx | 8 +-
.../input-format-step/input-format-step.tsx | 15 +-
.../src/views/workbench-view/workbench-view.tsx | 9 +-
47 files changed, 1683 insertions(+), 384 deletions(-)
diff --git a/licenses.yaml b/licenses.yaml
index f1a4cba1159..f081b8adfff 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5056,7 +5056,7 @@ license_category: binary
module: web-console
license_name: Apache License version 2.0
copyright: Imply Data
-version: 0.21.4
+version: 0.21.9
---
diff --git a/web-console/assets/delta.png b/web-console/assets/delta.png
index db535506c7b..1dc3c072dad 100644
Binary files a/web-console/assets/delta.png and b/web-console/assets/delta.png
differ
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index e4e92ef3a84..005e38f068f 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -14,7 +14,7 @@
"@blueprintjs/datetime2": "^0.9.35",
"@blueprintjs/icons": "^4.16.0",
"@blueprintjs/popover2": "^1.14.9",
- "@druid-toolkit/query": "^0.21.4",
+ "@druid-toolkit/query": "^0.21.9",
"@druid-toolkit/visuals-core": "^0.3.3",
"@druid-toolkit/visuals-react": "^0.3.3",
"ace-builds": "~1.4.14",
@@ -1074,9 +1074,9 @@
}
},
"node_modules/@druid-toolkit/query": {
- "version": "0.21.4",
- "resolved":
"https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.4.tgz";,
- "integrity":
"sha512-rZYRrtahy68ZMp3XDWa2Z3Pa28yiQMgDVHbB7ZAqynNFbKOgqS1j08LS122CRmNrvpAUyzwCnMj3Og4BvWeq1Q==",
+ "version": "0.21.9",
+ "resolved":
"https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.9.tgz";,
+ "integrity":
"sha512-g8bs9cOqyrxPzf1qdvO4FAG0rv7aBR2le+OLbF/n/KC3YXq49CUifPUYIHVfVx/jwoXKrJd1w1jVLES8OusnTg==",
"dependencies": {
"tslib": "^2.5.2"
}
@@ -20702,9 +20702,9 @@
"dev": true
},
"@druid-toolkit/query": {
- "version": "0.21.4",
- "resolved":
"https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.4.tgz";,
- "integrity":
"sha512-rZYRrtahy68ZMp3XDWa2Z3Pa28yiQMgDVHbB7ZAqynNFbKOgqS1j08LS122CRmNrvpAUyzwCnMj3Og4BvWeq1Q==",
+ "version": "0.21.9",
+ "resolved":
"https://registry.npmjs.org/@druid-toolkit/query/-/query-0.21.9.tgz";,
+ "integrity":
"sha512-g8bs9cOqyrxPzf1qdvO4FAG0rv7aBR2le+OLbF/n/KC3YXq49CUifPUYIHVfVx/jwoXKrJd1w1jVLES8OusnTg==",
"requires": {
"tslib": "^2.5.2"
}
diff --git a/web-console/package.json b/web-console/package.json
index 4e1ac583059..7fae9ef1007 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -68,7 +68,7 @@
"@blueprintjs/datetime2": "^0.9.35",
"@blueprintjs/icons": "^4.16.0",
"@blueprintjs/popover2": "^1.14.9",
- "@druid-toolkit/query": "^0.21.4",
+ "@druid-toolkit/query": "^0.21.9",
"@druid-toolkit/visuals-core": "^0.3.3",
"@druid-toolkit/visuals-react": "^0.3.3",
"ace-builds": "~1.4.14",
diff --git a/web-console/script/create-sql-docs.js
b/web-console/script/create-sql-docs.js
index 794afb06e55..82328fd74b5 100755
--- a/web-console/script/create-sql-docs.js
+++ b/web-console/script/create-sql-docs.js
@@ -29,7 +29,15 @@ const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
const initialFunctionDocs = {
TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an
external.')]],
EXTERN: [
- ['inputSource, inputFormat, rowSignature?', convertMarkdownToHtml('Reads
external data')],
+ ['inputSource, inputFormat, rowSignature?', convertMarkdownToHtml('Reads
external data.')],
+ ],
+ TYPE: [
+ [
+ 'nativeType',
+ convertMarkdownToHtml(
+ 'A purely type system modification function what wraps a Druid native
type to make it into a SQL type.',
+ ),
+ ],
],
};
@@ -70,7 +78,7 @@ const readDoc = async () => {
await fs.readFile('../docs/querying/sql-array-functions.md', 'utf-8'),
await fs.readFile('../docs/querying/sql-multivalue-string-functions.md',
'utf-8'),
await fs.readFile('../docs/querying/sql-json-functions.md', 'utf-8'),
- await fs.readFile('../docs/querying/sql-operators.md', 'utf-8')
+ await fs.readFile('../docs/querying/sql-operators.md', 'utf-8'),
].join('\n');
const lines = data.split('\n');
diff --git a/web-console/src/blueprint-overrides/_index.scss
b/web-console/src/blueprint-overrides/_index.scss
index faffe22bbb9..c345d815b80 100644
--- a/web-console/src/blueprint-overrides/_index.scss
+++ b/web-console/src/blueprint-overrides/_index.scss
@@ -24,3 +24,4 @@
@import 'components/forms/common';
@import 'components/navbar/navbar';
@import 'components/card/card';
+@import 'special-switch-modes';
diff --git a/web-console/src/views/home-view/status-card/status-card.spec.tsx
b/web-console/src/blueprint-overrides/_special-switch-modes.scss
similarity index 66%
copy from web-console/src/views/home-view/status-card/status-card.spec.tsx
copy to web-console/src/blueprint-overrides/_special-switch-modes.scss
index 5b7d419421d..267ac4a2726 100644
--- a/web-console/src/views/home-view/status-card/status-card.spec.tsx
+++ b/web-console/src/blueprint-overrides/_special-switch-modes.scss
@@ -16,16 +16,24 @@
* limitations under the License.
*/
-import { render } from '@testing-library/react';
-import React from 'react';
+.bp4-dark .bp4-switch.bp4-control {
+ &.legacy-switch {
+ input:checked ~ .bp4-control-indicator {
+ background: $orange5;
+ }
-import { StatusCard } from './status-card';
+ &:hover input:checked ~ .bp4-control-indicator {
+ background: $orange2;
+ }
+ }
-describe('StatusCard', () => {
- it('matches snapshot', () => {
- const statusCard = <StatusCard />;
+ &.danger-switch {
+ input:checked ~ .bp4-control-indicator {
+ background: $red5;
+ }
- const { container } = render(statusCard);
- expect(container.firstChild).toMatchSnapshot();
- });
-});
+ &:hover input:checked ~ .bp4-control-indicator {
+ background: $red2;
+ }
+ }
+}
diff --git
a/web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
b/web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
new file mode 100644
index 00000000000..88d7564823a
--- /dev/null
+++
b/web-console/src/components/array-mode-switch/__snapshots__/array-mode-swtich.spec.tsx.snap
@@ -0,0 +1,47 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`ArrayModeSwitch matches snapshot 1`] = `
+<div
+ class="bp4-form-group form-group-with-info"
+>
+ <div
+ class="bp4-form-content"
+ >
+ <label
+ class="bp4-control bp4-switch legacy-switch"
+ >
+ <input
+ checked=""
+ type="checkbox"
+ />
+ <span
+ class="bp4-control-indicator"
+ />
+ Store ARRAYs as MVDs
+ </label>
+ <span
+ aria-haspopup="true"
+ class="info-popover bp4-popover2-target"
+ >
+ <span
+ aria-hidden="true"
+ class="bp4-icon bp4-icon-info-sign"
+ icon="info-sign"
+ >
+ <svg
+ data-icon="info-sign"
+ height="14"
+ role="img"
+ viewBox="0 0 16 16"
+ width="14"
+ >
+ <path
+ d="M8 0C3.58 0 0 3.58 0 8s3.58 8 8 8 8-3.58 8-8-3.58-8-8-8zM7
3h2v2H7V3zm3 10H6v-1h1V7H6V6h3v6h1v1z"
+ fill-rule="evenodd"
+ />
+ </svg>
+ </span>
+ </span>
+ </div>
+</div>
+`;
diff --git a/web-console/src/components/array-mode-switch/array-mode-switch.tsx
b/web-console/src/components/array-mode-switch/array-mode-switch.tsx
new file mode 100644
index 00000000000..4d9de1274b9
--- /dev/null
+++ b/web-console/src/components/array-mode-switch/array-mode-switch.tsx
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Switch } from '@blueprintjs/core';
+import React from 'react';
+
+import type { ArrayMode } from '../../druid-models';
+import { getLink } from '../../links';
+import { ExternalLink } from '../external-link/external-link';
+import { FormGroupWithInfo } from
'../form-group-with-info/form-group-with-info';
+import { PopoverText } from '../popover-text/popover-text';
+
+export interface ArrayModeSwitchProps {
+ arrayMode: ArrayMode;
+ changeArrayMode(newArrayMode: ArrayMode): void;
+}
+
+export const ArrayModeSwitch = React.memo(function ArrayModeSwitch(props:
ArrayModeSwitchProps) {
+ const { arrayMode, changeArrayMode } = props;
+
+ return (
+ <FormGroupWithInfo
+ inlineInfo
+ info={
+ <PopoverText>
+ <p>
+ Store arrays as multi-value string columns instead of arrays. Note
that all detected
+ array elements will be coerced to strings if you choose this
option, and data will
+ behave more like a string than an array at query time. See{' '}
+ <ExternalLink href={`${getLink('DOCS')}/querying/arrays`}>array
docs</ExternalLink> and{' '}
+ <ExternalLink
href={`${getLink('DOCS')}/querying/multi-value-dimensions`}>
+ mvd docs
+ </ExternalLink>{' '}
+ for more details about the differences between arrays and
multi-value strings.
+ </p>
+ </PopoverText>
+ }
+ >
+ <Switch
+ label="Store ARRAYs as MVDs"
+ className="legacy-switch"
+ checked={arrayMode === 'multi-values'}
+ onChange={() => changeArrayMode(arrayMode === 'arrays' ?
'multi-values' : 'arrays')}
+ />
+ </FormGroupWithInfo>
+ );
+});
diff --git a/web-console/src/views/home-view/status-card/status-card.spec.tsx
b/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
similarity index 80%
copy from web-console/src/views/home-view/status-card/status-card.spec.tsx
copy to web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
index 5b7d419421d..4e738c38842 100644
--- a/web-console/src/views/home-view/status-card/status-card.spec.tsx
+++ b/web-console/src/components/array-mode-switch/array-mode-swtich.spec.tsx
@@ -19,13 +19,13 @@
import { render } from '@testing-library/react';
import React from 'react';
-import { StatusCard } from './status-card';
+import { ArrayModeSwitch } from './array-mode-switch';
-describe('StatusCard', () => {
+describe('ArrayModeSwitch', () => {
it('matches snapshot', () => {
- const statusCard = <StatusCard />;
+ const arrayInput = <ArrayModeSwitch arrayMode="multi-values"
changeArrayMode={() => {}} />;
- const { container } = render(statusCard);
+ const { container } = render(arrayInput);
expect(container.firstChild).toMatchSnapshot();
});
});
diff --git a/web-console/src/components/index.ts
b/web-console/src/components/index.ts
index a84dc063f8d..8e43f9ec94f 100644
--- a/web-console/src/components/index.ts
+++ b/web-console/src/components/index.ts
@@ -19,6 +19,7 @@
export * from './action-cell/action-cell';
export * from './action-icon/action-icon';
export * from './array-input/array-input';
+export * from './array-mode-switch/array-mode-switch';
export * from './auto-form/auto-form';
export * from './braced-text/braced-text';
export * from './center-message/center-message';
diff --git
a/web-console/src/components/warning-checklist/__snapshots__/warning-checklist.spec.tsx.snap
b/web-console/src/components/warning-checklist/__snapshots__/warning-checklist.spec.tsx.snap
index 4112eb45841..f3228aac108 100644
---
a/web-console/src/components/warning-checklist/__snapshots__/warning-checklist.spec.tsx.snap
+++
b/web-console/src/components/warning-checklist/__snapshots__/warning-checklist.spec.tsx.snap
@@ -5,7 +5,7 @@ exports[`WarningChecklist matches snapshot 1`] = `
class="warning-checklist"
>
<label
- class="bp4-control bp4-switch"
+ class="bp4-control bp4-switch danger-switch"
>
<input
type="checkbox"
@@ -16,7 +16,7 @@ exports[`WarningChecklist matches snapshot 1`] = `
Check A
</label>
<label
- class="bp4-control bp4-switch"
+ class="bp4-control bp4-switch danger-switch"
>
<input
type="checkbox"
@@ -27,7 +27,7 @@ exports[`WarningChecklist matches snapshot 1`] = `
Check B
</label>
<label
- class="bp4-control bp4-switch"
+ class="bp4-control bp4-switch danger-switch"
>
<input
type="checkbox"
diff --git a/web-console/src/components/warning-checklist/warning-checklist.tsx
b/web-console/src/components/warning-checklist/warning-checklist.tsx
index 351203fe727..fa496b26a53 100644
--- a/web-console/src/components/warning-checklist/warning-checklist.tsx
+++ b/web-console/src/components/warning-checklist/warning-checklist.tsx
@@ -40,7 +40,7 @@ export const WarningChecklist = React.memo(function
WarningChecklist(props: Warn
return (
<div className="warning-checklist">
{checks.map((check, i) => (
- <Switch key={i} onChange={() => doCheck(i)}>
+ <Switch key={i} className="danger-switch" onChange={() => doCheck(i)}>
{check}
</Switch>
))}
diff --git a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
index bb8d26a5578..f8d8f229dab 100644
--- a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
+++ b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
@@ -35,9 +35,20 @@ export interface DimensionSpec {
readonly name: string;
readonly createBitmapIndex?: boolean;
readonly multiValueHandling?: string;
+ readonly castToType?: string;
}
-const KNOWN_TYPES = ['string', 'long', 'float', 'double', 'json'];
+// This is a web console internal made up column type that represents a multi
value dimension
+const MADE_UP_MV_COLUMN_TYPE = 'mv-string';
+function makeMadeUpMvDimensionSpec(name: string): DimensionSpec {
+ return {
+ type: 'string',
+ name,
+ multiValueHandling: 'SORTED_ARRAY',
+ };
+}
+
+const KNOWN_TYPES = ['auto', 'string', 'long', 'float', 'double', 'json'];
export const DIMENSION_SPEC_FIELDS: Field<DimensionSpec>[] = [
{
name: 'name',
@@ -59,10 +70,25 @@ export const DIMENSION_SPEC_FIELDS: Field<DimensionSpec>[]
= [
},
{
name: 'multiValueHandling',
+ label: 'Multi-value handling',
type: 'string',
defined: typeIsKnown(KNOWN_TYPES, 'string'),
- defaultValue: 'SORTED_ARRAY',
- suggestions: ['SORTED_ARRAY', 'SORTED_SET', 'ARRAY'],
+ placeholder: 'unset (defaults to SORTED_ARRAY)',
+ suggestions: [undefined, 'SORTED_ARRAY', 'SORTED_SET', 'ARRAY'],
+ },
+ {
+ name: 'castToType',
+ type: 'string',
+ defined: typeIsKnown(KNOWN_TYPES, 'auto'),
+ suggestions: [
+ undefined,
+ 'STRING',
+ 'LONG',
+ 'DOUBLE',
+ 'ARRAY<STRING>',
+ 'ARRAY<LONG>',
+ 'ARRAY<DOUBLE>',
+ ],
},
];
@@ -70,8 +96,59 @@ export function getDimensionSpecName(dimensionSpec: string |
DimensionSpec): str
return typeof dimensionSpec === 'string' ? dimensionSpec :
dimensionSpec.name;
}
-export function getDimensionSpecType(dimensionSpec: string | DimensionSpec):
string {
- return typeof dimensionSpec === 'string' ? 'string' : dimensionSpec.type;
+export function getDimensionSpecColumnType(dimensionSpec: string |
DimensionSpec): string {
+ if (typeof dimensionSpec === 'string') return 'string';
+ switch (dimensionSpec.type) {
+ case 'string':
+ return typeof dimensionSpec.multiValueHandling === 'string'
+ ? MADE_UP_MV_COLUMN_TYPE
+ : 'string';
+
+ case 'auto':
+ return dimensionSpec.castToType ?? 'auto';
+
+ default:
+ return dimensionSpec.type;
+ }
+}
+
+export function getDimensionSpecUserType(
+ dimensionSpec: string | DimensionSpec,
+ identifyMv?: boolean,
+): string {
+ if (typeof dimensionSpec === 'string') return 'string';
+ switch (dimensionSpec.type) {
+ case 'string':
+ return identifyMv && typeof dimensionSpec.multiValueHandling === 'string'
+ ? 'string (multi-value)'
+ : 'string';
+
+ case 'auto':
+ return dimensionSpec.castToType ?? 'auto';
+
+ default:
+ return dimensionSpec.type;
+ }
+}
+
+export function getDimensionSpecClassType(
+ dimensionSpec: string | DimensionSpec,
+ identifyMv?: boolean,
+): string | undefined {
+ if (typeof dimensionSpec === 'string') return 'string';
+ switch (dimensionSpec.type) {
+ case 'string':
+ return identifyMv && typeof dimensionSpec.multiValueHandling === 'string'
+ ? MADE_UP_MV_COLUMN_TYPE
+ : 'string';
+
+ case 'auto':
+ if (String(dimensionSpec.castToType).startsWith('ARRAY')) return 'array';
+ return dimensionSpec.castToType?.toLowerCase();
+
+ default:
+ return dimensionSpec.type;
+ }
}
export function inflateDimensionSpec(dimensionSpec: string | DimensionSpec):
DimensionSpec {
@@ -82,18 +159,47 @@ export function inflateDimensionSpec(dimensionSpec: string
| DimensionSpec): Dim
export function getDimensionSpecs(
sampleResponse: SampleResponse,
- typeHints: Record<string, string>,
+ columnTypeHints: Record<string, string>,
guessNumericStringsAsNumbers: boolean,
+ forceMvdInsteadOfArray: boolean,
hasRollup: boolean,
): (string | DimensionSpec)[] {
return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, 'ignore'),
h => {
- const dimensionType =
- typeHints[h] ||
- guessColumnTypeFromSampleResponse(sampleResponse, h,
guessNumericStringsAsNumbers);
- if (dimensionType === 'string') return h;
+ const columnTypeHint = columnTypeHints[h];
+ const guessedColumnType = guessColumnTypeFromSampleResponse(
+ sampleResponse,
+ h,
+ guessNumericStringsAsNumbers,
+ );
+ let columnType = columnTypeHint || guessedColumnType;
+
+ if (forceMvdInsteadOfArray) {
+ if (columnType.startsWith('ARRAY')) {
+ columnType = MADE_UP_MV_COLUMN_TYPE;
+ }
+
+ if (columnType === MADE_UP_MV_COLUMN_TYPE) {
+ return makeMadeUpMvDimensionSpec(h);
+ }
+ } else {
+ // Ignore the type hint if it is MVD and we don't want to force people
into them
+ if (columnTypeHint === MADE_UP_MV_COLUMN_TYPE) {
+ columnType = guessedColumnType;
+ }
+ }
+
+ if (columnType === 'string') return h;
+ if (columnType.startsWith('ARRAY')) {
+ return {
+ type: 'auto',
+ name: h,
+ castToType: columnType.toUpperCase(),
+ };
+ }
+
if (hasRollup) return;
return {
- type: dimensionType === 'COMPLEX<json>' ? 'json' : dimensionType,
+ type: columnType === 'COMPLEX<json>' ? 'json' : columnType,
name: h,
};
});
diff --git a/web-console/src/druid-models/external-config/external-config.ts
b/web-console/src/druid-models/external-config/external-config.ts
index eb7aefefe92..2cd841a1d68 100644
--- a/web-console/src/druid-models/external-config/external-config.ts
+++ b/web-console/src/druid-models/external-config/external-config.ts
@@ -32,6 +32,7 @@ import {
import * as JSONBig from 'json-bigint-native';
import { nonEmptyArray } from '../../utils';
+import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
import type { InputFormat } from '../input-format/input-format';
import type { InputSource } from '../input-source/input-source';
@@ -127,15 +128,18 @@ export function externalConfigToTableExpression(config:
ExternalConfig): SqlExpr
export function externalConfigToInitDimensions(
config: ExternalConfig,
- isArrays: boolean[],
timeExpression: SqlExpression | undefined,
+ arrayMode: ArrayMode,
): SqlExpression[] {
return (timeExpression ? [timeExpression.as('__time')] : [])
.concat(
- filterMap(config.signature, (columnDeclaration, i) => {
+ filterMap(config.signature, columnDeclaration => {
const columnName = columnDeclaration.getColumnName();
if (timeExpression && timeExpression.containsColumnName(columnName))
return;
- return C(columnName).applyIf(isArrays[i], ex => F('MV_TO_ARRAY',
ex).as(columnName) as any);
+ return C(columnName).applyIf(
+ arrayMode === 'multi-values' &&
columnDeclaration.columnType.isArray(),
+ ex => F('ARRAY_TO_MV', ex).as(columnName),
+ );
}),
)
.slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS);
diff --git
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
index 872ae5251ba..0ac9012e273 100644
---
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
+++
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.spec.ts
@@ -21,7 +21,7 @@ import { sane, SqlQuery } from '@druid-toolkit/query';
import { fitIngestQueryPattern, ingestQueryPatternToQuery } from
'./ingest-query-pattern';
describe('ingest-query-pattern', () => {
- it('works', () => {
+ it('works with no group by', () => {
const query = SqlQuery.parse(sane`
INSERT INTO "kttm-2019"
WITH "ext" AS (
@@ -45,7 +45,142 @@ describe('ingest-query-pattern', () => {
`);
const insertQueryPattern = fitIngestQueryPattern(query);
- expect(insertQueryPattern.partitionedBy).toEqual('hour');
+
+ expect(insertQueryPattern).toMatchInlineSnapshot(`
+ Object {
+ "clusteredBy": Array [
+ 3,
+ ],
+ "destinationTableName": "kttm-2019",
+ "dimensions": Array [
+ "TIME_PARSE(\\"timestamp\\") AS __time",
+ "agent_category",
+ "agent_type",
+ "browser",
+ "browser_version",
+ ],
+ "filters": Array [],
+ "mainExternalConfig": Object {
+ "inputFormat": Object {
+ "type": "json",
+ },
+ "inputSource": Object {
+ "type": "http",
+ "uris": Array [
+ "https://example.com/data.json.gz";,
+ ],
+ },
+ "signature": Array [
+ "\\"timestamp\\" VARCHAR",
+ "\\"agent_category\\" VARCHAR",
+ "\\"agent_type\\" VARCHAR",
+ "\\"browser\\" VARCHAR",
+ "\\"browser_version\\" VARCHAR",
+ "\\"city\\" VARCHAR",
+ "\\"continent\\" VARCHAR",
+ "\\"country\\" VARCHAR",
+ "\\"version\\" VARCHAR",
+ "\\"event_type\\" VARCHAR",
+ "\\"event_subtype\\" VARCHAR",
+ "\\"loaded_image\\" VARCHAR",
+ "\\"adblock_list\\" VARCHAR",
+ "\\"forwarded_for\\" VARCHAR",
+ "\\"language\\" VARCHAR",
+ "\\"number\\" BIGINT",
+ "\\"os\\" VARCHAR",
+ "\\"path\\" VARCHAR",
+ "\\"platform\\" VARCHAR",
+ "\\"referrer\\" VARCHAR",
+ "\\"referrer_host\\" VARCHAR",
+ "\\"region\\" VARCHAR",
+ "\\"remote_address\\" VARCHAR",
+ "\\"screen\\" VARCHAR",
+ "\\"session\\" VARCHAR",
+ "\\"session_length\\" BIGINT",
+ "\\"timezone\\" VARCHAR",
+ "\\"timezone_offset\\" BIGINT",
+ "\\"window\\" VARCHAR",
+ ],
+ },
+ "mainExternalName": "ext",
+ "metrics": undefined,
+ "mode": "insert",
+ "overwriteWhere": undefined,
+ "partitionedBy": "hour",
+ }
+ `);
+
+ const query2 = ingestQueryPatternToQuery(insertQueryPattern);
+
+ expect(query2.toString()).toEqual(query.toString());
+ });
+
+ it('works with group by', () => {
+ const query = SqlQuery.parse(sane`
+ REPLACE INTO "inline_data" OVERWRITE ALL
+ WITH "ext" AS (
+ SELECT *
+ FROM TABLE(
+ EXTERN(
+
'{"type":"inline","data":"{\\"name\\":\\"Moon\\",\\"num\\":11,\\"strings\\":[\\"A\\",
\\"B\\"],\\"ints\\":[1,2],\\"floats\\":[1.1,2.2],\\"msg\\":{\\"hello\\":\\"world\\",\\"letters\\":[{\\"letter\\":\\"A\\",\\"index\\":1},{\\"letter\\":\\"B\\",\\"index\\":2}]}}\\n{\\"name\\":\\"Beam\\",\\"num\\":12,\\"strings\\":[\\"A\\",
\\"C\\"],\\"ints\\":[3,4],\\"floats\\":[3.3,4,4],\\"msg\\":{\\"where\\":\\"go\\",\\"for\\":\\"food\\",\\"letters\\":[{\\"letter\\":\\"C\\",\\"index\\":3},{\
[...]
+ '{"type":"json"}'
+ )
+ ) EXTEND ("name" VARCHAR, "num" BIGINT, "strings" VARCHAR ARRAY,
"ints" BIGINT ARRAY, "floats" DOUBLE ARRAY, "msg" TYPE('COMPLEX<json>'))
+ )
+ SELECT
+ "name",
+ "num",
+ ARRAY_TO_MV("strings") AS "strings",
+ "ints",
+ "floats",
+ COUNT(*) AS "count"
+ FROM "ext"
+ GROUP BY 1, 2, "strings", 4, 5
+ PARTITIONED BY ALL
+ `);
+
+ const insertQueryPattern = fitIngestQueryPattern(query);
+
+ expect(insertQueryPattern).toMatchInlineSnapshot(`
+ Object {
+ "clusteredBy": Array [],
+ "destinationTableName": "inline_data",
+ "dimensions": Array [
+ "\\"name\\"",
+ "\\"num\\"",
+ "ARRAY_TO_MV(\\"strings\\") AS \\"strings\\"",
+ "\\"ints\\"",
+ "\\"floats\\"",
+ ],
+ "filters": Array [],
+ "mainExternalConfig": Object {
+ "inputFormat": Object {
+ "type": "json",
+ },
+ "inputSource": Object {
+ "data":
"{\\"name\\":\\"Moon\\",\\"num\\":11,\\"strings\\":[\\"A\\",
\\"B\\"],\\"ints\\":[1,2],\\"floats\\":[1.1,2.2],\\"msg\\":{\\"hello\\":\\"world\\",\\"letters\\":[{\\"letter\\":\\"A\\",\\"index\\":1},{\\"letter\\":\\"B\\",\\"index\\":2}]}}
+ {\\"name\\":\\"Beam\\",\\"num\\":12,\\"strings\\":[\\"A\\",
\\"C\\"],\\"ints\\":[3,4],\\"floats\\":[3.3,4,4],\\"msg\\":{\\"where\\":\\"go\\",\\"for\\":\\"food\\",\\"letters\\":[{\\"letter\\":\\"C\\",\\"index\\":3},{\\"letter\\":\\"D\\",\\"index\\":4}]}}
+ ",
+ "type": "inline",
+ },
+ "signature": Array [
+ "\\"name\\" VARCHAR",
+ "\\"num\\" BIGINT",
+ "\\"strings\\" VARCHAR ARRAY",
+ "\\"ints\\" BIGINT ARRAY",
+ "\\"floats\\" DOUBLE ARRAY",
+ "\\"msg\\" TYPE('COMPLEX<json>')",
+ ],
+ },
+ "mainExternalName": "ext",
+ "metrics": Array [
+ "COUNT(*) AS \\"count\\"",
+ ],
+ "mode": "replace",
+ "overwriteWhere": undefined,
+ "partitionedBy": "all",
+ }
+ `);
const query2 = ingestQueryPatternToQuery(insertQueryPattern);
diff --git
a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
index b221411e63d..6730ecec659 100644
--- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
+++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
@@ -34,16 +34,16 @@ import {
externalConfigToTableExpression,
fitExternalConfigPattern,
} from '../external-config/external-config';
+import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
import { guessDataSourceNameFromInputSource } from
'../ingestion-spec/ingestion-spec';
export type IngestMode = 'insert' | 'replace';
-function removeMvToArray(dimension: SqlExpression): SqlExpression {
- return dimension.walk(ex =>
- ex instanceof SqlFunction && ex.getEffectiveFunctionName() ===
'MV_TO_ARRAY'
- ? ex.getArg(0)
- : ex,
- ) as SqlExpression;
+function removeTopLevelArrayToMvOrUndefined(dimension: SqlExpression):
SqlExpression | undefined {
+ const ex = dimension.getUnderlyingExpression();
+ return ex instanceof SqlFunction && ex.getEffectiveFunctionName() ===
'ARRAY_TO_MV'
+ ? ex.getArg(0)
+ : undefined;
}
export interface IngestQueryPattern {
@@ -61,9 +61,9 @@ export interface IngestQueryPattern {
export function externalConfigToIngestQueryPattern(
config: ExternalConfig,
- isArrays: boolean[],
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
+ arrayMode: ArrayMode,
): IngestQueryPattern {
return {
destinationTableName:
guessDataSourceNameFromInputSource(config.inputSource) || 'data',
@@ -71,7 +71,7 @@ export function externalConfigToIngestQueryPattern(
mainExternalName: 'ext',
mainExternalConfig: config,
filters: [],
- dimensions: externalConfigToInitDimensions(config, isArrays,
timeExpression),
+ dimensions: externalConfigToInitDimensions(config, timeExpression,
arrayMode),
partitionedBy: partitionedByHint || (timeExpression ? 'day' : 'all'),
clusteredBy: [],
};
@@ -268,7 +268,12 @@ export function ingestQueryPatternToQuery(
),
])
.applyForEach(dimensions, (query, ex) =>
- query.addSelect(preview ? removeMvToArray(ex) : ex, metrics ? {
addToGroupBy: 'end' } : {}),
+ query.addSelect(
+ ex,
+ metrics
+ ? { addToGroupBy: 'end', groupByExpression:
removeTopLevelArrayToMvOrUndefined(ex) }
+ : {},
+ ),
)
.applyForEach(metrics || [], (query, ex) => query.addSelect(ex))
.applyIf(filters.length, query =>
query.changeWhereExpression(SqlExpression.and(...filters)))
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
index 79a4d93e307..86787010f27 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.spec.ts
@@ -709,25 +709,93 @@ describe('spec utils', () => {
});
it('works for long', () => {
- expect(guessColumnTypeFromInput([1, 2, 3], false)).toEqual('long');
- expect(guessColumnTypeFromInput([1, 2, 3], true)).toEqual('long');
- expect(guessColumnTypeFromInput(['1', '2', '3'],
false)).toEqual('string');
- expect(guessColumnTypeFromInput(['1', '2', '3'], true)).toEqual('long');
+ expect(guessColumnTypeFromInput([null, 1, 2, 3], false)).toEqual('long');
+ expect(guessColumnTypeFromInput([null, 1, 2, 3], true)).toEqual('long');
+ expect(guessColumnTypeFromInput([null, '1', '2', '3'],
false)).toEqual('string');
+ expect(guessColumnTypeFromInput([null, '1', '2', '3'],
true)).toEqual('long');
});
it('works for double', () => {
- expect(guessColumnTypeFromInput([1, 2.1, 3], false)).toEqual('double');
- expect(guessColumnTypeFromInput([1, 2.1, 3], true)).toEqual('double');
- expect(guessColumnTypeFromInput(['1', '2.1', '3'],
false)).toEqual('string');
- expect(guessColumnTypeFromInput(['1', '2.1', '3'],
true)).toEqual('double');
+ expect(guessColumnTypeFromInput([null, 1, 2.1, 3],
false)).toEqual('double');
+ expect(guessColumnTypeFromInput([null, 1, 2.1, 3],
true)).toEqual('double');
+ expect(guessColumnTypeFromInput([null, '1', '2.1', '3'],
false)).toEqual('string');
+ expect(guessColumnTypeFromInput([null, '1', '2.1', '3'],
true)).toEqual('double');
});
- it('works for multi-value', () => {
- expect(guessColumnTypeFromInput(['a', ['b'], 'c'],
false)).toEqual('string');
- expect(guessColumnTypeFromInput([1, [2], 3], false)).toEqual('string');
- expect(guessColumnTypeFromInput([true, [true, 7, false], false, 'x'],
false)).toEqual(
- 'string',
- );
+ it('works for ARRAY<string>', () => {
+ expect(
+ guessColumnTypeFromInput(
+ [
+ ['A', 'B'],
+ ['A', 'C'],
+ ],
+ false,
+ ),
+ ).toEqual('ARRAY<string>');
+ });
+
+ it('works for ARRAY<long>', () => {
+ expect(
+ guessColumnTypeFromInput(
+ [
+ [1, 2],
+ [3, 4],
+ ],
+ false,
+ ),
+ ).toEqual('ARRAY<long>');
+
+ expect(
+ guessColumnTypeFromInput(
+ [
+ ['1', '2'],
+ ['3', '4'],
+ ],
+ false,
+ ),
+ ).toEqual('ARRAY<string>');
+
+ expect(
+ guessColumnTypeFromInput(
+ [
+ ['1', '2'],
+ ['3', '4'],
+ ],
+ true,
+ ),
+ ).toEqual('ARRAY<long>');
+ });
+
+ it('works for ARRAY<double>', () => {
+ expect(
+ guessColumnTypeFromInput(
+ [
+ [1.1, 2.2],
+ [3.3, 4.4],
+ ],
+ false,
+ ),
+ ).toEqual('ARRAY<double>');
+
+ expect(
+ guessColumnTypeFromInput(
+ [
+ ['1.1', '2.2'],
+ ['3.3', '4.4'],
+ ],
+ false,
+ ),
+ ).toEqual('ARRAY<string>');
+
+ expect(
+ guessColumnTypeFromInput(
+ [
+ ['1.1', '2.2'],
+ ['3.3', '4.4'],
+ ],
+ true,
+ ),
+ ).toEqual('ARRAY<double>');
});
it('works for complex arrays', () => {
@@ -751,25 +819,115 @@ describe('spec utils', () => {
expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'followers',
false)).toEqual('string');
expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'followers',
true)).toEqual('long');
expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'spend',
true)).toEqual('double');
- expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'nums',
false)).toEqual('string');
- expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'nums',
true)).toEqual('string');
+ expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'nums',
false)).toEqual('ARRAY<string>');
+ expect(guessColumnTypeFromSampleResponse(CSV_SAMPLE, 'nums',
true)).toEqual('ARRAY<long>');
});
});
- it('updateSchemaWithSample', () => {
- const withRollup = updateSchemaWithSample(ingestionSpec, JSON_SAMPLE,
'fixed', true);
+ describe('updateSchemaWithSample', () => {
+ it('works with rollup, arrays', () => {
+ const updateSpec = updateSchemaWithSample(
+ ingestionSpec,
+ JSON_SAMPLE,
+ 'fixed',
+ 'arrays',
+ true,
+ );
+ expect(updateSpec.spec).toMatchInlineSnapshot(`
+ Object {
+ "dataSchema": Object {
+ "dataSource": "wikipedia",
+ "dimensionsSpec": Object {
+ "dimensions": Array [
+ "user",
+ "id",
+ Object {
+ "castToType": "ARRAY<STRING>",
+ "name": "tags",
+ "type": "auto",
+ },
+ Object {
+ "castToType": "ARRAY<LONG>",
+ "name": "nums",
+ "type": "auto",
+ },
+ ],
+ },
+ "granularitySpec": Object {
+ "queryGranularity": "hour",
+ "rollup": true,
+ "segmentGranularity": "day",
+ },
+ "metricsSpec": Array [
+ Object {
+ "name": "count",
+ "type": "count",
+ },
+ Object {
+ "fieldName": "followers",
+ "name": "sum_followers",
+ "type": "longSum",
+ },
+ Object {
+ "fieldName": "spend",
+ "name": "sum_spend",
+ "type": "doubleSum",
+ },
+ ],
+ "timestampSpec": Object {
+ "column": "timestamp",
+ "format": "iso",
+ },
+ },
+ "ioConfig": Object {
+ "inputFormat": Object {
+ "type": "json",
+ },
+ "inputSource": Object {
+ "type": "http",
+ "uris": Array [
+ "https://website.com/wikipedia.json.gz";,
+ ],
+ },
+ "type": "index_parallel",
+ },
+ "tuningConfig": Object {
+ "forceGuaranteedRollup": true,
+ "partitionsSpec": Object {
+ "type": "hashed",
+ },
+ "type": "index_parallel",
+ },
+ }
+ `);
+ });
- expect(withRollup).toMatchInlineSnapshot(`
- Object {
- "spec": Object {
+ it('works with rollup, MVDs', () => {
+ const updateSpec = updateSchemaWithSample(
+ ingestionSpec,
+ JSON_SAMPLE,
+ 'fixed',
+ 'multi-values',
+ true,
+ );
+ expect(updateSpec.spec).toMatchInlineSnapshot(`
+ Object {
"dataSchema": Object {
"dataSource": "wikipedia",
"dimensionsSpec": Object {
"dimensions": Array [
"user",
"id",
- "tags",
- "nums",
+ Object {
+ "multiValueHandling": "SORTED_ARRAY",
+ "name": "tags",
+ "type": "string",
+ },
+ Object {
+ "multiValueHandling": "SORTED_ARRAY",
+ "name": "nums",
+ "type": "string",
+ },
],
},
"granularitySpec": Object {
@@ -817,16 +975,88 @@ describe('spec utils', () => {
},
"type": "index_parallel",
},
- },
- "type": "index_parallel",
- }
- `);
+ }
+ `);
+ });
- const noRollup = updateSchemaWithSample(ingestionSpec, JSON_SAMPLE,
'fixed', false);
+ it('works without rollup, arrays', () => {
+ const updatedSpec = updateSchemaWithSample(
+ ingestionSpec,
+ JSON_SAMPLE,
+ 'fixed',
+ 'arrays',
+ false,
+ );
+ expect(updatedSpec.spec).toMatchInlineSnapshot(`
+ Object {
+ "dataSchema": Object {
+ "dataSource": "wikipedia",
+ "dimensionsSpec": Object {
+ "dimensions": Array [
+ "user",
+ Object {
+ "name": "followers",
+ "type": "long",
+ },
+ Object {
+ "name": "spend",
+ "type": "double",
+ },
+ "id",
+ Object {
+ "castToType": "ARRAY<STRING>",
+ "name": "tags",
+ "type": "auto",
+ },
+ Object {
+ "castToType": "ARRAY<LONG>",
+ "name": "nums",
+ "type": "auto",
+ },
+ ],
+ },
+ "granularitySpec": Object {
+ "queryGranularity": "none",
+ "rollup": false,
+ "segmentGranularity": "day",
+ },
+ "timestampSpec": Object {
+ "column": "timestamp",
+ "format": "iso",
+ },
+ },
+ "ioConfig": Object {
+ "inputFormat": Object {
+ "type": "json",
+ },
+ "inputSource": Object {
+ "type": "http",
+ "uris": Array [
+ "https://website.com/wikipedia.json.gz";,
+ ],
+ },
+ "type": "index_parallel",
+ },
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "dynamic",
+ },
+ "type": "index_parallel",
+ },
+ }
+ `);
+ });
- expect(noRollup).toMatchInlineSnapshot(`
- Object {
- "spec": Object {
+ it('works without rollup, MVDs', () => {
+ const updatedSpec = updateSchemaWithSample(
+ ingestionSpec,
+ JSON_SAMPLE,
+ 'fixed',
+ 'multi-values',
+ false,
+ );
+ expect(updatedSpec.spec).toMatchInlineSnapshot(`
+ Object {
"dataSchema": Object {
"dataSource": "wikipedia",
"dimensionsSpec": Object {
@@ -841,8 +1071,16 @@ describe('spec utils', () => {
"type": "double",
},
"id",
- "tags",
- "nums",
+ Object {
+ "multiValueHandling": "SORTED_ARRAY",
+ "name": "tags",
+ "type": "string",
+ },
+ Object {
+ "multiValueHandling": "SORTED_ARRAY",
+ "name": "nums",
+ "type": "string",
+ },
],
},
"granularitySpec": Object {
@@ -873,10 +1111,9 @@ describe('spec utils', () => {
},
"type": "index_parallel",
},
- },
- "type": "index_parallel",
- }
- `);
+ }
+ `);
+ });
});
it('adjustId', () => {
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 6bf10232e89..4f4b1e9a51b 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -44,11 +44,11 @@ import {
typeIsKnown,
} from '../../utils';
import type { SampleResponse } from '../../utils/sampler';
-import type { DimensionsSpec } from '../dimension-spec/dimension-spec';
+import type { DimensionSpec, DimensionsSpec } from
'../dimension-spec/dimension-spec';
import {
+ getDimensionSpecColumnType,
getDimensionSpecName,
getDimensionSpecs,
- getDimensionSpecType,
} from '../dimension-spec/dimension-spec';
import type { IndexSpec } from '../index-spec/index-spec';
import { summarizeIndexSpec } from '../index-spec/index-spec';
@@ -73,6 +73,7 @@ export interface IngestionSpec {
readonly type: IngestionType;
readonly spec: IngestionSpecInner;
readonly context?: { taskLockType?: 'APPEND' | 'REPLACE' };
+ readonly suspended?: boolean;
}
export interface IngestionSpecInner {
@@ -233,6 +234,9 @@ export function getRequiredModule(ingestionType:
IngestionComboTypeWithExtra): s
case 'index_parallel:azure':
return 'druid-azure-extensions';
+ case 'index_parallel:delta':
+ return 'druid-deltalake-extensions';
+
case 'index_parallel:google':
return 'druid-google-extensions';
@@ -279,6 +283,8 @@ export interface DataSchema {
export type SchemaMode = 'fixed' | 'string-only-discovery' |
'type-aware-discovery';
+export type ArrayMode = 'arrays' | 'multi-values';
+
export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
if (deepGet(spec, 'spec.dataSchema.dimensionsSpec.useSchemaDiscovery') ===
true) {
return 'type-aware-discovery';
@@ -290,9 +296,66 @@ export function getSchemaMode(spec:
Partial<IngestionSpec>): SchemaMode {
return Array.isArray(dimensions) && dimensions.length === 0 ?
'string-only-discovery' : 'fixed';
}
-export function getRollup(spec: Partial<IngestionSpec>): boolean {
+export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
+ const schemaMode = getSchemaMode(spec);
+ switch (schemaMode) {
+ case 'type-aware-discovery':
+ return 'arrays';
+
+ case 'string-only-discovery':
+ return 'multi-values';
+
+ default: {
+ const dimensions: (DimensionSpec | string)[] = deepGet(
+ spec,
+ 'spec.dataSchema.dimensionsSpec.dimensions',
+ );
+
+ if (
+ dimensions.some(
+ d =>
+ typeof d === 'object' && d.type === 'auto' &&
String(d.castToType).startsWith('ARRAY'),
+ )
+ ) {
+ return 'arrays';
+ }
+
+ if (
+ dimensions.some(
+ d =>
+ typeof d === 'object' &&
+ d.type === 'string' &&
+ typeof d.multiValueHandling === 'string',
+ )
+ ) {
+ return 'multi-values';
+ }
+
+ return 'arrays';
+ }
+ }
+}
+
+export function showArrayModeToggle(spec: Partial<IngestionSpec>): boolean {
+ const schemaMode = getSchemaMode(spec);
+ if (schemaMode !== 'fixed') return false;
+
+ const dimensions: (DimensionSpec | string)[] = deepGet(
+ spec,
+ 'spec.dataSchema.dimensionsSpec.dimensions',
+ );
+
+ return dimensions.some(
+ d =>
+ typeof d === 'object' &&
+ ((d.type === 'auto' && String(d.castToType).startsWith('ARRAY')) ||
+ (d.type === 'string' && typeof d.multiValueHandling === 'string')),
+ );
+}
+
+export function getRollup(spec: Partial<IngestionSpec>, valueIfUnset = true):
boolean {
const specRollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup');
- return typeof specRollup === 'boolean' ? specRollup : true;
+ return typeof specRollup === 'boolean' ? specRollup : valueIfUnset;
}
export function getSpecType(spec: Partial<IngestionSpec>): IngestionType {
@@ -372,7 +435,7 @@ export function cleanSpec(
): Partial<IngestionSpec> {
return allowKeys(
spec,
- ['type', 'spec', 'context'].concat(allowSuspended ? ['suspended'] : []),
+ ['type', 'spec', 'context'].concat(allowSuspended ? ['suspended'] : []) as
any,
) as IngestionSpec;
}
@@ -2400,11 +2463,26 @@ function inputFormatFromType(options:
InputFormatFromTypeOptions): InputFormat {
// ------------------------
-export function guessIsArrayFromSampleResponse(
- sampleResponse: SampleResponse,
- column: string,
-): boolean {
- return sampleResponse.data.some(r => isSimpleArray(r.input?.[column]));
+function checkArray(array: any[], checkFn: (x: any) => boolean): boolean {
+ return array.every(as => checkFn(as) || (Array.isArray(as) &&
as.every(checkFn)));
+}
+
+function isIntegerOrNull(x: any): boolean {
+ return x == null || (typeof x === 'number' && Number.isInteger(x));
+}
+
+function isIntegerOrNullAcceptString(x: any): boolean {
+ return (
+ x == null || ((typeof x === 'number' || typeof x === 'string') &&
Number.isInteger(Number(x)))
+ );
+}
+
+function isNumberOrNull(x: any): boolean {
+ return x == null || (typeof x === 'number' && !isNaN(x));
+}
+
+function isNumberOrNullAcceptString(x: any): boolean {
+ return x == null || ((typeof x === 'number' || typeof x === 'string') &&
!isNaN(Number(x)));
}
export function guessColumnTypeFromInput(
@@ -2417,23 +2495,50 @@ export function guessColumnTypeFromInput(
if (!definedValues.length) return 'string';
// If we see any arrays in the input this is a multi-value dimension that
must be a string
- if (definedValues.some(v => isSimpleArray(v))) return 'string';
+ if (definedValues.some(v => isSimpleArray(v))) {
+ if (guessNumericStringsAsNumbers) {
+ if (checkArray(definedValues, isIntegerOrNullAcceptString)) {
+ return 'ARRAY<long>';
+ }
+
+ if (checkArray(definedValues, isNumberOrNullAcceptString)) {
+ return 'ARRAY<double>';
+ }
+ } else {
+ if (checkArray(definedValues, isIntegerOrNull)) {
+ return 'ARRAY<long>';
+ }
+
+ if (checkArray(definedValues, isNumberOrNull)) {
+ return 'ARRAY<double>';
+ }
+ }
+
+ return 'ARRAY<string>';
+ }
// If we see any JSON objects in the input assume COMPLEX<json>
if (definedValues.some(v => v && typeof v === 'object')) return
'COMPLEX<json>';
- if (
- definedValues.every(v => {
- return (
- (typeof v === 'number' || (guessNumericStringsAsNumbers && typeof v
=== 'string')) &&
- !isNaN(Number(v))
- );
- })
- ) {
- return definedValues.every(v => v % 1 === 0) ? 'long' : 'double';
+ if (guessNumericStringsAsNumbers) {
+ if (definedValues.every(isIntegerOrNullAcceptString)) {
+ return 'long';
+ }
+
+ if (definedValues.every(isNumberOrNullAcceptString)) {
+ return 'double';
+ }
} else {
- return 'string';
+ if (definedValues.every(isIntegerOrNull)) {
+ return 'long';
+ }
+
+ if (definedValues.every(isNumberOrNull)) {
+ return 'double';
+ }
}
+
+ return 'string';
}
export function guessColumnTypeFromSampleResponse(
@@ -2451,11 +2556,12 @@ export function
inputFormatOutputsNumericStrings(inputFormat: InputFormat | unde
return oneOf(inputFormat?.type, 'csv', 'tsv', 'regex');
}
-function getTypeHintsFromSpec(spec: Partial<IngestionSpec>): Record<string,
string> {
- const typeHints: Record<string, string> = {};
+function getColumnTypeHintsFromSpec(spec: Partial<IngestionSpec>):
Record<string, string> {
+ const columnTypeHints: Record<string, string> = {};
const currentDimensions = deepGet(spec,
'spec.dataSchema.dimensionsSpec.dimensions') || [];
for (const currentDimension of currentDimensions) {
- typeHints[getDimensionSpecName(currentDimension)] =
getDimensionSpecType(currentDimension);
+ columnTypeHints[getDimensionSpecName(currentDimension)] =
+ getDimensionSpecColumnType(currentDimension);
}
const currentMetrics = deepGet(spec, 'spec.dataSchema.metricsSpec') || [];
@@ -2463,21 +2569,22 @@ function getTypeHintsFromSpec(spec:
Partial<IngestionSpec>): Record<string, stri
const singleFieldName = getMetricSpecSingleFieldName(currentMetric);
const metricOutputType = getMetricSpecOutputType(currentMetric);
if (singleFieldName && metricOutputType) {
- typeHints[singleFieldName] = metricOutputType;
+ columnTypeHints[singleFieldName] = metricOutputType;
}
}
- return typeHints;
+ return columnTypeHints;
}
export function updateSchemaWithSample(
spec: Partial<IngestionSpec>,
sampleResponse: SampleResponse,
schemaMode: SchemaMode,
+ arrayMode: ArrayMode,
rollup: boolean,
forcePartitionInitialization = false,
): Partial<IngestionSpec> {
- const typeHints = getTypeHintsFromSpec(spec);
+ const columnTypeHints = getColumnTypeHintsFromSpec(spec);
const guessNumericStringsAsNumbers = inputFormatOutputsNumericStrings(
deepGet(spec, 'spec.ioConfig.inputFormat'),
);
@@ -2506,7 +2613,13 @@ export function updateSchemaWithSample(
newSpec = deepSet(
newSpec,
'spec.dataSchema.dimensionsSpec.dimensions',
- getDimensionSpecs(sampleResponse, typeHints,
guessNumericStringsAsNumbers, rollup),
+ getDimensionSpecs(
+ sampleResponse,
+ columnTypeHints,
+ guessNumericStringsAsNumbers,
+ arrayMode === 'multi-values',
+ rollup,
+ ),
);
break;
}
@@ -2514,7 +2627,7 @@ export function updateSchemaWithSample(
if (rollup) {
newSpec = deepSet(newSpec,
'spec.dataSchema.granularitySpec.queryGranularity', 'hour');
- const metrics = getMetricSpecs(sampleResponse, typeHints,
guessNumericStringsAsNumbers);
+ const metrics = getMetricSpecs(sampleResponse, columnTypeHints,
guessNumericStringsAsNumbers);
if (metrics) {
newSpec = deepSet(newSpec, 'spec.dataSchema.metricsSpec', metrics);
}
diff --git a/web-console/src/druid-models/metric-spec/metric-spec.tsx
b/web-console/src/druid-models/metric-spec/metric-spec.tsx
index 99cf11aaed4..367f922ca48 100644
--- a/web-console/src/druid-models/metric-spec/metric-spec.tsx
+++ b/web-console/src/druid-models/metric-spec/metric-spec.tsx
@@ -431,7 +431,7 @@ export function getMetricSpecOutputType(metricSpec:
MetricSpec): string | undefi
export function getMetricSpecs(
sampleResponse: SampleResponse,
- typeHints: Record<string, string>,
+ columnTypeHints: Record<string, string>,
guessNumericStringsAsNumbers: boolean,
): MetricSpec[] {
return [{ name: 'count', type: 'count' }].concat(
@@ -439,7 +439,7 @@ export function getMetricSpecs(
const h = s.name;
if (h === '__time') return;
const type =
- typeHints[h] ||
+ columnTypeHints[h] ||
guessColumnTypeFromSampleResponse(sampleResponse, h,
guessNumericStringsAsNumbers);
switch (type) {
case 'double':
diff --git a/web-console/src/druid-models/query-context/query-context.tsx
b/web-console/src/druid-models/query-context/query-context.tsx
index 50036f63fbd..838ab1f3a50 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -32,6 +32,7 @@ export interface QueryContext {
durableShuffleStorage?: boolean;
maxParseExceptions?: number;
groupByEnableMultiValueUnnesting?: boolean;
+ arrayIngestMode?: 'array' | 'mvd';
[key: string]: any;
}
diff --git a/web-console/src/druid-models/stages/stages.ts
b/web-console/src/druid-models/stages/stages.ts
index 7cf41a8f464..2f97e7bc96f 100644
--- a/web-console/src/druid-models/stages/stages.ts
+++ b/web-console/src/druid-models/stages/stages.ts
@@ -383,7 +383,7 @@ export class Stages {
if (!counters) return {};
return sumByKey(
filterMap(this.getCountersForStage(stage), c => {
- const warningCounter = c.warnings;
+ const warningCounter = c.warnings as Record<string, number> |
undefined;
if (!warningCounter) return;
return deleteKeys(warningCounter, ['type']);
}),
diff --git a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
index 0107fa3be6d..a88ffc3d0ef 100644
--- a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
+++ b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
@@ -30,7 +30,7 @@ import {
} from '../time/time';
import type { Transform } from '../transform-spec/transform-spec';
-const NO_SUCH_COLUMN = '!!!_no_such_column_!!!';
+export const NO_SUCH_COLUMN = '!!!_no_such_column_!!!';
export const TIME_COLUMN = '__time';
diff --git
a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts
b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts
index 36c108d7e5f..ec057b3efc7 100644
--- a/web-console/src/druid-models/workbench-query/workbench-query.spec.ts
+++ b/web-console/src/druid-models/workbench-query/workbench-query.spec.ts
@@ -289,6 +289,7 @@ describe('WorkbenchQuery', () => {
context: {
sqlOuterLimit: 1001,
sqlQueryId: 'deadbeef-9fb0-499c-8475-ea461e96a4fd',
+ sqlStringifyArrays: false,
useCache: false,
},
header: true,
@@ -316,6 +317,7 @@ describe('WorkbenchQuery', () => {
context: {
sqlOuterLimit: 1001,
sqlQueryId: 'lol',
+ sqlStringifyArrays: false,
},
header: true,
query: 'SELECT * FROM wikipedia',
@@ -354,6 +356,7 @@ describe('WorkbenchQuery', () => {
context: {
sqlOuterLimit: 1001,
sqlQueryId: 'deadbeef-9fb0-499c-8475-ea461e96a4fd',
+ sqlStringifyArrays: false,
useCache: false,
x: 1,
},
@@ -394,6 +397,7 @@ describe('WorkbenchQuery', () => {
context: {
sqlOuterLimit: 1001,
sqlQueryId: 'lol',
+ sqlStringifyArrays: false,
x: 1,
},
header: true,
@@ -422,6 +426,7 @@ describe('WorkbenchQuery', () => {
executionMode: 'async',
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
+ sqlStringifyArrays: false,
useCache: false,
waitUntilSegmentsLoad: true,
},
diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts
b/web-console/src/druid-models/workbench-query/workbench-query.ts
index c97c478379e..f027efdc0a5 100644
--- a/web-console/src/druid-models/workbench-query/workbench-query.ts
+++ b/web-console/src/druid-models/workbench-query/workbench-query.ts
@@ -45,6 +45,7 @@ import {
externalConfigToIngestQueryPattern,
ingestQueryPatternToQuery,
} from '../ingest-query-pattern/ingest-query-pattern';
+import type { ArrayMode } from '../ingestion-spec/ingestion-spec';
import type { QueryContext } from '../query-context/query-context';
const ISSUE_MARKER = '--:ISSUE:';
@@ -89,20 +90,22 @@ export class WorkbenchQuery {
static fromInitExternalConfig(
externalConfig: ExternalConfig,
- isArrays: boolean[],
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
+ arrayMode: ArrayMode,
): WorkbenchQuery {
return new WorkbenchQuery({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
externalConfig,
- isArrays,
timeExpression,
partitionedByHint,
+ arrayMode,
),
).toString(),
- queryContext: {},
+ queryContext: {
+ arrayIngestMode: 'array',
+ },
});
}
@@ -558,6 +561,10 @@ export class WorkbenchQuery {
}
}
+ if (engine === 'sql-native' || engine === 'sql-msq-task') {
+ apiQuery.context.sqlStringifyArrays ??= false;
+ }
+
if (Array.isArray(queryParameters) && queryParameters.length) {
apiQuery.parameters = queryParameters;
}
diff --git a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
index b6dceb034a6..11f2a388ad2 100644
--- a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
+++ b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
@@ -208,3 +208,26 @@ FROM "source"
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
+
+exports[`spec conversion works with ARRAY mode 1`] = `
+-- This SQL query was auto generated from an ingestion spec
+REPLACE INTO "lol" OVERWRITE ALL
+WITH "source" AS (SELECT * FROM TABLE(
+ EXTERN(
+ '{"type":"inline","data":"{\\"s\\":\\"X\\", \\"l\\":10, \\"f\\":10.1,
\\"array_s\\":[\\"A\\", \\"B\\"], \\"array_l\\":[1,2], \\"array_f\\":[1.1,2.2],
\\"mix1\\":[1, \\"lol\\"], \\"mix2\\":[1.1, 77]}\\n{\\"s\\":\\"Y\\",
\\"l\\":11, \\"f\\":11.1, \\"array_s\\":[\\"C\\", \\"D\\"],
\\"array_l\\":[3,4], \\"array_f\\":[3.3,4.4], \\"mix1\\":[2, \\"zoz\\"],
\\"mix2\\":[1.2, 88]}"}',
+ '{"type":"json"}'
+ )
+) EXTEND ("s" VARCHAR, "l" BIGINT, "f" DOUBLE, "array_s" VARCHAR ARRAY,
"array_l" BIGINT ARRAY, "array_f" DOUBLE ARRAY, "mix1" VARCHAR ARRAY, "mix2"
DOUBLE ARRAY))
+SELECT
+ TIME_PARSE('2010-01-01T00:00:00Z') AS "__time",
+ "s",
+ "l",
+ "f",
+ "array_s",
+ "array_l",
+ "array_f",
+ "mix1",
+ "mix2"
+FROM "source"
+PARTITIONED BY DAY
+`;
diff --git a/web-console/src/helpers/spec-conversion.spec.ts
b/web-console/src/helpers/spec-conversion.spec.ts
index 154bfacbb1b..bea3f730318 100644
--- a/web-console/src/helpers/spec-conversion.spec.ts
+++ b/web-console/src/helpers/spec-conversion.spec.ts
@@ -123,6 +123,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
+ arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 3,
finalizeAggregations: false,
@@ -232,6 +233,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
+ arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
finalizeAggregations: false,
});
@@ -356,6 +358,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
+ arrayIngestMode: 'array',
groupByEnableMultiValueUnnesting: false,
finalizeAggregations: false,
});
@@ -585,4 +588,80 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
});
+
+ it('works with ARRAY mode', () => {
+ const converted = convertSpecToSql({
+ type: 'index_parallel',
+ spec: {
+ ioConfig: {
+ type: 'index_parallel',
+ inputSource: {
+ type: 'inline',
+ data: '{"s":"X", "l":10, "f":10.1, "array_s":["A", "B"],
"array_l":[1,2], "array_f":[1.1,2.2], "mix1":[1, "lol"], "mix2":[1.1,
77]}\n{"s":"Y", "l":11, "f":11.1, "array_s":["C", "D"], "array_l":[3,4],
"array_f":[3.3,4.4], "mix1":[2, "zoz"], "mix2":[1.2, 88]}',
+ },
+ inputFormat: {
+ type: 'json',
+ },
+ },
+ tuningConfig: {
+ type: 'index_parallel',
+ partitionsSpec: {
+ type: 'dynamic',
+ },
+ },
+ dataSchema: {
+ dataSource: 'lol',
+ timestampSpec: {
+ column: '!!!_no_such_column_!!!',
+ missingValue: '2010-01-01T00:00:00Z',
+ },
+ dimensionsSpec: {
+ dimensions: [
+ 's',
+ {
+ type: 'long',
+ name: 'l',
+ },
+ {
+ type: 'double',
+ name: 'f',
+ },
+ {
+ type: 'auto',
+ name: 'array_s',
+ castToType: 'ARRAY<STRING>',
+ },
+ {
+ type: 'auto',
+ name: 'array_l',
+ castToType: 'ARRAY<LONG>',
+ },
+ {
+ type: 'auto',
+ name: 'array_f',
+ castToType: 'ARRAY<DOUBLE>',
+ },
+ {
+ type: 'auto',
+ name: 'mix1',
+ castToType: 'ARRAY<STRING>',
+ },
+ {
+ type: 'auto',
+ name: 'mix2',
+ castToType: 'ARRAY<DOUBLE>',
+ },
+ ],
+ },
+ granularitySpec: {
+ queryGranularity: 'none',
+ rollup: false,
+ segmentGranularity: 'day',
+ },
+ },
+ },
+ });
+
+ expect(converted.queryString).toMatchSnapshot();
+ });
});
diff --git a/web-console/src/helpers/spec-conversion.ts
b/web-console/src/helpers/spec-conversion.ts
index 9b67dac4cf6..f25406e5028 100644
--- a/web-console/src/helpers/spec-conversion.ts
+++ b/web-console/src/helpers/spec-conversion.ts
@@ -36,7 +36,7 @@ import type {
TimestampSpec,
Transform,
} from '../druid-models';
-import { inflateDimensionSpec, TIME_COLUMN, upgradeSpec } from
'../druid-models';
+import { inflateDimensionSpec, NO_SUCH_COLUMN, TIME_COLUMN, upgradeSpec } from
'../druid-models';
import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils';
export function getSpecDatasourceName(spec: Partial<IngestionSpec>): string {
@@ -76,6 +76,7 @@ export function convertSpecToSql(spec: any): QueryWithContext
{
const context: Record<string, any> = {
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
+ arrayIngestMode: 'array',
};
const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
@@ -94,17 +95,17 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
const timestampSpec: TimestampSpec = deepGet(spec,
'spec.dataSchema.timestampSpec');
if (!timestampSpec) throw new Error(`spec.dataSchema.timestampSpec is not
defined`);
- let dimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions');
- if (!Array.isArray(dimensions)) {
+ const specDimensions: (string | DimensionSpec)[] = deepGet(
+ spec,
+ 'spec.dataSchema.dimensionsSpec.dimensions',
+ );
+ if (!Array.isArray(specDimensions)) {
throw new Error(`spec.dataSchema.dimensionsSpec.dimensions must be an
array`);
}
- dimensions = dimensions.map(inflateDimensionSpec);
+ const dimensions = specDimensions.map(inflateDimensionSpec);
- let columnDeclarations: SqlColumnDeclaration[] = dimensions.map((d:
DimensionSpec) =>
- SqlColumnDeclaration.create(
- d.name,
- SqlType.fromNativeType(dimensionSpecTypeToNativeDataType(d.type)),
- ),
+ let columnDeclarations: SqlColumnDeclaration[] = dimensions.map(d =>
+ SqlColumnDeclaration.create(d.name, dimensionSpecToSqlType(d)),
);
const metricsSpec = deepGet(spec, 'spec.dataSchema.metricsSpec');
@@ -130,65 +131,71 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
let timeExpression: string;
const timestampColumnName = timestampSpec.column || 'timestamp';
- const timestampColumn = C(timestampColumnName);
- const format = timestampSpec.format || 'auto';
const timeTransform = transforms.find(t => t.name === TIME_COLUMN);
- if (timeTransform) {
- timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
- } else if (timestampColumnName === TIME_COLUMN) {
- timeExpression = String(timestampColumn);
-
columnDeclarations.unshift(SqlColumnDeclaration.create(timestampColumnName,
SqlType.BIGINT));
+ if (timestampColumnName === NO_SUCH_COLUMN) {
+ timeExpression = timestampSpec.missingValue
+ ? `TIME_PARSE(${L(timestampSpec.missingValue)})`
+ : `TIMESTAMP '1970-01-01'`;
} else {
- let timestampColumnType: SqlType;
- switch (format) {
- case 'auto':
- timestampColumnType = SqlType.VARCHAR;
- timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0
THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE
TIME_PARSE(TRIM(${timestampColumn})) END`;
- break;
-
- case 'iso':
- timestampColumnType = SqlType.VARCHAR;
- timeExpression = `TIME_PARSE(${timestampColumn})`;
- break;
-
- case 'posix':
- timestampColumnType = SqlType.BIGINT;
- timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`;
- break;
-
- case 'millis':
- timestampColumnType = SqlType.BIGINT;
- timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`;
- break;
-
- case 'micro':
- timestampColumnType = SqlType.BIGINT;
- timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`;
- break;
-
- case 'nano':
- timestampColumnType = SqlType.BIGINT;
- timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`;
- break;
-
- default:
- timestampColumnType = SqlType.VARCHAR;
- timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`;
- break;
+ const timestampColumn = C(timestampColumnName);
+ const format = timestampSpec.format || 'auto';
+ if (timeTransform) {
+ timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
+ } else if (timestampColumnName === TIME_COLUMN) {
+ timeExpression = String(timestampColumn);
+
columnDeclarations.unshift(SqlColumnDeclaration.create(timestampColumnName,
SqlType.BIGINT));
+ } else {
+ let timestampColumnType: SqlType;
+ switch (format) {
+ case 'auto':
+ timestampColumnType = SqlType.VARCHAR;
+ timeExpression = `CASE WHEN CAST(${timestampColumn} AS BIGINT) > 0
THEN MILLIS_TO_TIMESTAMP(CAST(${timestampColumn} AS BIGINT)) ELSE
TIME_PARSE(TRIM(${timestampColumn})) END`;
+ break;
+
+ case 'iso':
+ timestampColumnType = SqlType.VARCHAR;
+ timeExpression = `TIME_PARSE(${timestampColumn})`;
+ break;
+
+ case 'posix':
+ timestampColumnType = SqlType.BIGINT;
+ timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} * 1000)`;
+ break;
+
+ case 'millis':
+ timestampColumnType = SqlType.BIGINT;
+ timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn})`;
+ break;
+
+ case 'micro':
+ timestampColumnType = SqlType.BIGINT;
+ timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000)`;
+ break;
+
+ case 'nano':
+ timestampColumnType = SqlType.BIGINT;
+ timeExpression = `MILLIS_TO_TIMESTAMP(${timestampColumn} / 1000000)`;
+ break;
+
+ default:
+ timestampColumnType = SqlType.VARCHAR;
+ timeExpression = `TIME_PARSE(${timestampColumn}, ${L(format)})`;
+ break;
+ }
+ columnDeclarations.unshift(
+ SqlColumnDeclaration.create(timestampColumnName, timestampColumnType),
+ );
}
- columnDeclarations.unshift(
- SqlColumnDeclaration.create(timestampColumnName, timestampColumnType),
- );
- }
- if (timestampSpec.missingValue) {
- timeExpression = `COALESCE(${timeExpression},
TIME_PARSE(${L(timestampSpec.missingValue)}))`;
- }
+ if (timestampSpec.missingValue) {
+ timeExpression = `COALESCE(${timeExpression},
TIME_PARSE(${L(timestampSpec.missingValue)}))`;
+ }
- timeExpression = convertQueryGranularity(
- timeExpression,
- deepGet(spec, 'spec.dataSchema.granularitySpec.queryGranularity'),
- );
+ timeExpression = convertQueryGranularity(
+ timeExpression,
+ deepGet(spec, 'spec.dataSchema.granularitySpec.queryGranularity'),
+ );
+ }
lines.push(`-- This SQL query was auto generated from an ingestion spec`);
@@ -385,13 +392,20 @@ const QUERY_GRANULARITY_MAP: Record<string, string> = {
year: `TIME_FLOOR(?, 'P1Y')`,
};
-function dimensionSpecTypeToNativeDataType(dimensionSpecType: string): string {
- switch (dimensionSpecType) {
+function dimensionSpecToSqlType(dimensionSpec: DimensionSpec): SqlType {
+ switch (dimensionSpec.type) {
+ case 'auto':
+ if (dimensionSpec.castToType) {
+ return SqlType.fromNativeType(dimensionSpec.castToType);
+ } else {
+ return SqlType.VARCHAR;
+ }
+
case 'json':
- return 'COMPLEX<json>';
+ return SqlType.fromNativeType('COMPLEX<json>');
default:
- return dimensionSpecType;
+ return SqlType.fromNativeType(dimensionSpec.type);
}
}
diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx
index e7dc33e7a65..244eec5372c 100644
--- a/web-console/src/utils/index.tsx
+++ b/web-console/src/utils/index.tsx
@@ -27,6 +27,7 @@ export * from './general';
export * from './intermediate-query-state';
export * from './local-storage-backed-visibility';
export * from './local-storage-keys';
+export * from './null-mode-detection';
export * from './object-change';
export * from './query-action';
export * from './query-manager';
diff --git a/web-console/src/utils/null-mode-detection.ts
b/web-console/src/utils/null-mode-detection.ts
new file mode 100644
index 00000000000..1f72309391e
--- /dev/null
+++ b/web-console/src/utils/null-mode-detection.ts
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface NullModeDetection {
+ useDefaultValueForNull?: boolean; // Modern expected value: false
+ useStrictBooleans?: boolean; // Modern expected value: true
+ useThreeValueLogicForNativeFilters?: boolean; // Modern expected value: true
+}
+
+const NULL_PROPERTY_DESCRIPTION: { prop: keyof NullModeDetection; newDefault:
boolean }[] = [
+ {
+ prop: 'useDefaultValueForNull',
+ newDefault: false,
+ },
+ {
+ prop: 'useStrictBooleans',
+ newDefault: true,
+ },
+ {
+ prop: 'useThreeValueLogicForNativeFilters',
+ newDefault: true,
+ },
+];
+
+export const NULL_DETECTION_QUERY = {
+ queryType: 'timeseries',
+ dataSource: {
+ type: 'inline',
+ columnNames: ['n', 'v'],
+ columnTypes: ['STRING', 'LONG'],
+ rows: [
+ [null, 2],
+ ['A', 3],
+ ],
+ },
+ intervals: '1000/2000',
+ granularity: 'all',
+ aggregations: [
+ {
+ type: 'filtered',
+ name: 'sum_b', // There is no B so this will produce nothing but whether
it gives null or 0 lets us determine the setting for useDefaultValueForNull
+ aggregator: {
+ type: 'longSum',
+ name: '_',
+ fieldName: 'v',
+ },
+ filter: {
+ type: 'selector',
+ dimension: 'n',
+ value: 'B',
+ },
+ },
+ {
+ type: 'filtered',
+ name: 'sum_not_b', // Whether `!= 'B'` matches the null row lets us
determine the setting for useThreeValueLogicForNativeFilters
+ aggregator: {
+ type: 'longSum',
+ name: '_',
+ fieldName: 'v',
+ },
+ filter: {
+ type: 'not',
+ field: {
+ type: 'selector',
+ dimension: 'n',
+ value: 'B',
+ },
+ },
+ },
+ ],
+ postAggregations: [
+ {
+ type: 'expression',
+ name: 'two_or_three', // useStrictBooleans will force this expression to
1 (true)
+ expression: '2 || 3',
+ },
+ ],
+};
+
+export interface NullDetectionQueryResult {
+ sum_b: number | null;
+ sum_not_b: number | null;
+ two_or_three: number | null;
+}
+
+export function nullDetectionQueryResultDecoder(
+ result: NullDetectionQueryResult,
+): NullModeDetection {
+ let useThreeValueLogicForNativeFilters;
+ if (result.sum_not_b === 3) useThreeValueLogicForNativeFilters = true;
+ if (result.sum_not_b === 5) useThreeValueLogicForNativeFilters = false;
+
+ let useStrictBooleans;
+ if (result.two_or_three === 1) useStrictBooleans = true;
+ if (result.two_or_three === 2) useStrictBooleans = false;
+
+ let useDefaultValueForNull;
+ if (result.sum_b === 0) useDefaultValueForNull = true;
+ if (result.sum_b === null) useDefaultValueForNull = false;
+
+ return {
+ useThreeValueLogicForNativeFilters,
+ useStrictBooleans,
+ useDefaultValueForNull,
+ };
+}
+
+export function summarizeNullModeDetection({
+ useThreeValueLogicForNativeFilters,
+ useStrictBooleans,
+ useDefaultValueForNull,
+}: NullModeDetection): string {
+ if (
+ typeof useThreeValueLogicForNativeFilters === 'undefined' ||
+ typeof useStrictBooleans === 'undefined' ||
+ typeof useDefaultValueForNull === 'undefined'
+ ) {
+ return 'Could not determine NULL mode';
+ }
+
+ if (useThreeValueLogicForNativeFilters && useStrictBooleans &&
!useDefaultValueForNull) {
+ return 'SQL compliant NULL mode';
+ }
+
+ if (!useThreeValueLogicForNativeFilters && !useStrictBooleans &&
useDefaultValueForNull) {
+ return 'Legacy NULL mode';
+ }
+
+ return 'Mixed NULL mode';
+}
+
+export function explainNullModeDetection(nullMode: NullModeDetection):
string[] {
+ return NULL_PROPERTY_DESCRIPTION.map(({ prop, newDefault }) => {
+ const v = nullMode[prop];
+ return `${prop}: ${typeof v === 'undefined' ? 'Not detected' : v}${
+ v === newDefault ? ' (SQL compliant)' : ''
+ }${v === !newDefault ? ' (Legacy mode)' : ''}`;
+ });
+}
diff --git a/web-console/src/utils/object-change.ts
b/web-console/src/utils/object-change.ts
index 1e06fc01fb5..9b29cda5dbf 100644
--- a/web-console/src/utils/object-change.ts
+++ b/web-console/src/utils/object-change.ts
@@ -159,8 +159,8 @@ export function deepExtend<T extends Record<string,
any>>(target: T, diff: Recor
return newValue;
}
-export function allowKeys(obj: Record<string, any>, keys: string[]):
Record<string, any> {
- const newObj: Record<string, any> = {};
+export function allowKeys<T>(obj: T, keys: (keyof T)[]): T {
+ const newObj: T = {} as any;
for (const key of keys) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
newObj[key] = obj[key];
@@ -169,8 +169,8 @@ export function allowKeys(obj: Record<string, any>, keys:
string[]): Record<stri
return newObj;
}
-export function deleteKeys(obj: Record<string, any>, keys: string[]):
Record<string, any> {
- const newObj: Record<string, any> = { ...obj };
+export function deleteKeys<T>(obj: T, keys: (keyof T)[]): T {
+ const newObj: T = { ...obj };
for (const key of keys) {
delete newObj[key];
}
diff --git a/web-console/src/utils/sample-query.tsx
b/web-console/src/utils/sample-query.tsx
index 44747395cf7..2c32618fac1 100644
--- a/web-console/src/utils/sample-query.tsx
+++ b/web-console/src/utils/sample-query.tsx
@@ -31,7 +31,7 @@ import {
import { oneOf } from './general';
-const SAMPLE_ARRAY_SEPARATOR = '-3432-d401-';
+const SAMPLE_ARRAY_SEPARATOR = '<#>'; // Note that this is a regexp so don't
add anything that is a special regexp thing
function nullForColumn(column: Column): LiteralValue {
return oneOf(column.sqlType, 'BIGINT', 'DOUBLE', 'FLOAT') ? 0 : '';
@@ -39,7 +39,6 @@ function nullForColumn(column: Column): LiteralValue {
export function sampleDataToQuery(sample: QueryResult): SqlQuery {
const { header, rows } = sample;
- const arrayIndexes: Record<number, boolean> = {};
return SqlQuery.create(
new SqlAlias({
expression: SqlValues.create(
@@ -48,12 +47,14 @@ export function sampleDataToQuery(sample: QueryResult):
SqlQuery {
row.map((r, i) => {
if (header[i].nativeType === 'COMPLEX<json>') {
return L(JSON.stringify(r));
- } else if (Array.isArray(r)) {
- arrayIndexes[i] = true;
+ } else if (String(header[i].sqlType).endsWith(' ARRAY')) {
return L(r.join(SAMPLE_ARRAY_SEPARATOR));
- } else {
+ } else if (r == null || typeof r === 'object') {
// Avoid actually using NULL literals as they create havoc in
the VALUES type system and throw errors.
- return L(r == null ? nullForColumn(header[i]) : r);
+ // Also, cleanup array if it happens to get here, it shouldn't.
+ return L(nullForColumn(header[i]));
+ } else {
+ return L(r);
}
}),
),
@@ -63,16 +64,19 @@ export function sampleDataToQuery(sample: QueryResult):
SqlQuery {
columns: SqlColumnList.create(header.map((_, i) =>
RefName.create(`c${i}`, true))),
}),
).changeSelectExpressions(
- header.map((h, i) => {
+ header.map(({ name, nativeType, sqlType }, i) => {
let ex: SqlExpression = C(`c${i}`);
- if (h.nativeType === 'COMPLEX<json>') {
+ if (nativeType === 'COMPLEX<json>') {
ex = F('PARSE_JSON', ex);
- } else if (arrayIndexes[i]) {
- ex = F('STRING_TO_MV', ex, SAMPLE_ARRAY_SEPARATOR);
- } else if (h.sqlType) {
- ex = ex.cast(h.sqlType);
+ } else if (sqlType && sqlType.endsWith(' ARRAY')) {
+ ex = F('STRING_TO_ARRAY', ex, SAMPLE_ARRAY_SEPARATOR);
+ if (sqlType !== 'VARCHAR ARRAY') {
+ ex = ex.cast(sqlType);
+ }
+ } else if (sqlType) {
+ ex = ex.cast(sqlType);
}
- return ex.as(h.name, true);
+ return ex.as(name, true);
}),
);
}
diff --git
a/web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
b/web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
index 640fe8900be..c78b0a80f7e 100644
--- a/web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
+++ b/web-console/src/views/home-view/__snapshots__/home-view.spec.tsx.snap
@@ -4,7 +4,17 @@ exports[`HomeView matches snapshot (coordinator) 1`] = `
<div
className="home-view app-view"
>
- <Memo(StatusCard) />
+ <Memo(StatusCard)
+ capabilities={
+ Capabilities {
+ "clusterCapacity": undefined,
+ "coordinator": true,
+ "multiStageQuery": false,
+ "overlord": false,
+ "queryType": "none",
+ }
+ }
+ />
<React.Fragment>
<Memo(DatasourcesCard)
capabilities={
@@ -58,7 +68,17 @@ exports[`HomeView matches snapshot (full) 1`] = `
<div
className="home-view app-view"
>
- <Memo(StatusCard) />
+ <Memo(StatusCard)
+ capabilities={
+ Capabilities {
+ "clusterCapacity": undefined,
+ "coordinator": true,
+ "multiStageQuery": true,
+ "overlord": true,
+ "queryType": "nativeAndSql",
+ }
+ }
+ />
<React.Fragment>
<Memo(DatasourcesCard)
capabilities={
@@ -136,7 +156,17 @@ exports[`HomeView matches snapshot (overlord) 1`] = `
<div
className="home-view app-view"
>
- <Memo(StatusCard) />
+ <Memo(StatusCard)
+ capabilities={
+ Capabilities {
+ "clusterCapacity": undefined,
+ "coordinator": false,
+ "multiStageQuery": false,
+ "overlord": true,
+ "queryType": "none",
+ }
+ }
+ />
<React.Fragment>
<Memo(SupervisorsCard)
capabilities={
diff --git a/web-console/src/views/home-view/home-view.tsx
b/web-console/src/views/home-view/home-view.tsx
index 4a9158b9d4c..bfd110d4a8d 100644
--- a/web-console/src/views/home-view/home-view.tsx
+++ b/web-console/src/views/home-view/home-view.tsx
@@ -39,7 +39,7 @@ export const HomeView = React.memo(function HomeView(props:
HomeViewProps) {
return (
<div className="home-view app-view">
- <StatusCard />
+ <StatusCard capabilities={capabilities} />
{capabilities.hasSqlOrCoordinatorAccess() && (
<>
<DatasourcesCard capabilities={capabilities} />
diff --git a/web-console/src/views/home-view/status-card/status-card.spec.tsx
b/web-console/src/views/home-view/status-card/status-card.spec.tsx
index 5b7d419421d..bb3d4500ff3 100644
--- a/web-console/src/views/home-view/status-card/status-card.spec.tsx
+++ b/web-console/src/views/home-view/status-card/status-card.spec.tsx
@@ -19,11 +19,13 @@
import { render } from '@testing-library/react';
import React from 'react';
+import { Capabilities } from '../../../helpers';
+
import { StatusCard } from './status-card';
describe('StatusCard', () => {
it('matches snapshot', () => {
- const statusCard = <StatusCard />;
+ const statusCard = <StatusCard capabilities={Capabilities.FULL} />;
const { container } = render(statusCard);
expect(container.firstChild).toMatchSnapshot();
diff --git a/web-console/src/views/home-view/status-card/status-card.tsx
b/web-console/src/views/home-view/status-card/status-card.tsx
index 09ea1f375d1..f1db97ac6c6 100644
--- a/web-console/src/views/home-view/status-card/status-card.tsx
+++ b/web-console/src/views/home-view/status-card/status-card.tsx
@@ -17,12 +17,23 @@
*/
import { IconNames } from '@blueprintjs/icons';
+import { Tooltip2 } from '@blueprintjs/popover2';
import React, { useState } from 'react';
import { StatusDialog } from '../../../dialogs/status-dialog/status-dialog';
+import type { Capabilities } from '../../../helpers';
import { useQueryManager } from '../../../hooks';
import { Api } from '../../../singletons';
-import { pluralIfNeeded } from '../../../utils';
+import type { NullModeDetection } from '../../../utils';
+import {
+ deepGet,
+ explainNullModeDetection,
+ NULL_DETECTION_QUERY,
+ nullDetectionQueryResultDecoder,
+ pluralIfNeeded,
+ queryDruidRune,
+ summarizeNullModeDetection,
+} from '../../../utils';
import { HomeViewCard } from '../home-view-card/home-view-card';
interface StatusSummary {
@@ -30,11 +41,15 @@ interface StatusSummary {
extensionCount: number;
}
-export interface StatusCardProps {}
+export interface StatusCardProps {
+ capabilities: Capabilities;
+}
-export const StatusCard = React.memo(function StatusCard(_props:
StatusCardProps) {
+export const StatusCard = React.memo(function StatusCard(props:
StatusCardProps) {
+ const { capabilities } = props;
const [showStatusDialog, setShowStatusDialog] = useState(false);
const [statusSummaryState] = useQueryManager<null, StatusSummary>({
+ initQuery: null,
processQuery: async () => {
const statusResp = await Api.instance.get('/status');
return {
@@ -42,10 +57,19 @@ export const StatusCard = React.memo(function
StatusCard(_props: StatusCardProps
extensionCount: statusResp.data.modules.length,
};
},
- initQuery: null,
+ });
+
+ const [nullModeDetectionState] = useQueryManager<Capabilities,
NullModeDetection>({
+ initQuery: capabilities,
+ processQuery: async capabilities => {
+ if (!capabilities.hasQuerying()) return {};
+ const nullDetectionResponse = await queryDruidRune(NULL_DETECTION_QUERY);
+ return nullDetectionQueryResultDecoder(deepGet(nullDetectionResponse,
'0.result'));
+ },
});
const statusSummary = statusSummaryState.data;
+ const nullModeDetection = nullModeDetectionState.data;
return (
<>
<HomeViewCard
@@ -64,6 +88,22 @@ export const StatusCard = React.memo(function
StatusCard(_props: StatusCardProps
<p>{`${pluralIfNeeded(statusSummary.extensionCount, 'extension')}
loaded`}</p>
</>
)}
+ {nullModeDetection && (
+ <Tooltip2
+ content={
+ <div>
+ <p>
+ <strong>Null related server properties</strong>
+ </p>
+ {explainNullModeDetection(nullModeDetection).map((line, i) => (
+ <p key={i}>{line}</p>
+ ))}
+ </div>
+ }
+ >
+ <p>{summarizeNullModeDetection(nullModeDetection)}</p>
+ </Tooltip2>
+ )}
</HomeViewCard>
{showStatusDialog && (
<StatusDialog
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 18fdf045a5b..942e5d0aded 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -45,6 +45,7 @@ import type { JSX } from 'react';
import React from 'react';
import {
+ ArrayModeSwitch,
AutoForm,
CenterMessage,
ClearableInput,
@@ -56,6 +57,7 @@ import {
} from '../../components';
import { AlertDialog, AsyncActionDialog } from '../../dialogs';
import type {
+ ArrayMode,
DimensionSpec,
DruidFilter,
FlattenField,
@@ -85,6 +87,7 @@ import {
FILTER_FIELDS,
FILTERS_FIELDS,
FLATTEN_FIELD_FIELDS,
+ getArrayMode,
getDimensionSpecName,
getIngestionComboType,
getIngestionImage,
@@ -118,6 +121,7 @@ import {
possibleDruidFormatForValues,
PRIMARY_PARTITION_RELATED_FORM_FIELDS,
removeTimestampTransform,
+ showArrayModeToggle,
splitFilter,
STREAMING_INPUT_FORMAT_FIELDS,
TIME_COLUMN,
@@ -201,8 +205,6 @@ import {
import './load-data-view.scss';
-const DEFAULT_ROLLUP_SETTING = false;
-
function showRawLine(line: SampleEntry): string {
if (!line.parsed) return 'No parse';
const raw = line.parsed.raw;
@@ -283,6 +285,14 @@ function getTimestampSpec(sampleResponse: SampleResponse |
null): TimestampSpec
return chooseByBestTimestamp(timestampSpecs) || CONSTANT_TIMESTAMP_SPEC;
}
+function initializeSchemaWithSampleIfNeeded(
+ spec: Partial<IngestionSpec>,
+ sample: SampleResponse,
+): Partial<IngestionSpec> {
+ if (deepGet(spec, 'spec.dataSchema.dimensionsSpec')) return spec;
+ return updateSchemaWithSample(spec, sample, 'fixed', 'multi-values',
getRollup(spec, false));
+}
+
type Step =
| 'welcome'
| 'connect'
@@ -364,6 +374,7 @@ export interface LoadDataViewState {
showResetConfirm: boolean;
newRollup?: boolean;
newSchemaMode?: SchemaMode;
+ newArrayMode?: ArrayMode;
// welcome
overlordModules?: string[];
@@ -1642,6 +1653,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
if (selectedFlattenField) {
return (
<FormEditor
+ key={selectedFlattenField.index}
fields={FLATTEN_FIELD_FIELDS}
initValue={selectedFlattenField.value}
onClose={this.resetSelected}
@@ -1999,19 +2011,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
disabled: !transformQueryState.data,
onNextStep: () => {
if (!transformQueryState.data) return false;
-
- let newSpec = spec;
- if (!deepGet(newSpec, 'spec.dataSchema.dimensionsSpec')) {
- const currentRollup = deepGet(newSpec,
'spec.dataSchema.granularitySpec.rollup');
- newSpec = updateSchemaWithSample(
- newSpec,
- transformQueryState.data,
- 'fixed',
- typeof currentRollup === 'boolean' ? currentRollup :
DEFAULT_ROLLUP_SETTING,
- );
- }
-
- this.updateSpec(newSpec);
+ this.updateSpec(initializeSchemaWithSampleIfNeeded(spec,
transformQueryState.data));
return true;
},
})}
@@ -2034,6 +2034,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
if (selectedTransform) {
return (
<FormEditor
+ key={selectedTransform.index}
fields={TRANSFORM_FIELDS}
initValue={selectedTransform.value}
onClose={this.resetSelected}
@@ -2202,19 +2203,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
{this.renderNextBar({
onNextStep: () => {
if (!filterQueryState.data) return false;
-
- let newSpec = spec;
- if (!deepGet(newSpec, 'spec.dataSchema.dimensionsSpec')) {
- const currentRollup = deepGet(newSpec,
'spec.dataSchema.granularitySpec.rollup');
- newSpec = updateSchemaWithSample(
- newSpec,
- filterQueryState.data,
- 'fixed',
- typeof currentRollup === 'boolean' ? currentRollup :
DEFAULT_ROLLUP_SETTING,
- );
- }
-
- this.updateSpec(newSpec);
+ this.updateSpec(initializeSchemaWithSampleIfNeeded(spec,
filterQueryState.data));
return true;
},
})}
@@ -2234,6 +2223,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
return (
<FormEditor
+ key={selectedFilter.index}
fields={FILTER_FIELDS}
initValue={selectedFilter.value}
showCustom={f => !KNOWN_FILTER_TYPES.includes(f.type || '')}
@@ -2321,6 +2311,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
selectedAutoDimension || selectedDimensionSpec || selectedMetricSpec,
);
const schemaMode = getSchemaMode(spec);
+ const arrayMode = getArrayMode(spec);
let mainFill: JSX.Element | string;
if (schemaQueryState.isInit()) {
@@ -2396,6 +2387,16 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
label="Explicitly specify schema"
/>
</FormGroupWithInfo>
+ {showArrayModeToggle(spec) && (
+ <ArrayModeSwitch
+ arrayMode={arrayMode}
+ changeArrayMode={newArrayMode => {
+ this.setState({
+ newArrayMode,
+ });
+ }}
+ />
+ )}
{schemaMode !== 'fixed' && (
<AutoForm
fields={[
@@ -2523,7 +2524,8 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
{this.renderDimensionSpecControls()}
{this.renderMetricSpecControls()}
{this.renderChangeRollupAction()}
- {this.renderChangeDimensionModeAction()}
+ {this.renderChangeSchemaModeAction()}
+ {this.renderChangeArrayModeAction()}
</div>
{this.renderNextBar({
disabled: !schemaQueryState.data,
@@ -2618,7 +2620,14 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
action={async () => {
const sampleResponse = await sampleForTransform(spec, cacheRows);
this.updateSpec(
- updateSchemaWithSample(spec, sampleResponse, getSchemaMode(spec),
newRollup, true),
+ updateSchemaWithSample(
+ spec,
+ sampleResponse,
+ getSchemaMode(spec),
+ getArrayMode(spec),
+ newRollup,
+ true,
+ ),
);
}}
confirmButtonText={`Yes - ${newRollup ? 'enable' : 'disable'} rollup`}
@@ -2633,7 +2642,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
);
}
- renderChangeDimensionModeAction() {
+ renderChangeSchemaModeAction() {
const { newSchemaMode, spec, cacheRows } = this.state;
if (!newSchemaMode || !cacheRows) return;
const autoDetect = newSchemaMode !== 'fixed';
@@ -2643,7 +2652,13 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
action={async () => {
const sampleResponse = await sampleForTransform(spec, cacheRows);
this.updateSpec(
- updateSchemaWithSample(spec, sampleResponse, newSchemaMode,
getRollup(spec)),
+ updateSchemaWithSample(
+ spec,
+ sampleResponse,
+ newSchemaMode,
+ getArrayMode(spec),
+ getRollup(spec),
+ ),
);
}}
confirmButtonText={`Yes - ${autoDetect ? 'auto detect' : 'explicitly
define'} schema`}
@@ -2694,6 +2709,41 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
);
}
+ renderChangeArrayModeAction() {
+ const { newArrayMode, spec, cacheRows } = this.state;
+ if (!newArrayMode || !cacheRows) return;
+ const multiValues = newArrayMode === 'multi-values';
+
+ return (
+ <AsyncActionDialog
+ action={async () => {
+ const sampleResponse = await sampleForTransform(spec, cacheRows);
+ this.updateSpec(
+ updateSchemaWithSample(
+ spec,
+ sampleResponse,
+ getSchemaMode(spec),
+ newArrayMode,
+ getRollup(spec),
+ ),
+ );
+ }}
+ confirmButtonText={`Yes - ${multiValues ? 'use MVDs' : 'ARRAYs'}`}
+ successText={`Array mode changed to ${multiValues ? 'multi-values' :
'arrays'}.`}
+ failText="Could not change array mode"
+ intent={Intent.WARNING}
+ onClose={() => this.setState({ newArrayMode: undefined })}
+ >
+ <p>
+ {multiValues
+ ? `Are you sure you want to use multi-value dimensions?`
+ : `Are you sure you want to use ARRAYs?`}
+ </p>
+ <p>Making this change will reset all schema configuration done so
far.</p>
+ </AsyncActionDialog>
+ );
+ }
+
renderAutoDimensionControls() {
const { spec, selectedAutoDimension } = this.state;
if (!selectedAutoDimension) return;
@@ -2799,6 +2849,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
return (
<FormEditor
+ key={selectedDimensionSpec.index}
fields={DIMENSION_SPEC_FIELDS}
initValue={selectedDimensionSpec.value}
onClose={this.resetSelected}
@@ -2887,6 +2938,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
return (
<FormEditor
+ key={selectedMetricSpec.index}
fields={METRIC_SPEC_FIELDS}
initValue={selectedMetricSpec.value}
onClose={this.resetSelected}
diff --git
a/web-console/src/views/load-data-view/schema-table/schema-table.scss
b/web-console/src/views/load-data-view/schema-table/schema-table.scss
index 7906e58afa1..3eb6cbe0c94 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.scss
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.scss
@@ -24,14 +24,18 @@
&.dimension {
background: rgba($green3, 0.3);
- &.long {
+ &.long,
+ &.float,
+ &.double {
background: rgba($blue3, 0.3);
}
- &.float {
- background: rgba($blue3, 0.3);
+
+ &.array {
+ background: rgba($orange2, 0.3);
}
- &.double {
- background: rgba($blue3, 0.3);
+
+ &.mv-string {
+ background: rgba($orange4, 0.3);
}
}
@@ -44,14 +48,18 @@
&.dimension {
background: rgba($green3, 0.12);
- &.long {
+ &.long,
+ &.float,
+ &.double {
background: rgba($blue3, 0.1);
}
- &.float {
- background: rgba($blue3, 0.1);
+
+ &.array {
+ background: rgba($orange2, 0.1);
}
- &.double {
- background: rgba($blue3, 0.1);
+
+ &.mv-string {
+ background: rgba($orange4, 0.1);
}
}
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index b1245d6b39c..ff3a54785ba 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -24,8 +24,9 @@ import ReactTable from 'react-table';
import { TableCell } from '../../../components';
import type { DimensionSpec, MetricSpec } from '../../../druid-models';
import {
+ getDimensionSpecClassType,
getDimensionSpecName,
- getDimensionSpecType,
+ getDimensionSpecUserType,
getMetricSpecName,
inflateDimensionSpec,
TIME_COLUMN,
@@ -115,11 +116,16 @@ export const SchemaTable = React.memo(function
SchemaTable(props: SchemaTablePro
? dimensions.findIndex(d => getDimensionSpecName(d) === columnName)
: -1;
const dimensionSpec = dimensions ? dimensions[dimensionSpecIndex] :
undefined;
- const dimensionSpecType = dimensionSpec ?
getDimensionSpecType(dimensionSpec) : undefined;
+ const dimensionSpecUserType = dimensionSpec
+ ? getDimensionSpecUserType(dimensionSpec, definedDimensions)
+ : undefined;
+ const dimensionSpecClassType = dimensionSpec
+ ? getDimensionSpecClassType(dimensionSpec, definedDimensions)
+ : undefined;
const columnClassName = classNames(
isTimestamp ? 'timestamp' : 'dimension',
- dimensionSpecType || 'string',
+ dimensionSpecClassType || 'string',
{
selected:
(dimensionSpec && dimensionSpecIndex ===
selectedDimensionSpecIndex) ||
@@ -142,7 +148,7 @@ export const SchemaTable = React.memo(function
SchemaTable(props: SchemaTablePro
>
<div className="column-name">{columnName}</div>
<div className="column-detail">
- {isTimestamp ? 'long (time column)' : dimensionSpecType ||
'(auto)'}
+ {isTimestamp ? 'long (time column)' : dimensionSpecUserType
|| '(auto)'}
</div>
</div>
),
diff --git
a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
index fcae24eb797..a981ce4140b 100644
--- a/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
+++ b/web-console/src/views/sql-data-loader-view/column-editor/column-editor.tsx
@@ -16,77 +16,109 @@
* limitations under the License.
*/
-import { Button, FormGroup, InputGroup, Intent, Menu, MenuItem, Position }
from '@blueprintjs/core';
+import {
+ Button,
+ FormGroup,
+ InputGroup,
+ Intent,
+ Menu,
+ MenuDivider,
+ MenuItem,
+ Position,
+} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { Popover2 } from '@blueprintjs/popover2';
import type { QueryResult } from '@druid-toolkit/query';
-import { SqlExpression, SqlFunction } from '@druid-toolkit/query';
+import { F, SqlExpression, SqlFunction, SqlType } from '@druid-toolkit/query';
import type { JSX } from 'react';
import React, { useState } from 'react';
import { AppToaster } from '../../../singletons';
-import { tickIcon } from '../../../utils';
+import { deepDelete, deleteKeys, tickIcon } from '../../../utils';
import { FlexibleQueryInput } from
'../../workbench-view/flexible-query-input/flexible-query-input';
import './column-editor.scss';
-const NATIVE_TYPES: (undefined | string)[] = ['', 'string', 'long', 'double'];
+function getTargetTypes(isArray: boolean): SqlType[] {
+ return isArray
+ ? [SqlType.VARCHAR_ARRAY, SqlType.BIGINT_ARRAY, SqlType.DOUBLE_ARRAY]
+ : [SqlType.VARCHAR, SqlType.BIGINT, SqlType.DOUBLE];
+}
-interface Breakdown {
- expression: SqlExpression;
- nativeType?: string;
+interface CastBreakdown {
+ formula: string;
+ castType?: SqlType;
+ forceMultiValue: boolean;
outputName: string;
}
-function breakdownExpression(expression: SqlExpression): Breakdown {
+function expressionToCastBreakdown(expression: SqlExpression): CastBreakdown {
const outputName = expression.getOutputName() || '';
expression = expression.getUnderlyingExpression();
- let nativeType: string | undefined;
if (expression instanceof SqlFunction) {
const asType = expression.getCastType();
+ const formula = String(expression.getArg(0));
if (asType) {
- switch (asType.value.toUpperCase()) {
- case 'VARCHAR':
- nativeType = 'string';
- expression = expression.getArg(0)!;
- break;
-
- case 'BIGINT':
- nativeType = 'long';
- expression = expression.getArg(0)!;
- break;
-
- case 'DOUBLE':
- nativeType = 'double';
- expression = expression.getArg(0)!;
- break;
- }
+ return {
+ formula,
+ castType: asType,
+ forceMultiValue: false,
+ outputName,
+ };
+ } else if (expression.getEffectiveFunctionName() === 'ARRAY_TO_MV') {
+ return {
+ formula,
+ forceMultiValue: true,
+ outputName,
+ };
}
}
return {
- expression,
- nativeType,
+ formula: String(expression),
+ forceMultiValue: false,
outputName,
};
}
-function nativeTypeToSqlType(nativeType: string): string {
- switch (nativeType) {
- case 'string':
- return 'VARCHAR';
- case 'long':
- return 'BIGINT';
- case 'double':
- return 'DOUBLE';
- default:
- return 'VARCHAR';
+function castBreakdownToExpression({
+ formula,
+ castType,
+ forceMultiValue,
+ outputName,
+}: CastBreakdown): SqlExpression {
+ let newExpression = SqlExpression.parse(formula);
+ const defaultOutputName = newExpression.getOutputName();
+
+ if (castType) {
+ newExpression = newExpression.cast(castType);
+ } else if (forceMultiValue) {
+ newExpression = F('ARRAY_TO_MV', newExpression);
+ }
+
+ if (!defaultOutputName && !outputName) {
+ throw new Error('Must explicitly define an output name');
+ }
+
+ if (newExpression.getOutputName() !== outputName) {
+ newExpression = newExpression.as(outputName);
}
+
+ return newExpression;
+}
+
+function castBreakdownsEqual(a: CastBreakdown, b: CastBreakdown): boolean {
+ return (
+ a.formula === b.formula &&
+ String(a.castType) === String(b.castType) &&
+ a.forceMultiValue === b.forceMultiValue &&
+ a.outputName === b.outputName
+ );
}
interface ColumnEditorProps {
- expression?: SqlExpression;
+ initExpression?: SqlExpression;
onApply(expression: SqlExpression | undefined): void;
onCancel(): void;
dirty(): void;
@@ -96,52 +128,128 @@ interface ColumnEditorProps {
}
export const ColumnEditor = React.memo(function ColumnEditor(props:
ColumnEditorProps) {
- const { expression, onApply, onCancel, dirty, queryResult, headerIndex } =
props;
+ const { initExpression, onApply, onCancel, dirty, queryResult, headerIndex }
= props;
- const breakdown = expression ? breakdownExpression(expression) : undefined;
-
- const [outputName, setOutputName] = useState<string | undefined>();
- const [nativeType, setNativeType] = useState<string | undefined>();
- const [expressionString, setExpressionString] = useState<string |
undefined>();
-
- const effectiveOutputName = outputName ?? (breakdown?.outputName || '');
- const effectiveNativeType = nativeType ?? breakdown?.nativeType;
- const effectiveExpressionString = expressionString ??
(breakdown?.expression.toString() || '');
+ const [initBreakdown] = useState(
+ initExpression ? expressionToCastBreakdown(initExpression) : undefined,
+ );
+ const [currentBreakdown, setCurrentBreakdown] = useState(
+ initBreakdown || { formula: '', forceMultiValue: false, outputName: '' },
+ );
let typeButton: JSX.Element | undefined;
-
const sqlQuery = queryResult?.sqlQuery;
if (queryResult && sqlQuery && headerIndex !== -1) {
const column = queryResult.header[headerIndex];
+ const isArray =
String(column.nativeType).toUpperCase().startsWith('ARRAY');
const expression =
queryResult.sqlQuery?.getSelectExpressionForIndex(headerIndex);
- if (expression && column.sqlType !== 'TIMESTAMP') {
- const implicitText =
- 'implicit' + (breakdown?.nativeType ? '' : `
(${String(column.nativeType).toLowerCase()})`);
+ if (expression && column.sqlType !== 'TIMESTAMP' && column.sqlType !==
'OTHER') {
const selectExpression =
sqlQuery.getSelectExpressionForIndex(headerIndex);
+ const initCastType = initBreakdown?.castType;
+ const initForceMultiValue = initBreakdown?.forceMultiValue;
if (selectExpression) {
+ const castToItems = (
+ <>
+ <MenuDivider title="Cast to..." />
+ {getTargetTypes(isArray).map((targetType, i) => (
+ <MenuItem
+ key={i}
+ icon={tickIcon(
+ targetType.equals(currentBreakdown.castType) &&
!currentBreakdown.forceMultiValue,
+ )}
+ text={targetType.value}
+ label={targetType.getNativeType()}
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown({ ...currentBreakdown, castType:
targetType });
+ }}
+ />
+ ))}
+ {isArray && (
+ <>
+ <MenuDivider />
+ <MenuItem
+ icon={tickIcon(currentBreakdown.forceMultiValue)}
+ text="Make multi-value"
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown({
+ ...deepDelete(currentBreakdown, 'castType'),
+ forceMultiValue: true,
+ });
+ }}
+ />
+ </>
+ )}
+ </>
+ );
+
typeButton = (
<Popover2
position={Position.BOTTOM_LEFT}
minimal
content={
- <Menu>
- {NATIVE_TYPES.map((type, i) => {
- return (
- <MenuItem
- key={i}
- icon={tickIcon(type === (effectiveNativeType || ''))}
- text={type || implicitText}
- onClick={() => setNativeType(type)}
- />
- );
- })}
- </Menu>
+ initCastType ? (
+ <Menu>
+ <MenuItem
+ icon={tickIcon(!currentBreakdown.castType &&
!currentBreakdown.forceMultiValue)}
+ text="Remove explicit cast"
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown(
+ deleteKeys(currentBreakdown, ['castType',
'forceMultiValue']),
+ );
+ }}
+ />
+ {castToItems}
+ </Menu>
+ ) : initForceMultiValue ? (
+ <Menu>
+ <MenuItem
+ icon={tickIcon(!currentBreakdown.castType &&
currentBreakdown.forceMultiValue)}
+ text="VARCHAR (multi-value)"
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown({ ...currentBreakdown,
forceMultiValue: true });
+ }}
+ />
+ <MenuItem
+ icon={tickIcon(!currentBreakdown.castType &&
!currentBreakdown.forceMultiValue)}
+ text="Remove multi-value coercion"
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown(
+ deleteKeys(currentBreakdown, ['castType',
'forceMultiValue']),
+ );
+ }}
+ />
+ </Menu>
+ ) : (
+ <Menu>
+ <MenuItem
+ icon={tickIcon(!currentBreakdown.castType &&
!currentBreakdown.forceMultiValue)}
+ text={`implicit (${column.sqlType})`}
+ onClick={() => {
+ dirty();
+ setCurrentBreakdown(
+ deleteKeys(currentBreakdown, ['castType',
'forceMultiValue']),
+ );
+ }}
+ />
+ {castToItems}
+ </Menu>
+ )
}
>
<Button
- text={`Type: ${effectiveNativeType || implicitText}`}
+ text={`Type: ${
+ currentBreakdown.castType ||
+ (currentBreakdown.forceMultiValue
+ ? 'VARCHAR (multi-value)'
+ : `implicit (${column.sqlType})`)
+ }`}
rightIcon={IconNames.CARET_DOWN}
/>
</Popover2>
@@ -152,13 +260,13 @@ export const ColumnEditor = React.memo(function
ColumnEditor(props: ColumnEditor
return (
<div className="column-editor">
- <div className="title">{expression ? 'Edit column' : 'Add column'}</div>
+ <div className="title">{initExpression ? 'Edit column' : 'Add
column'}</div>
<FormGroup label="Name">
<InputGroup
- value={effectiveOutputName}
+ value={currentBreakdown.outputName}
onChange={e => {
- if (!outputName) dirty();
- setOutputName(e.target.value);
+ dirty();
+ setCurrentBreakdown({ ...currentBreakdown, outputName:
e.target.value });
}}
/>
</FormGroup>
@@ -166,17 +274,17 @@ export const ColumnEditor = React.memo(function
ColumnEditor(props: ColumnEditor
<FlexibleQueryInput
showGutter={false}
placeholder="expression"
- queryString={effectiveExpressionString}
- onQueryStringChange={f => {
- if (!expressionString) dirty();
- setExpressionString(f);
+ queryString={currentBreakdown.formula}
+ onQueryStringChange={formula => {
+ dirty();
+ setCurrentBreakdown({ ...currentBreakdown, formula });
}}
columnMetadata={undefined}
/>
</FormGroup>
{typeButton && <FormGroup>{typeButton}</FormGroup>}
<div className="apply-cancel-buttons">
- {expression && (
+ {initExpression && (
<Button
className="delete"
icon={IconNames.TRASH}
@@ -191,27 +299,19 @@ export const ColumnEditor = React.memo(function
ColumnEditor(props: ColumnEditor
<Button
text="Apply"
intent={Intent.PRIMARY}
- disabled={!outputName && !expressionString && typeof nativeType ===
'undefined'}
+ disabled={Boolean(initBreakdown &&
castBreakdownsEqual(initBreakdown, currentBreakdown))}
onClick={() => {
let newExpression: SqlExpression;
try {
- newExpression = SqlExpression.parse(effectiveExpressionString);
+ newExpression = castBreakdownToExpression(currentBreakdown);
} catch (e) {
AppToaster.show({
- message: `Could not parse SQL expression: ${e.message}`,
+ message: e.message,
intent: Intent.DANGER,
});
return;
}
- if (nativeType) {
- newExpression =
newExpression.cast(nativeTypeToSqlType(nativeType));
- }
-
- if (newExpression.getOutputName() !== effectiveOutputName) {
- newExpression = newExpression.as(effectiveOutputName);
- }
-
onApply(newExpression);
onCancel();
}}
diff --git
a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
index 2a185067427..462a852e8df 100644
---
a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
+++
b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.scss
@@ -58,6 +58,14 @@
&.double {
background: rgba($blue3, 0.3);
}
+
+ &.array {
+ background: rgba($orange2, 0.3);
+ }
+
+ &.multi-value {
+ background: rgba($orange4, 0.3);
+ }
}
&.metric {
@@ -119,6 +127,14 @@
&.double {
background: rgba($blue3, 0.1);
}
+
+ &.array {
+ background: rgba($orange2, 0.1);
+ }
+
+ &.multi-value {
+ background: rgba($orange4, 0.1);
+ }
}
&.metric {
diff --git
a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
index 5feb7009d12..a674a3e6028 100644
---
a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
+++
b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
@@ -19,8 +19,8 @@
import { Icon } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { Popover2 } from '@blueprintjs/popover2';
-import type { Column, QueryResult, SqlQuery } from '@druid-toolkit/query';
-import { SqlAlias, SqlStar } from '@druid-toolkit/query';
+import type { Column, QueryResult, SqlExpression, SqlQuery } from
'@druid-toolkit/query';
+import { SqlAlias, SqlFunction, SqlStar } from '@druid-toolkit/query';
import classNames from 'classnames';
import React, { useState } from 'react';
import type { RowRenderProps } from 'react-table';
@@ -44,7 +44,13 @@ function isDate(v: any): v is Date {
return Boolean(v && typeof v.toISOString === 'function');
}
-function getExpressionIfAlias(query: SqlQuery, selectIndex: number): string {
+function isWrappedInArrayToMv(ex: SqlExpression | undefined) {
+ if (!ex) return false;
+ ex = ex.getUnderlyingExpression();
+ return ex instanceof SqlFunction && ex.getEffectiveFunctionName() ===
'ARRAY_TO_MV';
+}
+
+function formatFormulaAtIndex(query: SqlQuery, selectIndex: number): string {
const ex = query.getSelectExpressionForIndex(selectIndex);
if (query.isRealOutputColumnAtSelectIndex(selectIndex)) {
@@ -123,7 +129,10 @@ export const PreviewTable = React.memo(function
PreviewTable(props: PreviewTable
column.isTimeColumn() ? 'timestamp' : 'dimension',
`column${i}`,
column.sqlType?.toLowerCase(),
- { selected },
+ {
+ selected,
+ 'multi-value':
isWrappedInArrayToMv(parsedQuery.getSelectExpressionForIndex(i)),
+ },
);
return {
@@ -137,7 +146,7 @@ export const PreviewTable = React.memo(function
PreviewTable(props: PreviewTable
<Icon className="filter-icon" icon={IconNames.FILTER}
size={14} />
)}
</div>
- <div className="formula">{getExpressionIfAlias(parsedQuery,
i)}</div>
+ <div className="formula">{formatFormulaAtIndex(parsedQuery,
i)}</div>
</div>
);
},
diff --git
a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
index 8b5ee9f82f8..c52ee91abff 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
@@ -430,11 +430,19 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
dimensions: filterMap(sampleExternalConfig.signature, s => {
const columnName = s.getColumnName();
if (columnName === TIME_COLUMN) return;
- const t = s.columnType.getNativeType();
- return {
- name: columnName,
- type: t === 'COMPLEX<json>' ? 'json' : t,
- };
+ if (s.columnType.isArray()) {
+ return {
+ type: 'auto',
+ castToType: s.columnType.getNativeType().toUpperCase(),
+ name: columnName,
+ };
+ } else {
+ const t = s.columnType.getNativeType();
+ return {
+ name: columnName,
+ type: t === 'COMPLEX<json>' ? 'json' : t,
+ };
+ }
}),
},
granularitySpec: {
@@ -899,7 +907,8 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
{editorColumn && ingestQueryPattern && (
<>
<ColumnEditor
- expression={editorColumn.expression}
+ key={editorColumn.index}
+ initExpression={editorColumn.expression}
onApply={newColumn => {
if (!editorColumn) return;
updatePattern(
@@ -912,7 +921,10 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
);
}}
onCancel={() => setEditorColumn(undefined)}
- dirty={() => setEditorColumn({ ...editorColumn, dirty: true
})}
+ dirty={() => {
+ if (!editorColumn.dirty) return;
+ setEditorColumn({ ...editorColumn, dirty: true });
+ }}
queryResult={previewResultState.data}
headerIndex={editorColumn.index}
/>
diff --git
a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 8e967487a32..4b99da236b3 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -45,6 +45,12 @@ import { TitleFrame } from './title-frame/title-frame';
import './sql-data-loader-view.scss';
+const INITIAL_QUERY_CONTEXT: QueryContext = {
+ finalizeAggregations: false,
+ groupByEnableMultiValueUnnesting: false,
+ arrayIngestMode: 'array',
+};
+
interface LoaderContent extends QueryWithContext {
id?: string;
}
@@ -183,31 +189,28 @@ export const SqlDataLoaderView = React.memo(function
SqlDataLoaderView(
inputSource={inputSource}
initInputFormat={inputFormat}
doneButton={false}
- onSet={({ inputFormat, signature, isArrays, timeExpression }) => {
+ onSet={({ inputFormat, signature, timeExpression, arrayMode }) => {
setContent({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
{ inputSource, inputFormat, signature },
- isArrays,
timeExpression,
undefined,
+ arrayMode,
),
).toString(),
- queryContext: {
- finalizeAggregations: false,
- groupByEnableMultiValueUnnesting: false,
- },
+ queryContext: INITIAL_QUERY_CONTEXT,
});
}}
altText="Skip the wizard and continue with custom SQL"
- onAltSet={({ inputFormat, signature, isArrays, timeExpression })
=> {
+ onAltSet={({ inputFormat, signature, timeExpression, arrayMode })
=> {
goToQuery({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
{ inputSource, inputFormat, signature },
- isArrays,
timeExpression,
undefined,
+ arrayMode,
),
).toString(),
});
diff --git
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
index bd8fbff9fd0..9d4974332ef 100644
---
a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
+++
b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
@@ -20,7 +20,7 @@ import { Classes, Dialog } from '@blueprintjs/core';
import type { SqlExpression } from '@druid-toolkit/query';
import React, { useState } from 'react';
-import type { ExternalConfig, InputFormat, InputSource } from
'../../../druid-models';
+import type { ArrayMode, ExternalConfig, InputFormat, InputSource } from
'../../../druid-models';
import { InputFormatStep } from '../input-format-step/input-format-step';
import { InputSourceStep } from '../input-source-step/input-source-step';
@@ -30,9 +30,9 @@ export interface ConnectExternalDataDialogProps {
initExternalConfig?: Partial<ExternalConfig>;
onSetExternalConfig(
config: ExternalConfig,
- isArrays: boolean[],
timeExpression: SqlExpression | undefined,
partitionedByHint: string | undefined,
+ arrayMode: ArrayMode,
): void;
onClose(): void;
}
@@ -67,12 +67,12 @@ export const ConnectExternalDataDialog =
React.memo(function ConnectExternalData
inputSource={inputSource}
initInputFormat={inputFormat}
doneButton
- onSet={({ inputFormat, signature, isArrays, timeExpression }) => {
+ onSet={({ inputFormat, signature, timeExpression, arrayMode }) => {
onSetExternalConfig(
{ inputSource, inputFormat, signature },
- isArrays,
timeExpression,
partitionedByHint,
+ arrayMode,
);
onClose();
}}
diff --git
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
index 7e5064867a3..e6ea2d9081b 100644
---
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
+++
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
@@ -22,14 +22,13 @@ import type { SqlExpression } from '@druid-toolkit/query';
import { C, SqlColumnDeclaration, SqlType } from '@druid-toolkit/query';
import React, { useState } from 'react';
-import { AutoForm, CenterMessage, LearnMore, Loader } from
'../../../components';
-import type { InputFormat, InputSource } from '../../../druid-models';
+import { ArrayModeSwitch, AutoForm, CenterMessage, LearnMore, Loader } from
'../../../components';
+import type { ArrayMode, InputFormat, InputSource } from
'../../../druid-models';
import {
BATCH_INPUT_FORMAT_FIELDS,
chooseByBestTimestamp,
DETECTION_TIMESTAMP_SPEC,
guessColumnTypeFromSampleResponse,
- guessIsArrayFromSampleResponse,
inputFormatOutputsNumericStrings,
possibleDruidFormatForValues,
TIME_COLUMN,
@@ -52,8 +51,8 @@ import './input-format-step.scss';
export interface InputFormatAndMore {
inputFormat: InputFormat;
signature: SqlColumnDeclaration[];
- isArrays: boolean[];
timeExpression: SqlExpression | undefined;
+ arrayMode: ArrayMode;
}
interface PossibleTimeExpression {
@@ -80,6 +79,7 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
AutoForm.isValidModel(initInputFormat, BATCH_INPUT_FORMAT_FIELDS) ?
initInputFormat : undefined,
);
const [selectTimestamp, setSelectTimestamp] = useState(true);
+ const [arrayMode, setArrayMode] = useState<ArrayMode>('multi-values');
const [previewState] = useQueryManager<InputFormat, SampleResponse>({
query: inputFormatToSample,
@@ -166,13 +166,13 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
),
),
),
- isArrays: headerNames.map(name =>
- guessIsArrayFromSampleResponse(previewSampleResponse, name),
- ),
timeExpression: selectTimestamp ?
possibleTimeExpression?.timeExpression : undefined,
+ arrayMode,
}
: undefined;
+ const hasArrays = inputFormatAndMore?.signature.some(d =>
d.columnType.isArray());
+
return (
<div className="input-format-step">
<div className="preview">
@@ -224,6 +224,7 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
)}
</div>
<div className="bottom-controls">
+ {hasArrays && <ArrayModeSwitch arrayMode={arrayMode}
changeArrayMode={setArrayMode} />}
{possibleTimeExpression && (
<FormGroup>
<Callout>
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx
b/web-console/src/views/workbench-view/workbench-view.tsx
index 9c859f9b462..7d721da413f 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -320,13 +320,18 @@ export class WorkbenchView extends
React.PureComponent<WorkbenchViewProps, Workb
return (
<ConnectExternalDataDialog
- onSetExternalConfig={(externalConfig, isArrays, timeExpression,
partitionedByHint) => {
+ onSetExternalConfig={(
+ externalConfig,
+ timeExpression,
+ partitionedByHint,
+ forceMultiValue,
+ ) => {
this.handleNewTab(
WorkbenchQuery.fromInitExternalConfig(
externalConfig,
- isArrays,
timeExpression,
partitionedByHint,
+ forceMultiValue,
),
'Ext ' +
guessDataSourceNameFromInputSource(externalConfig.inputSource),
);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
