This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch 26.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/26.0.0 by this push:
new 6d2208803f Web console: allow __time in MSQ (#14165) (#14182)
6d2208803f is described below
commit 6d2208803fce69b5bb8c732f64a7b2ea8d902e58
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Thu Apr 27 21:43:23 2023 -0700
Web console: allow __time in MSQ (#14165) (#14182)
* works in MSQ
* fix spec conversion
---
.../druid-models/dimension-spec/dimension-spec.ts | 2 +-
.../druid-models/timestamp-spec/timestamp-spec.tsx | 6 +
.../__snapshots__/spec-conversion.spec.ts.snap | 29 +-
web-console/src/helpers/spec-conversion.spec.ts | 46 ++
web-console/src/helpers/spec-conversion.ts | 9 +-
web-console/src/utils/sampler.ts | 28 +-
.../__snapshots__/parse-data-table.spec.tsx.snap | 489 ++++++++++++++++++++-
.../parse-data-table/parse-data-table.tsx | 11 +-
.../schema-step/schema-step.tsx | 17 +-
.../input-format-step/input-format-step.tsx | 29 +-
10 files changed, 625 insertions(+), 41 deletions(-)
diff --git a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
index 2b8e7c0202..306111e068 100644
--- a/web-console/src/druid-models/dimension-spec/dimension-spec.ts
+++ b/web-console/src/druid-models/dimension-spec/dimension-spec.ts
@@ -85,7 +85,7 @@ export function getDimensionSpecs(
guessNumericStringsAsNumbers: boolean,
hasRollup: boolean,
): (string | DimensionSpec)[] {
- return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, true), h
=> {
+ return filterMap(getHeaderNamesFromSampleResponse(sampleResponse, 'ignore'),
h => {
const dimensionType =
typeHints[h] ||
guessColumnTypeFromSampleResponse(sampleResponse, h,
guessNumericStringsAsNumbers);
diff --git a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
index fb133e58b4..0107fa3be6 100644
--- a/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
+++ b/web-console/src/druid-models/timestamp-spec/timestamp-spec.tsx
@@ -39,6 +39,12 @@ export const PLACEHOLDER_TIMESTAMP_SPEC: TimestampSpec = {
missingValue: '1970-01-01T00:00:00Z',
};
+export const DETECTION_TIMESTAMP_SPEC: TimestampSpec = {
+ column: TIME_COLUMN,
+ format: 'millis',
+ missingValue: '1970-01-01T00:00:00Z',
+};
+
export const REINDEX_TIMESTAMP_SPEC: TimestampSpec = {
column: TIME_COLUMN,
format: 'millis',
diff --git a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
index f0b47b848c..ac4dd8655e 100644
--- a/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
+++ b/web-console/src/helpers/__snapshots__/spec-conversion.spec.ts.snap
@@ -10,7 +10,7 @@ WITH "source" AS (SELECT * FROM TABLE(
)
) EXTEND ("event_ts" VARCHAR, "col1" VARCHAR, "col2" VARCHAR, "col3" VARCHAR,
"col4" VARCHAR, "field1" DOUBLE, "field2" DOUBLE, "field3" DOUBLE, "field4"
VARCHAR, "field5" VARCHAR, "field6" BIGINT, "field7" DOUBLE))
SELECT
- TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts")
END, 'PT1H') AS __time,
+ TIME_FLOOR(CASE WHEN CAST("event_ts" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("event_ts" AS BIGINT)) ELSE TIME_PARSE("event_ts")
END, 'PT1H') AS "__time",
"col1",
"col2",
"col3",
@@ -38,7 +38,7 @@ WITH "source" AS (SELECT * FROM TABLE(
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags"
VARCHAR, "isUnpatrolled" VARCHAR, "comment" VARCHAR, "isNew" VARCHAR, "isMinor"
VARCHAR, "isAnonymous" VARCHAR, "user" VARCHAR, "namespace" VARCHAR, "cityName"
VARCHAR, "countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR,
"countryIsoCode" VARCHAR, "regionName" VARCHAR, "added" BIGINT, "commentLength"
BIGINT, "delta" BIGINT, "deltaBucket" BIGINT, "deleted" BIGINT, "page" VARCHAR))
SELECT
- TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS __time,
+ TIME_FLOOR(TIME_PARSE("timestamp"), 'PT1H') AS "__time",
"isRobot",
"channel",
"flags",
@@ -77,7 +77,7 @@ WITH "source" AS (SELECT * FROM TABLE(
)
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags"
VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added"
BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor"
VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket"
BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR,
"countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR,
"countryIsoCode" VARCHAR, "regionName" VARCHAR [...]
SELECT
- CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp")
END AS __time,
+ CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp")
END AS "__time",
"isRobot",
"channel",
"flags",
@@ -119,7 +119,7 @@ WITH "source" AS (SELECT * FROM TABLE(
) EXTEND ("isRobot" VARCHAR, "channel" VARCHAR, "flags" VARCHAR,
"isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added" BIGINT,
"comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor" VARCHAR,
"delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket" BIGINT,
"deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR, "countryName"
VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR, "countryIsoCode"
VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically
converted.
- REWRITE_[_some_time_parse_expression_]_TO_SQL AS __time, --:ISSUE: Transform
for __time could not be converted
+ REWRITE_[_some_time_parse_expression_]_TO_SQL AS "__time", --:ISSUE:
Transform for __time could not be converted
"isRobot",
"channel",
"flags",
@@ -159,7 +159,7 @@ WITH "source" AS (SELECT * FROM TABLE(
) EXTEND ("timestamp" VARCHAR, "isRobot" VARCHAR, "channel" VARCHAR, "flags"
VARCHAR, "isUnpatrolled" VARCHAR, "page" VARCHAR, "diffUrl" VARCHAR, "added"
BIGINT, "comment" VARCHAR, "commentLength" BIGINT, "isNew" VARCHAR, "isMinor"
VARCHAR, "delta" BIGINT, "isAnonymous" VARCHAR, "user" VARCHAR, "deltaBucket"
BIGINT, "deleted" BIGINT, "namespace" VARCHAR, "cityName" VARCHAR,
"countryName" VARCHAR, "regionIsoCode" VARCHAR, "metroCode" VARCHAR,
"countryIsoCode" VARCHAR, "regionName" VARCHAR))
SELECT
--:ISSUE: The spec contained transforms that could not be automatically
converted.
- CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp")
END AS __time,
+ CASE WHEN CAST("timestamp" AS BIGINT) > 0 THEN
MILLIS_TO_TIMESTAMP(CAST("timestamp" AS BIGINT)) ELSE TIME_PARSE("timestamp")
END AS "__time",
"isRobot",
"channel",
"flags",
@@ -188,3 +188,22 @@ WHERE REWRITE_[{"type":"strange"}]_TO_SQL --:ISSUE: The
spec contained a filter
PARTITIONED BY HOUR
CLUSTERED BY "isRobot"
`;
+
+exports[`spec conversion converts with when the __time column is used as the
__time column 1`] = `
+-- This SQL query was auto generated from an ingestion spec
+REPLACE INTO "wikipedia" OVERWRITE ALL
+WITH "source" AS (SELECT * FROM TABLE(
+ EXTERN(
+
'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}',
+ '{"type":"json"}'
+ )
+) EXTEND ("__time" BIGINT, "isRobot" VARCHAR, "channel" VARCHAR, "flags"
VARCHAR))
+SELECT
+ "__time" AS "__time",
+ "isRobot",
+ "channel",
+ "flags"
+FROM "source"
+PARTITIONED BY HOUR
+CLUSTERED BY "isRobot"
+`;
diff --git a/web-console/src/helpers/spec-conversion.spec.ts
b/web-console/src/helpers/spec-conversion.spec.ts
index d1fdf5b606..fa72378b15 100644
--- a/web-console/src/helpers/spec-conversion.spec.ts
+++ b/web-console/src/helpers/spec-conversion.spec.ts
@@ -444,6 +444,52 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
});
+ it('converts with when the __time column is used as the __time column', ()
=> {
+ const converted = convertSpecToSql({
+ type: 'index_parallel',
+ spec: {
+ ioConfig: {
+ type: 'index_parallel',
+ inputSource: {
+ type: 'http',
+ uris: ['https://druid.apache.org/data/wikipedia.json.gz'],
+ },
+ inputFormat: {
+ type: 'json',
+ },
+ },
+ dataSchema: {
+ granularitySpec: {
+ segmentGranularity: 'hour',
+ queryGranularity: 'none',
+ rollup: false,
+ },
+ dataSource: 'wikipedia',
+ timestampSpec: {
+ column: '__time',
+ format: 'millis',
+ },
+ dimensionsSpec: {
+ dimensions: ['isRobot', 'channel', 'flags'],
+ },
+ },
+ tuningConfig: {
+ type: 'index_parallel',
+ partitionsSpec: {
+ type: 'single_dim',
+ partitionDimension: 'isRobot',
+ targetRowsPerSegment: 150000,
+ },
+ forceGuaranteedRollup: true,
+ maxNumConcurrentSubTasks: 4,
+ maxParseExceptions: 3,
+ },
+ },
+ });
+
+ expect(converted.queryString).toMatchSnapshot();
+ });
+
it('converts with issue when there is a dimension transform and strange
filter', () => {
const converted = convertSpecToSql({
type: 'index_parallel',
diff --git a/web-console/src/helpers/spec-conversion.ts
b/web-console/src/helpers/spec-conversion.ts
index 990147b57a..62412e7eca 100644
--- a/web-console/src/helpers/spec-conversion.ts
+++ b/web-console/src/helpers/spec-conversion.ts
@@ -35,7 +35,7 @@ import type {
TimestampSpec,
Transform,
} from '../druid-models';
-import { inflateDimensionSpec, upgradeSpec } from '../druid-models';
+import { inflateDimensionSpec, TIME_COLUMN, upgradeSpec } from
'../druid-models';
import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils';
export function getSpecDatasourceName(spec: IngestionSpec): string {
@@ -129,9 +129,12 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
const timestampColumnName = timestampSpec.column || 'timestamp';
const timestampColumn = C(timestampColumnName);
const format = timestampSpec.format || 'auto';
- const timeTransform = transforms.find(t => t.name === '__time');
+ const timeTransform = transforms.find(t => t.name === TIME_COLUMN);
if (timeTransform) {
timeExpression = `REWRITE_[${timeTransform.expression}]_TO_SQL`;
+ } else if (timestampColumnName === TIME_COLUMN) {
+ timeExpression = String(timestampColumn);
+
columnDeclarations.unshift(SqlColumnDeclaration.create(timestampColumnName,
SqlType.BIGINT));
} else {
let timestampColumnType: SqlType;
switch (format) {
@@ -271,7 +274,7 @@ export function convertSpecToSql(spec: any):
QueryWithContext {
}
const dimensionExpressions = [
- ` ${timeExpression} AS __time,${
+ ` ${timeExpression} AS "__time",${
timeTransform ? ` --:ISSUE: Transform for __time could not be converted`
: ''
}`,
].concat(
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index ed5c5e2cc0..67f883a580 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -32,6 +32,7 @@ import type {
TransformSpec,
} from '../druid-models';
import {
+ DETECTION_TIMESTAMP_SPEC,
getDimensionNamesFromTransforms,
getDimensionSpecName,
getSpecType,
@@ -72,10 +73,25 @@ export interface SampleResponse {
export function getHeaderNamesFromSampleResponse(
sampleResponse: SampleResponse,
- ignoreTimeColumn = false,
-) {
- return filterMap(sampleResponse.logicalSegmentSchema, s =>
- ignoreTimeColumn && s.name === '__time' ? undefined : s.name,
+ timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
+): string[] {
+ return getHeaderFromSampleResponse(sampleResponse, timeColumnAction).map(s
=> s.name);
+}
+
+export function getHeaderFromSampleResponse(
+ sampleResponse: SampleResponse,
+ timeColumnAction: 'preserve' | 'ignore' | 'ignoreIfZero' = 'preserve',
+): { name: string; type: string }[] {
+ const ignoreTimeColumn =
+ timeColumnAction === 'ignore' ||
+ (timeColumnAction === 'ignoreIfZero' &&
+ !sampleResponse.data.some(d => {
+ const t = d.parsed?.[TIME_COLUMN];
+ return typeof t === 'number' && t > 0;
+ }));
+
+ return sampleResponse.logicalSegmentSchema.filter(
+ s => !ignoreTimeColumn || s.name !== TIME_COLUMN,
);
}
@@ -328,7 +344,7 @@ export async function sampleForParser(
ioConfig,
dataSchema: {
dataSource: 'sample',
- timestampSpec: reingestMode ? REINDEX_TIMESTAMP_SPEC :
PLACEHOLDER_TIMESTAMP_SPEC,
+ timestampSpec: reingestMode ? REINDEX_TIMESTAMP_SPEC :
DETECTION_TIMESTAMP_SPEC,
dimensionsSpec: {
useSchemaDiscovery: true,
},
@@ -532,7 +548,7 @@ export async function sampleForFilter(
specialDimensionSpec,
'dimensions',
dedupe(
- getHeaderNamesFromSampleResponse(sampleResponseHack, true).concat(
+ getHeaderNamesFromSampleResponse(sampleResponseHack, 'ignore').concat(
getDimensionNamesFromTransforms(transforms),
),
),
diff --git
a/web-console/src/views/load-data-view/parse-data-table/__snapshots__/parse-data-table.spec.tsx.snap
b/web-console/src/views/load-data-view/parse-data-table/__snapshots__/parse-data-table.spec.tsx.snap
index ccc726a856..04b0d15135 100644
---
a/web-console/src/views/load-data-view/parse-data-table/__snapshots__/parse-data-table.spec.tsx.snap
+++
b/web-console/src/views/load-data-view/parse-data-table/__snapshots__/parse-data-table.spec.tsx.snap
@@ -10,7 +10,7 @@ exports[`ParseDataTable matches snapshot 1`] = `
>
<div
class="rt-thead -header"
- style="min-width: 875px;"
+ style="min-width: 1015px;"
>
<div
class="rt-tr"
@@ -26,6 +26,35 @@ exports[`ParseDataTable matches snapshot 1`] = `
class=""
/>
</div>
+ <div
+ class="rt-th rt-resizable-header"
+ role="columnheader"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ tabindex="-1"
+ >
+ <div
+ class="rt-resizable-header-content"
+ >
+ <div
+ class=""
+ >
+ <div
+ class="column-name"
+ >
+ __time
+ </div>
+ <div
+ class="column-detail"
+ >
+
+
+ </div>
+ </div>
+ </div>
+ <div
+ class="rt-resizer"
+ />
+ </div>
<div
class="rt-th rt-resizable-header"
role="columnheader"
@@ -204,7 +233,7 @@ exports[`ParseDataTable matches snapshot 1`] = `
</div>
<div
class="rt-tbody"
- style="min-width: 875px;"
+ style="min-width: 1015px;"
>
<div
class="rt-tr-group"
@@ -225,6 +254,17 @@ exports[`ParseDataTable matches snapshot 1`] = `
•
</div>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <div
+ class="table-cell plain"
+ >
+ 1460366400000
+ </div>
+ </div>
<div
class="rt-td"
role="gridcell"
@@ -312,6 +352,17 @@ exports[`ParseDataTable matches snapshot 1`] = `
•
</div>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <div
+ class="table-cell plain"
+ >
+ 1460366460000
+ </div>
+ </div>
<div
class="rt-td"
role="gridcell"
@@ -399,6 +450,17 @@ exports[`ParseDataTable matches snapshot 1`] = `
•
</div>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <div
+ class="table-cell plain"
+ >
+ 1460366520000
+ </div>
+ </div>
<div
class="rt-td"
role="gridcell"
@@ -538,6 +600,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -611,6 +682,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -684,6 +764,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -757,6 +846,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -830,6 +928,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -903,6 +1010,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -976,6 +1092,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1049,6 +1174,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1122,6 +1256,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1195,6 +1338,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1268,6 +1420,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1341,6 +1502,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1414,6 +1584,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1487,6 +1666,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1560,6 +1748,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1633,6 +1830,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1706,6 +1912,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1779,6 +1994,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1852,6 +2076,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1925,6 +2158,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -1998,6 +2240,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2071,6 +2322,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2144,6 +2404,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2217,6 +2486,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2290,6 +2568,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2363,6 +2650,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2436,6 +2732,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2509,6 +2814,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2582,6 +2896,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2655,6 +2978,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2728,6 +3060,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2801,6 +3142,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2874,6 +3224,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -2947,6 +3306,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3020,6 +3388,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3093,6 +3470,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3166,6 +3552,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3239,6 +3634,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3312,6 +3716,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3385,6 +3798,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3458,6 +3880,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3531,6 +3962,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3604,6 +4044,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3677,6 +4126,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3750,6 +4208,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3823,6 +4290,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
<div
@@ -3896,6 +4372,15 @@ exports[`ParseDataTable matches snapshot 1`] = `
</span>
</div>
+ <div
+ class="rt-td"
+ role="gridcell"
+ style="flex: 140 0 auto; width: 140px; max-width: 140px;"
+ >
+ <span>
+
+ </span>
+ </div>
</div>
</div>
</div>
diff --git
a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
index 3206851dad..eb0faa9d8c 100644
--- a/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
+++ b/web-console/src/views/load-data-view/parse-data-table/parse-data-table.tsx
@@ -41,8 +41,7 @@ export interface ParseDataTableProps {
canFlatten: boolean;
flattenedColumnsOnly: boolean;
flattenFields: FlattenField[];
- onFlattenFieldSelect: (field: FlattenField, index: number) => void;
- useInput?: boolean;
+ onFlattenFieldSelect?: (field: FlattenField, index: number) => void;
}
export const ParseDataTable = React.memo(function ParseDataTable(props:
ParseDataTableProps) {
@@ -53,10 +52,8 @@ export const ParseDataTable = React.memo(function
ParseDataTable(props: ParseDat
flattenedColumnsOnly,
flattenFields,
onFlattenFieldSelect,
- useInput,
} = props;
- const key = useInput ? 'input' : 'parsed';
return (
<ReactTable
className={classNames('parse-data-table', DEFAULT_TABLE_CLASS_NAME)}
@@ -66,7 +63,7 @@ export const ParseDataTable = React.memo(function
ParseDataTable(props: ParseDat
pageSizeOptions={STANDARD_TABLE_PAGE_SIZE_OPTIONS}
showPagination={sampleResponse.data.length > STANDARD_TABLE_PAGE_SIZE}
columns={filterMap(
- getHeaderNamesFromSampleResponse(sampleResponse, true),
+ getHeaderNamesFromSampleResponse(sampleResponse, 'ignoreIfZero'),
(columnName, i) => {
if (!caseInsensitiveContains(columnName, columnFilter)) return;
const flattenFieldIndex = flattenFields.findIndex(f => f.name ===
columnName);
@@ -78,7 +75,7 @@ export const ParseDataTable = React.memo(function
ParseDataTable(props: ParseDat
className={classNames({ clickable: flattenField })}
onClick={() => {
if (!flattenField) return;
- onFlattenFieldSelect(flattenField, flattenFieldIndex);
+ onFlattenFieldSelect?.(flattenField, flattenFieldIndex);
}}
>
<div className="column-name">{columnName}</div>
@@ -88,7 +85,7 @@ export const ParseDataTable = React.memo(function
ParseDataTable(props: ParseDat
</div>
),
id: String(i),
- accessor: (row: SampleEntry) => (row[key] ? row[key]![columnName]
: null),
+ accessor: (row: SampleEntry) => row.parsed?.[columnName] ?? null,
width: 140,
Cell: function ParseDataTableCell(row: RowRenderProps) {
if (row.original.unparseable) {
diff --git
a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
index 76c536e3a4..7446b60550 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
@@ -49,12 +49,12 @@ import { AsyncActionDialog } from '../../../dialogs';
import type { Execution, ExternalConfig, IngestQueryPattern } from
'../../../druid-models';
import {
changeQueryPatternExpression,
+ DETECTION_TIMESTAMP_SPEC,
fitIngestQueryPattern,
getDestinationMode,
getQueryPatternExpression,
getQueryPatternExpressionType,
ingestQueryPatternToQuery,
- PLACEHOLDER_TIMESTAMP_SPEC,
possibleDruidFormatForValues,
TIME_COLUMN,
WorkbenchQueryPart,
@@ -84,7 +84,7 @@ import {
wait,
without,
} from '../../../utils';
-import { postToSampler } from '../../../utils/sampler';
+import { getHeaderFromSampleResponse, postToSampler } from
'../../../utils/sampler';
import { FlexibleQueryInput } from
'../../workbench-view/flexible-query-input/flexible-query-input';
import { ColumnActions } from '../column-actions/column-actions';
import { ColumnEditor } from '../column-editor/column-editor';
@@ -406,12 +406,14 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
},
dataSchema: {
dataSource: 'sample',
- timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC,
+ timestampSpec: DETECTION_TIMESTAMP_SPEC,
dimensionsSpec: {
- dimensions: sampleExternalConfig.signature.map(s => {
+ dimensions: filterMap(sampleExternalConfig.signature, s => {
+ const columnName = s.getColumnName();
+ if (columnName === TIME_COLUMN) return;
const t = s.columnType.getNativeType();
return {
- name: s.getColumnName(),
+ name: columnName,
type: t === 'COMPLEX<json>' ? 'json' : t,
};
}),
@@ -429,12 +431,11 @@ export const SchemaStep = function SchemaStep(props:
SchemaStepProps) {
'sample',
);
- const columns = filterMap(sampleResponse.logicalSegmentSchema, ({ name,
type }) => {
- if (name === '__time') return;
+ const columns = getHeaderFromSampleResponse(sampleResponse).map(({ name,
type }) => {
return new Column({
name,
nativeType: type,
- sqlType: SqlType.fromNativeType(type).toString(),
+ sqlType: name === TIME_COLUMN ? 'TIMESTAMP' :
SqlType.fromNativeType(type).toString(),
});
});
diff --git
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
index 12a2e16e10..5eda56bac6 100644
---
a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
+++
b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
@@ -26,11 +26,12 @@ import { AutoForm, CenterMessage, LearnMore, Loader } from
'../../../components'
import type { InputFormat, InputSource } from '../../../druid-models';
import {
BATCH_INPUT_FORMAT_FIELDS,
+ DETECTION_TIMESTAMP_SPEC,
guessColumnTypeFromSampleResponse,
guessIsArrayFromSampleResponse,
inputFormatOutputsNumericStrings,
- PLACEHOLDER_TIMESTAMP_SPEC,
possibleDruidFormatForValues,
+ TIME_COLUMN,
} from '../../../druid-models';
import { useQueryManager } from '../../../hooks';
import { getLink } from '../../../links';
@@ -47,8 +48,6 @@ import { ParseDataTable } from
'../../load-data-view/parse-data-table/parse-data
import './input-format-step.scss';
-const noop = () => {};
-
export interface InputFormatAndMore {
inputFormat: InputFormat;
signature: SqlColumnDeclaration[];
@@ -93,7 +92,7 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
},
dataSchema: {
dataSource: 'sample',
- timestampSpec: PLACEHOLDER_TIMESTAMP_SPEC,
+ timestampSpec: DETECTION_TIMESTAMP_SPEC,
dimensionsSpec: {
useSchemaDiscovery: true,
},
@@ -123,6 +122,14 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
const possibleDruidFormat = possibleDruidFormatForValues(values);
if (!possibleDruidFormat) return;
+ // The __time column is special because it already is a TIMESTAMP so
there is no need parse it in any way
+ if (column === TIME_COLUMN) {
+ return {
+ column,
+ timeExpression: C(column),
+ };
+ }
+
const formatSql = timeFormatToSql(possibleDruidFormat);
if (!formatSql) return;
@@ -134,11 +141,17 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
)[0];
}
+ const headerNames = previewSampleResponse
+ ? getHeaderNamesFromSampleResponse(previewSampleResponse, 'ignoreIfZero')
+ : undefined;
+
const inputFormatAndMore =
- previewSampleResponse && AutoForm.isValidModel(inputFormat,
BATCH_INPUT_FORMAT_FIELDS)
+ previewSampleResponse &&
+ headerNames &&
+ AutoForm.isValidModel(inputFormat, BATCH_INPUT_FORMAT_FIELDS)
? {
inputFormat,
- signature: getHeaderNamesFromSampleResponse(previewSampleResponse,
true).map(name =>
+ signature: headerNames.map(name =>
SqlColumnDeclaration.create(
name,
SqlType.fromNativeType(
@@ -150,7 +163,7 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
),
),
),
- isArrays: getHeaderNamesFromSampleResponse(previewSampleResponse,
true).map(name =>
+ isArrays: headerNames.map(name =>
guessIsArrayFromSampleResponse(previewSampleResponse, name),
),
timeExpression: selectTimestamp ?
possibleTimeExpression?.timeExpression : undefined,
@@ -177,8 +190,6 @@ export const InputFormatStep = React.memo(function
InputFormatStep(props: InputF
canFlatten={false}
flattenedColumnsOnly={false}
flattenFields={EMPTY_ARRAY}
- onFlattenFieldSelect={noop}
- useInput
/>
)}
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
