This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 8432d82 Web console: Do not put __time in the dimensions list (#11085)
8432d82 is described below
commit 8432d82c48bf971fb9be1ffd777cc987e1d88b11
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Mon Apr 12 09:48:10 2021 -0700
Web console: Do not put __time in the dimensions list (#11085)
* Do not make time dimensions
* update e2e test
---
web-console/README.md | 2 +
.../load-data/config/configure-timestamp.ts | 32 +++++++++++++++
.../e2e-tests/component/load-data/data-loader.ts | 15 +++++++-
web-console/e2e-tests/tutorial-batch.spec.ts | 43 ++++++++++-----------
.../src/druid-models/transform-spec.spec.ts | 45 ++++++++++++++++++++++
web-console/src/druid-models/transform-spec.tsx | 10 ++++-
web-console/src/utils/sampler.ts | 5 ++-
7 files changed, 123 insertions(+), 29 deletions(-)
diff --git a/web-console/README.md b/web-console/README.md
index cd57370..8b8618e 100644
--- a/web-console/README.md
+++ b/web-console/README.md
@@ -85,6 +85,8 @@ when the test fails. For example, if
`e2e-tests/tutorial-batch.spec.ts` fails, i
Disabling headless mode while running the tests can be helpful. This can be
done via the `DRUID_E2E_TEST_HEADLESS`
environment variable, which defaults to `true`.
+Like so: `DRUID_E2E_TEST_HEADLESS=false npm run test-e2e`
+
#### Running against alternate web console
The environment variable `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT` can be used to
target a web console running on a
diff --git
a/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts
b/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts
new file mode 100644
index 0000000..73e9bfd
--- /dev/null
+++ b/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Data loader configure timestamp step configuration.
+ */
+export class ConfigureTimestampConfig {
+ constructor(props: ConfigureTimestampConfigProps) {
+ Object.assign(this, props);
+ }
+}
+
+interface ConfigureTimestampConfigProps {
+ readonly timestampExpression: string;
+}
+
+export interface ConfigureTimestampConfig extends
ConfigureTimestampConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts
b/web-console/e2e-tests/component/load-data/data-loader.ts
index df16e71..b71555e 100644
--- a/web-console/e2e-tests/component/load-data/data-loader.ts
+++ b/web-console/e2e-tests/component/load-data/data-loader.ts
@@ -21,6 +21,7 @@ import * as playwright from 'playwright-chromium';
import { clickButton, setLabeledInput, setLabeledTextarea } from
'../../util/playwright';
import { ConfigureSchemaConfig } from './config/configure-schema';
+import { ConfigureTimestampConfig } from './config/configure-timestamp';
import { PartitionConfig } from './config/partition';
import { PublishConfig } from './config/publish';
import { DataConnector } from './data-connector/data-connector';
@@ -45,7 +46,7 @@ export class DataLoader {
await this.connect(this.connector, this.connectValidator);
if (this.connector.needParse) {
await this.parseData();
- await this.parseTime();
+ await this.parseTime(this.configureTimestampConfig);
}
await this.transform();
await this.filter();
@@ -81,8 +82,11 @@ export class DataLoader {
await clickButton(this.page, 'Next: Parse time');
}
- private async parseTime() {
+ private async parseTime(configureTimestampConfig?: ConfigureTimestampConfig)
{
await this.page.waitForSelector('.parse-time-table');
+ if (configureTimestampConfig) {
+ await this.applyConfigureTimestampConfig(configureTimestampConfig);
+ }
await clickButton(this.page, 'Next: Transform');
}
@@ -102,6 +106,12 @@ export class DataLoader {
await clickButton(this.page, 'Next: Partition');
}
+ private async applyConfigureTimestampConfig(configureTimestampConfig:
ConfigureTimestampConfig) {
+ await clickButton(this.page, 'Expression');
+ await setLabeledInput(this.page, 'Expression',
configureTimestampConfig.timestampExpression);
+ await clickButton(this.page, 'Apply');
+ }
+
private async applyConfigureSchemaConfig(configureSchemaConfig:
ConfigureSchemaConfig) {
const rollupSelector = '//*[text()="Rollup"]';
const rollupInput = await this.page.$(`${rollupSelector}/input`);
@@ -161,6 +171,7 @@ interface DataLoaderProps {
readonly unifiedConsoleUrl: string;
readonly connector: DataConnector;
readonly connectValidator: (preview: string) => void;
+ readonly configureTimestampConfig?: ConfigureTimestampConfig;
readonly configureSchemaConfig: ConfigureSchemaConfig;
readonly partitionConfig: PartitionConfig;
readonly publishConfig: PublishConfig;
diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts
b/web-console/e2e-tests/tutorial-batch.spec.ts
index 842d7a5..ebb9544 100644
--- a/web-console/e2e-tests/tutorial-batch.spec.ts
+++ b/web-console/e2e-tests/tutorial-batch.spec.ts
@@ -22,17 +22,15 @@ import * as playwright from 'playwright-chromium';
import { DatasourcesOverview } from './component/datasources/overview';
import { IngestionOverview } from './component/ingestion/overview';
import { ConfigureSchemaConfig } from
'./component/load-data/config/configure-schema';
-import { SegmentGranularity } from './component/load-data/config/partition';
-import { PartitionConfig } from './component/load-data/config/partition';
+import { ConfigureTimestampConfig } from
'./component/load-data/config/configure-timestamp';
+import { PartitionConfig, SegmentGranularity } from
'./component/load-data/config/partition';
import { PublishConfig } from './component/load-data/config/publish';
import { LocalFileDataConnector } from
'./component/load-data/data-connector/local-file';
import { DataLoader } from './component/load-data/data-loader';
import { QueryOverview } from './component/query/overview';
import { saveScreenshotIfError } from './util/debug';
-import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
-import { UNIFIED_CONSOLE_URL } from './util/druid';
-import { createBrowser } from './util/playwright';
-import { createPage } from './util/playwright';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, UNIFIED_CONSOLE_URL } from
'./util/druid';
+import { createBrowser, createPage } from './util/playwright';
import { retryIfJestAssertionError } from './util/retry';
import { waitTillWebConsoleReady } from './util/setup';
@@ -60,26 +58,24 @@ describe('Tutorial: Loading a file', () => {
it('Loads data from local disk', async () => {
const testName = 'load-data-from-local-disk-';
const datasourceName = testName + ALL_SORTS_OF_CHARS + new
Date().toISOString();
- const dataConnector = new LocalFileDataConnector(page, {
- baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
- fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
- });
- const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
- const partitionConfig = new PartitionConfig({
- segmentGranularity: SegmentGranularity.DAY,
- timeIntervals: null,
- partitionsSpec: null,
- });
- const publishConfig = new PublishConfig({ datasourceName: datasourceName
});
-
const dataLoader = new DataLoader({
page: page,
unifiedConsoleUrl: UNIFIED_CONSOLE_URL,
- connector: dataConnector,
+ connector: new LocalFileDataConnector(page, {
+ baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
+ fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
+ }),
connectValidator: validateConnectLocalData,
- configureSchemaConfig: configureSchemaConfig,
- partitionConfig: partitionConfig,
- publishConfig: publishConfig,
+ configureTimestampConfig: new ConfigureTimestampConfig({
+ timestampExpression: 'timestamp_parse("time") + 1',
+ }),
+ configureSchemaConfig: new ConfigureSchemaConfig({ rollup: false }),
+ partitionConfig: new PartitionConfig({
+ segmentGranularity: SegmentGranularity.DAY,
+ timeIntervals: null,
+ partitionsSpec: null,
+ }),
+ publishConfig: new PublishConfig({ datasourceName: datasourceName }),
});
await saveScreenshotIfError(testName, page, async () => {
@@ -176,7 +172,7 @@ async function validateQuery(page: playwright.Page,
datasourceName: string) {
expect(results).toBeDefined();
expect(results.length).toBeGreaterThan(0);
expect(results[0]).toStrictEqual([
- /* __time */ '2015-09-12T00:46:58.771Z',
+ /* __time */ '2015-09-12T00:46:58.772Z',
/* added */ '36',
/* channel */ '#en.wikipedia',
/* cityName */ 'null',
@@ -195,6 +191,7 @@ async function validateQuery(page: playwright.Page,
datasourceName: string) {
/* page */ 'Talk:Oswald Tilghman',
/* regionIsoCode */ 'null',
/* regionName */ 'null',
+ /* time */ '2015-09-12T00:46:58.771Z',
/* user */ 'GELongstreet',
]);
}
diff --git a/web-console/src/druid-models/transform-spec.spec.ts
b/web-console/src/druid-models/transform-spec.spec.ts
new file mode 100644
index 0000000..b221e34
--- /dev/null
+++ b/web-console/src/druid-models/transform-spec.spec.ts
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getDimensionNamesFromTransforms } from './transform-spec';
+
+describe('transform-spec', () => {
+ describe('getDimensionNamesFromTransforms', () => {
+ it('does not return the __time column', () => {
+ expect(
+ getDimensionNamesFromTransforms([
+ {
+ type: 'expression',
+ name: 'fooPage',
+ expression: "concat('foo' + page)",
+ },
+ {
+ name: '__time',
+ type: 'expression',
+ expression: "timestamp_shift(timestamp_parse(timestamp), 'P3Y',
1)",
+ },
+ {
+ type: 'expression',
+ name: 'barPage',
+ expression: "concat('foo' + page)",
+ },
+ ]),
+ ).toEqual(['fooPage', 'barPage']);
+ });
+ });
+});
diff --git a/web-console/src/druid-models/transform-spec.tsx
b/web-console/src/druid-models/transform-spec.tsx
index c1dc340..18ecee0 100644
--- a/web-console/src/druid-models/transform-spec.tsx
+++ b/web-console/src/druid-models/transform-spec.tsx
@@ -22,6 +22,8 @@ import React from 'react';
import { ExternalLink, Field } from '../components';
import { getLink } from '../links';
+import { TIME_COLUMN } from './timestamp-spec';
+
export interface TransformSpec {
transforms?: Transform[];
filter?: Record<string, any>;
@@ -92,7 +94,7 @@ export function getTimestampExpressionFields(transforms:
Transform[]): Field<Tra
export function addTimestampTransform(transforms: Transform[]): Transform[] {
return [
{
- name: '__time',
+ name: TIME_COLUMN,
type: 'expression',
expression: '',
},
@@ -100,6 +102,10 @@ export function addTimestampTransform(transforms:
Transform[]): Transform[] {
}
export function removeTimestampTransform(transforms: Transform[]): Transform[]
| undefined {
- const newTransforms = transforms.filter(transform => transform.name !==
'__time');
+ const newTransforms = transforms.filter(transform => transform.name !==
TIME_COLUMN);
return newTransforms.length ? newTransforms : undefined;
}
+
+export function getDimensionNamesFromTransforms(transforms: Transform[]):
string[] {
+ return transforms.map(t => t.name).filter(n => n !== TIME_COLUMN);
+}
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 0bd9ae8..a18c66f 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -20,6 +20,7 @@ import * as JSONBig from 'json-bigint-native';
import {
DimensionsSpec,
+ getDimensionNamesFromTransforms,
getSpecType,
getTimestampSchema,
IngestionSpec,
@@ -464,7 +465,7 @@ export async function sampleForTransform(
sampleResponse: sampleResponseHack,
ignoreTimeColumn: true,
columnOrder: [TIME_COLUMN].concat(inputFormatColumns),
- }).concat(transforms.map(t => t.name)),
+ }).concat(getDimensionNamesFromTransforms(transforms)),
);
}
@@ -523,7 +524,7 @@ export async function sampleForFilter(
sampleResponse: sampleResponseHack,
ignoreTimeColumn: true,
columnOrder: [TIME_COLUMN].concat(inputFormatColumns),
- }).concat(transforms.map(t => t.name)),
+ }).concat(getDimensionNamesFromTransforms(transforms)),
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
