This is an automated email from the ASF dual-hosted git repository.
fjy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new d677c83 Web console: Power up the data loader init step (#7947)
d677c83 is described below
commit d677c83ce43d4a31e1c08d68c2aa28ff13bdc753
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Wed Jun 26 15:50:48 2019 -0700
Web console: Power up the data loader init step (#7947)
* Power up the data loader init step
* update snapshot
* normalize spec
* allow deselect
* added HDFS tile
* update border style
* text updates
* goodies
* new reset icon
---
web-console/assets/druid.png | Bin 0 -> 15699 bytes
web-console/assets/example.png | Bin 0 -> 8873 bytes
web-console/assets/hadoop.png | Bin 0 -> 47082 bytes
web-console/assets/http.png | Bin 0 -> 10407 bytes
web-console/assets/kafka.png | Bin 0 -> 18022 bytes
web-console/assets/kinesis.png | Bin 0 -> 15631 bytes
web-console/assets/local.png | Bin 0 -> 9211 bytes
web-console/assets/other.png | Bin 0 -> 9549 bytes
web-console/assets/static-google-blobstore.png | Bin 0 -> 19344 bytes
web-console/assets/static-s3.png | Bin 0 -> 16541 bytes
web-console/package-lock.json | 12 +-
web-console/package.json | 3 +-
web-console/script/cp-to | 1 +
web-console/script/create-sql-function-doc.js | 16 +-
web-console/src/utils/ingestion-spec.tsx | 130 ++++++--
.../__snapshots__/load-data-view.spec.tsx.snap | 184 +++++++++--
.../src/views/load-data-view/load-data-view.scss | 43 ++-
.../src/views/load-data-view/load-data-view.tsx | 343 +++++++++++++++------
18 files changed, 561 insertions(+), 171 deletions(-)
diff --git a/web-console/assets/druid.png b/web-console/assets/druid.png
new file mode 100644
index 0000000..b494638
Binary files /dev/null and b/web-console/assets/druid.png differ
diff --git a/web-console/assets/example.png b/web-console/assets/example.png
new file mode 100644
index 0000000..faf9678
Binary files /dev/null and b/web-console/assets/example.png differ
diff --git a/web-console/assets/hadoop.png b/web-console/assets/hadoop.png
new file mode 100644
index 0000000..9b69d38
Binary files /dev/null and b/web-console/assets/hadoop.png differ
diff --git a/web-console/assets/http.png b/web-console/assets/http.png
new file mode 100644
index 0000000..ee041ba
Binary files /dev/null and b/web-console/assets/http.png differ
diff --git a/web-console/assets/kafka.png b/web-console/assets/kafka.png
new file mode 100644
index 0000000..0f0eb7c
Binary files /dev/null and b/web-console/assets/kafka.png differ
diff --git a/web-console/assets/kinesis.png b/web-console/assets/kinesis.png
new file mode 100644
index 0000000..d87a3cb
Binary files /dev/null and b/web-console/assets/kinesis.png differ
diff --git a/web-console/assets/local.png b/web-console/assets/local.png
new file mode 100644
index 0000000..25c523b
Binary files /dev/null and b/web-console/assets/local.png differ
diff --git a/web-console/assets/other.png b/web-console/assets/other.png
new file mode 100644
index 0000000..b2b977d
Binary files /dev/null and b/web-console/assets/other.png differ
diff --git a/web-console/assets/static-google-blobstore.png
b/web-console/assets/static-google-blobstore.png
new file mode 100644
index 0000000..960cbcf
Binary files /dev/null and b/web-console/assets/static-google-blobstore.png
differ
diff --git a/web-console/assets/static-s3.png b/web-console/assets/static-s3.png
new file mode 100644
index 0000000..e73112e
Binary files /dev/null and b/web-console/assets/static-s3.png differ
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index fde7305..7f7c804 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -229,9 +229,9 @@
}
},
"@blueprintjs/core": {
- "version": "3.15.1",
- "resolved":
"https://registry.npmjs.org/@blueprintjs/core/-/core-3.15.1.tgz";,
- "integrity":
"sha512-M8ltbqqlMZuZ6SEuqo/3Fr59ZcUfd8Er7ocbm7EACVfRW7dRhOCd/TKkf2kfICNtCDwznwXk0iAePLXZhUGtQg==",
+ "version": "3.16.2",
+ "resolved":
"https://registry.npmjs.org/@blueprintjs/core/-/core-3.16.2.tgz";,
+ "integrity":
"sha512-u+mSITWaNDwbdaPrbKx9XyxGsF4725SCAidWjd367ysX7AxCo4PK4SsFQVfXNylXpVWHQhJZekuo7+hdksc9lA==",
"requires": {
"@blueprintjs/icons": "^3.8.0",
"@types/dom4": "^2.0.1",
@@ -246,9 +246,9 @@
}
},
"@blueprintjs/icons": {
- "version": "3.8.0",
- "resolved":
"https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.8.0.tgz";,
- "integrity":
"sha512-yHaRQ3vfV9Gf3foZ4ONtxddz+u5ufkHqHj8Ia5VhPbFgG4el+cPdmsGGIIM72rgKS1KQa5Ay+ggjpByUlXvrKg==",
+ "version": "3.9.0",
+ "resolved":
"https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.9.0.tgz";,
+ "integrity":
"sha512-kq1Bh6PtOF4PcuxcDme8NmnSlkfO0IV89FriZGo6zSA1+OOzSwzvoKqa6S7vJe8xCPPLO5r7lE9AjeOuGeH97g==",
"requires": {
"classnames": "^2.2",
"tslib": "^1.9.0"
diff --git a/web-console/package.json b/web-console/package.json
index 3e07b65..a4369d6 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -50,7 +50,8 @@
"stylelint": "stylelint 'src/**/*.scss'"
},
"dependencies": {
- "@blueprintjs/core": "^3.15.1",
+ "@blueprintjs/core": "^3.16.2",
+ "@blueprintjs/icons": "^3.9.0",
"@types/memoize-one": "^4.1.1",
"axios": "^0.19.0",
"brace": "^0.11.1",
diff --git a/web-console/script/cp-to b/web-console/script/cp-to
index b8ad2eb..3943a10 100755
--- a/web-console/script/cp-to
+++ b/web-console/script/cp-to
@@ -25,3 +25,4 @@ cp -r coordinator-console "$1"
cp -r old-console "$1"
cp -r pages "$1"
cp -r public "$1"
+cp -r assets "$1"
diff --git a/web-console/script/create-sql-function-doc.js
b/web-console/script/create-sql-function-doc.js
index 51cac06..f8dd638 100755
--- a/web-console/script/create-sql-function-doc.js
+++ b/web-console/script/create-sql-function-doc.js
@@ -34,27 +34,31 @@ const readDoc = async () => {
if (functionMatch) {
functionDocs.push({
syntax: functionMatch[1],
- description: functionMatch[2]
- })
+ description: functionMatch[2],
+ });
}
const dataTypeMatch = line.match(/^\|([A-Z]+)\|([A-Z]+)\|(.*)\|(.*)\|$/);
if (dataTypeMatch) {
dataTypeDocs.push({
syntax: dataTypeMatch[1],
- description: dataTypeMatch[4] || `Druid runtime type:
${dataTypeMatch[2]}`
- })
+ description: dataTypeMatch[4] || `Druid runtime type:
${dataTypeMatch[2]}`,
+ });
}
}
// Make sure there are at least 10 functions for sanity
if (functionDocs.length < 10) {
- throw new Error(`Did not find enough function entries did the structure of
'${readfile}' change? (found ${functionDocs.length})`);
+ throw new Error(
+ `Did not find enough function entries did the structure of '${readfile}'
change? (found ${functionDocs.length})`,
+ );
}
// Make sure there are at least 5 data types for sanity
if (dataTypeDocs.length < 10) {
- throw new Error(`Did not find enough data type entries did the structure
of '${readfile}' change? (found ${dataTypeDocs.length})`);
+ throw new Error(
+ `Did not find enough data type entries did the structure of
'${readfile}' change? (found ${dataTypeDocs.length})`,
+ );
}
const content = `/*
diff --git a/web-console/src/utils/ingestion-spec.tsx
b/web-console/src/utils/ingestion-spec.tsx
index 84fa63c..dc115cf 100644
--- a/web-console/src/utils/ingestion-spec.tsx
+++ b/web-console/src/utils/ingestion-spec.tsx
@@ -37,6 +37,10 @@ export interface IngestionSpec {
tuningConfig?: TuningConfig;
}
+export function isEmptyIngestionSpec(spec: IngestionSpec) {
+ return Object.keys(spec).length === 0;
+}
+
export type IngestionType = 'kafka' | 'kinesis' | 'index_hadoop' | 'index' |
'index_parallel';
// A combination of IngestionType and firehose
@@ -48,6 +52,9 @@ export type IngestionComboType =
| 'index:static-s3'
| 'index:static-google-blobstore';
+// Some extra values that can be selected in the initial screen
+export type IngestionComboTypeWithExtra = IngestionComboType | 'hadoop' |
'example' | 'other';
+
function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType):
string {
switch (ingestionType) {
case 'kafka':
@@ -87,6 +94,65 @@ export function getIngestionComboType(spec: IngestionSpec):
IngestionComboType |
return null;
}
+export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra):
string {
+ switch (ingestionType) {
+ case 'index:local':
+ return 'Local disk';
+
+ case 'index:http':
+ return 'HTTP(s)';
+
+ case 'index:static-s3':
+ return 'Amazon S3';
+
+ case 'index:static-google-blobstore':
+ return 'Google Cloud Storage';
+
+ case 'kafka':
+ return 'Apache Kafka';
+
+ case 'kinesis':
+ return 'Amazon Kinesis';
+
+ case 'hadoop':
+ return 'HDFS';
+
+ case 'example':
+ return 'Example data';
+
+ case 'other':
+ return 'Other';
+
+ default:
+ return 'Unknown ingestion';
+ }
+}
+
+export function getIngestionImage(ingestionType: IngestionComboTypeWithExtra):
string {
+ const parts = ingestionType.split(':');
+ if (parts.length === 2) return parts[1];
+ return ingestionType;
+}
+
+export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra):
string | null {
+ switch (ingestionType) {
+ case 'index:static-s3':
+ return 'druid-s3-extensions';
+
+ case 'index:static-google-blobstore':
+ return 'druid-google-extensions';
+
+ case 'kafka':
+ return 'druid-kafka-indexing-service';
+
+ case 'kinesis':
+ return 'druid-kinesis-indexing-service';
+
+ default:
+ return null;
+ }
+}
+
// --------------
export interface DataSchema {
@@ -138,7 +204,7 @@ export function getRollup(spec: IngestionSpec): boolean {
return typeof specRollup === 'boolean' ? specRollup : true;
}
-export function getSpecType(spec: IngestionSpec): IngestionType | undefined {
+export function getSpecType(spec: Partial<IngestionSpec>): IngestionType |
undefined {
return (
deepGet(spec, 'type') || deepGet(spec, 'ioConfig.type') || deepGet(spec,
'tuningConfig.type')
);
@@ -158,13 +224,21 @@ export function changeParallel(spec: IngestionSpec,
parallel: boolean): Ingestio
* Make sure that the types are set in the root, ioConfig, and tuningConfig
* @param spec
*/
-export function normalizeSpecType(spec: IngestionSpec) {
+export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
+ if (!spec || typeof spec !== 'object') {
+ // This does not match the type of IngestionSpec but this dialog is robust
enough to deal with anything but spec must be an object
+ spec = {};
+ }
+
+ // Make sure that if we actually get a task payload we extract the spec
+ if (typeof (spec as any).spec === 'object') spec = (spec as any).spec;
+
const specType = getSpecType(spec);
- if (!specType) return spec;
+ if (!specType) return spec as IngestionSpec;
if (!deepGet(spec, 'type')) spec = deepSet(spec, 'type', specType);
if (!deepGet(spec, 'ioConfig.type')) spec = deepSet(spec, 'ioConfig.type',
specType);
if (!deepGet(spec, 'tuningConfig.type')) spec = deepSet(spec,
'tuningConfig.type', specType);
- return spec;
+ return spec as IngestionSpec;
}
const PARSE_SPEC_FORM_FIELDS: Field<ParseSpec>[] = [
@@ -851,7 +925,7 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
],
info: (
<>
- The AWS Kinesis stream endpoint for a region. You can find a
list of endpoints{' '}
+ The Amazon Kinesis stream endpoint for a region. You can find a
list of endpoints{' '}
<ExternalLink
href="http://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region";>
here
</ExternalLink>
@@ -1662,40 +1736,40 @@ export interface Bitmap {
// --------------
-export function getBlankSpec(comboType: IngestionComboType): IngestionSpec {
+export function updateIngestionType(
+ spec: IngestionSpec,
+ comboType: IngestionComboType,
+): IngestionSpec {
let [ingestionType, firehoseType] = comboType.split(':');
if (ingestionType === 'index') ingestionType = 'index_parallel';
const ioAndTuningConfigType = ingestionTypeToIoAndTuningConfigType(
ingestionType as IngestionType,
);
- const granularitySpec: GranularitySpec = {
- type: 'uniform',
- segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR',
- queryGranularity: 'HOUR',
- };
-
- const spec: IngestionSpec = {
- type: ingestionType,
- dataSchema: {
- dataSource: 'new-data-source',
- granularitySpec,
- },
- ioConfig: {
- type: ioAndTuningConfigType,
- },
- tuningConfig: {
- type: ioAndTuningConfigType,
- },
- } as any;
+ let newSpec = spec;
+ newSpec = deepSet(newSpec, 'type', ingestionType);
+ newSpec = deepSet(newSpec, 'ioConfig.type', ioAndTuningConfigType);
+ newSpec = deepSet(newSpec, 'tuningConfig.type', ioAndTuningConfigType);
if (firehoseType) {
- spec.ioConfig.firehose = {
- type: firehoseType,
+ newSpec = deepSet(newSpec, 'ioConfig.firehose', { type: firehoseType });
+ }
+
+ if (!deepGet(spec, 'dataSchema.dataSource')) {
+ newSpec = deepSet(newSpec, 'dataSchema.dataSource', 'new-data-source');
+ }
+
+ if (!deepGet(spec, 'dataSchema.granularitySpec')) {
+ const granularitySpec: GranularitySpec = {
+ type: 'uniform',
+ segmentGranularity: ingestionType === 'index_parallel' ? 'DAY' : 'HOUR',
+ queryGranularity: 'HOUR',
};
+
+ newSpec = deepSet(newSpec, 'dataSchema.granularitySpec', granularitySpec);
}
- return spec;
+ return newSpec;
}
export function fillParser(spec: IngestionSpec, sampleData: string[]):
IngestionSpec {
diff --git
a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
index 829003f..9718f80 100644
---
a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
+++
b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
@@ -2,41 +2,169 @@
exports[`load data view matches snapshot 1`] = `
<div
- className="load-data-view app-view init"
+ className="load-data-view app-view welcome"
>
<div
- className="intro"
+ className="bp3-tabs step-nav"
>
- Please specify where your raw data is located
+ <div
+ className="step-section"
+ key="Connect and parse raw data"
+ >
+ <div
+ className="step-nav-l1"
+ >
+ Connect and parse raw data
+ </div>
+ <Blueprint3.ButtonGroup
+ className="step-nav-l2"
+ >
+ <Blueprint3.Button
+ active={true}
+ className="welcome"
+ icon={false}
+ key="welcome"
+ onClick={[Function]}
+ text="Start"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="connect"
+ icon={false}
+ key="connect"
+ onClick={[Function]}
+ text="Connect"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="parser"
+ icon={false}
+ key="parser"
+ onClick={[Function]}
+ text="Parse data"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="timestamp"
+ icon={false}
+ key="timestamp"
+ onClick={[Function]}
+ text="Parse time"
+ />
+ </Blueprint3.ButtonGroup>
+ </div>
+ <div
+ className="step-section"
+ key="Transform and configure schema"
+ >
+ <div
+ className="step-nav-l1"
+ >
+ Transform and configure schema
+ </div>
+ <Blueprint3.ButtonGroup
+ className="step-nav-l2"
+ >
+ <Blueprint3.Button
+ active={false}
+ className="transform"
+ icon={false}
+ key="transform"
+ onClick={[Function]}
+ text="Transform"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="filter"
+ icon={false}
+ key="filter"
+ onClick={[Function]}
+ text="Filter"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="schema"
+ icon={false}
+ key="schema"
+ onClick={[Function]}
+ text="Configure schema"
+ />
+ </Blueprint3.ButtonGroup>
+ </div>
+ <div
+ className="step-section"
+ key="Tune parameters"
+ >
+ <div
+ className="step-nav-l1"
+ >
+ Tune parameters
+ </div>
+ <Blueprint3.ButtonGroup
+ className="step-nav-l2"
+ >
+ <Blueprint3.Button
+ active={false}
+ className="partition"
+ icon={false}
+ key="partition"
+ onClick={[Function]}
+ text="Partition"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="tuning"
+ icon={false}
+ key="tuning"
+ onClick={[Function]}
+ text="Tune"
+ />
+ <Blueprint3.Button
+ active={false}
+ className="publish"
+ icon={false}
+ key="publish"
+ onClick={[Function]}
+ text="Publish"
+ />
+ </Blueprint3.ButtonGroup>
+ </div>
+ <div
+ className="step-section"
+ key="Verify and submit"
+ >
+ <div
+ className="step-nav-l1"
+ >
+ Verify and submit
+ </div>
+ <Blueprint3.ButtonGroup
+ className="step-nav-l2"
+ >
+ <Blueprint3.Button
+ active={false}
+ className="spec"
+ icon="manually-entered-data"
+ key="spec"
+ onClick={[Function]}
+ text="Edit JSON spec"
+ />
+ </Blueprint3.ButtonGroup>
+ </div>
</div>
<div
- className="cards"
+ className="main"
+ />
+ <div
+ className="control"
>
- <Blueprint3.Card
- elevation={0}
- interactive={true}
- onClick={[Function]}
- >
- Other (streaming)
- </Blueprint3.Card>
- <Blueprint3.Card
- elevation={0}
- interactive={true}
- onClick={[Function]}
+ <Blueprint3.Callout
+ className="intro"
>
- Other (batch)
- </Blueprint3.Card>
+ <p>
+ Please specify where your raw data is located
+ </p>
+ </Blueprint3.Callout>
</div>
- <Blueprint3.Alert
- canEscapeKeyCancel={false}
- canOutsideClickCancel={false}
- confirmButtonText="Close"
- icon="warning-sign"
- intent="warning"
- isOpen={false}
- onConfirm={[Function]}
- >
- <p />
- </Blueprint3.Alert>
</div>
`;
diff --git a/web-console/src/views/load-data-view/load-data-view.scss
b/web-console/src/views/load-data-view/load-data-view.scss
index acd7ae0..847bea4 100644
--- a/web-console/src/views/load-data-view/load-data-view.scss
+++ b/web-console/src/views/load-data-view/load-data-view.scss
@@ -27,32 +27,45 @@
'main ctrl'
'main next';
- &.init {
- display: block;
+ &.welcome {
+ .main {
+ margin-left: -10px;
- & > * {
- margin-bottom: 15px;
- }
-
- .intro {
- font-size: 20px;
- }
-
- .cards {
.bp3-card {
+ position: relative;
display: inline-block;
vertical-align: top;
width: 250px;
height: 140px;
- margin-right: 15px;
+ margin-left: 15px;
margin-bottom: 15px;
- font-size: 24px;
+ font-size: 16px;
text-align: center;
- padding-top: 47px;
+
+ & > * {
+ user-select: none;
+ pointer-events: none;
+ }
+
+ &.active::after {
+ position: absolute;
+ top: 0;
+ right: 0;
+ bottom: 0;
+ left: 0;
+ content: '';
+ border: 2px solid #48aff0;
+ border-radius: 2px;
+ }
&.disabled {
opacity: 0.4;
}
+
+ img {
+ width: 100px;
+ display: inline-block;
+ }
}
}
}
@@ -192,7 +205,7 @@
text-align: right;
padding: 0 5px;
- .prev {
+ .left {
float: left;
}
}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 0c3e426..efc853d 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -25,6 +25,7 @@ import {
Card,
Classes,
Code,
+ Elevation,
FormGroup,
H5,
HTMLSelect,
@@ -50,6 +51,7 @@ import {
} from '../../components';
import { AsyncActionDialog } from '../../dialogs';
import { AppToaster } from '../../singletons/toaster';
+import { UrlBaser } from '../../singletons/url-baser';
import {
filterMap,
getDruidErrorMessage,
@@ -72,18 +74,20 @@ import {
fillDataSourceName,
fillParser,
FlattenField,
- getBlankSpec,
getDimensionMode,
getDimensionSpecFormFields,
getEmptyTimestampSpec,
getFilterFormFields,
getFlattenFieldFormFields,
getIngestionComboType,
+ getIngestionImage,
+ getIngestionTitle,
getIoConfigFormFields,
getIoConfigTuningFormFields,
getMetricSpecFormFields,
getParseSpecFormFields,
getPartitionRelatedTuningSpecFormFields,
+ getRequiredModule,
getRollup,
getSpecType,
getTimestampSpecFormFields,
@@ -91,16 +95,17 @@ import {
getTuningSpecFormFields,
GranularitySpec,
hasParallelAbility,
- IngestionComboType,
+ IngestionComboTypeWithExtra,
IngestionSpec,
IoConfig,
isColumnTimestampSpec,
+ isEmptyIngestionSpec,
isParallel,
issueWithIoConfig,
issueWithParser,
joinFilter,
MetricSpec,
- normalizeSpecType,
+ normalizeSpec,
Parser,
ParseSpec,
parseSpecHasFlatten,
@@ -108,6 +113,7 @@ import {
TimestampSpec,
Transform,
TuningConfig,
+ updateIngestionType,
} from '../../utils/ingestion-spec';
import { deepDelete, deepGet, deepSet } from '../../utils/object-change';
import {
@@ -163,6 +169,7 @@ function getTimestampSpec(headerAndRows: HeaderAndRows |
null): TimestampSpec {
}
type Step =
+ | 'welcome'
| 'connect'
| 'parser'
| 'timestamp'
@@ -172,9 +179,11 @@ type Step =
| 'partition'
| 'tuning'
| 'publish'
- | 'json-spec'
+ | 'spec'
| 'loading';
+
const STEPS: Step[] = [
+ 'welcome',
'connect',
'parser',
'timestamp',
@@ -184,18 +193,19 @@ const STEPS: Step[] = [
'partition',
'tuning',
'publish',
- 'json-spec',
+ 'spec',
'loading',
];
const SECTIONS: { name: string; steps: Step[] }[] = [
- { name: 'Connect and parse raw data', steps: ['connect', 'parser',
'timestamp'] },
+ { name: 'Connect and parse raw data', steps: ['welcome', 'connect',
'parser', 'timestamp'] },
{ name: 'Transform and configure schema', steps: ['transform', 'filter',
'schema'] },
{ name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] },
- { name: 'Verify and submit', steps: ['json-spec'] },
+ { name: 'Verify and submit', steps: ['spec'] },
];
const VIEW_TITLE: Record<Step, string> = {
+ welcome: 'Start',
connect: 'Connect',
parser: 'Parse data',
timestamp: 'Parse time',
@@ -205,7 +215,7 @@ const VIEW_TITLE: Record<Step, string> = {
partition: 'Partition',
tuning: 'Tune',
publish: 'Publish',
- 'json-spec': 'Edit JSON spec',
+ spec: 'Edit JSON spec',
loading: 'Loading',
};
@@ -224,9 +234,11 @@ export interface LoadDataViewState {
newRollup: boolean | null;
newDimensionMode: DimensionMode | null;
- // general
+ // welcome
overlordModules: string[] | null;
- overlordModuleNeededMessage: string | null;
+ selectedComboType: IngestionComboTypeWithExtra | null;
+
+ // general
sampleStrategy: SampleStrategy;
columnFilter: string;
specialColumnsOnly: boolean;
@@ -278,7 +290,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
let spec =
parseJson(String(localStorageGet(LocalStorageKeys.INGESTION_SPEC)));
if (!spec || typeof spec !== 'object') spec = {};
this.state = {
- step: 'connect',
+ step: 'welcome',
spec,
cacheKey: undefined,
@@ -287,9 +299,11 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
newRollup: null,
newDimensionMode: null,
- // general
+ // welcome
overlordModules: null,
- overlordModuleNeededMessage: null,
+ selectedComboType: null,
+
+ // general
sampleStrategy: 'start',
columnFilter: '',
specialColumnsOnly: false,
@@ -329,6 +343,8 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
}
componentDidMount(): void {
+ const { spec } = this.state;
+
this.getOverlordModules();
if (this.props.initTaskId) {
this.updateStep('loading');
@@ -336,6 +352,8 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
} else if (this.props.initSupervisorId) {
this.updateStep('loading');
this.getSupervisorJson();
+ } else if (isEmptyIngestionSpec(spec)) {
+ this.updateStep('welcome');
} else {
this.updateStep('connect');
}
@@ -380,28 +398,18 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
}
private updateSpec = (newSpec: IngestionSpec) => {
- if (!newSpec || typeof newSpec !== 'object') {
- // This does not match the type of IngestionSpec but this dialog is
robust enough to deal with anything but spec must be an object
- newSpec = {} as any;
- }
+ newSpec = normalizeSpec(newSpec);
this.setState({ spec: newSpec });
localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSON.stringify(newSpec));
};
render() {
- const { step, spec } = this.state;
- if (!Object.keys(spec).length && !this.props.initSupervisorId &&
!this.props.initTaskId) {
- return (
- <div className={classNames('load-data-view', 'app-view', 'init')}>
- {this.renderInitStep()}
- </div>
- );
- }
-
+ const { step } = this.state;
return (
<div className={classNames('load-data-view', 'app-view', step)}>
{this.renderStepNav()}
+ {step === 'welcome' && this.renderWelcomeStep()}
{step === 'connect' && this.renderConnectStep()}
{step === 'parser' && this.renderParserStep()}
{step === 'timestamp' && this.renderTimestampStep()}
@@ -414,7 +422,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
{step === 'tuning' && this.renderTuningStep()}
{step === 'publish' && this.renderPublishStep()}
- {step === 'json-spec' && this.renderJsonSpecStep()}
+ {step === 'spec' && this.renderSpecStep()}
{step === 'loading' && this.renderLoading()}
{this.renderResetConfirm()}
@@ -437,7 +445,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
key={s}
active={s === step}
onClick={() => this.updateStep(s)}
- icon={s === 'json-spec' && IconNames.MANUALLY_ENTERED_DATA}
+ icon={s === 'spec' && IconNames.MANUALLY_ENTERED_DATA}
text={VIEW_TITLE[s]}
/>
))}
@@ -484,79 +492,234 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
// ==================================================================
- initWith(comboType: IngestionComboType) {
- this.setState({
- spec: getBlankSpec(comboType),
- });
- setTimeout(() => {
- this.updateStep('connect');
- }, 10);
- }
-
- renderIngestionCard(title: string, comboType: IngestionComboType,
requiredModule?: string) {
- const { overlordModules } = this.state;
+ renderIngestionCard(comboType: IngestionComboTypeWithExtra) {
+ const { overlordModules, selectedComboType } = this.state;
if (!overlordModules) return null;
+ const requiredModule = getRequiredModule(comboType);
const goodToGo = !requiredModule ||
overlordModules.includes(requiredModule);
return (
<Card
- className={classNames({ disabled: !goodToGo })}
+ className={classNames({ disabled: !goodToGo, active: selectedComboType
=== comboType })}
interactive
onClick={() => {
- if (goodToGo) {
- this.initWith(comboType);
- } else {
- this.setState({
- overlordModuleNeededMessage: `${title} ingestion requires the
'${requiredModule}' to be loaded.`,
- });
- }
+ this.setState({ selectedComboType: selectedComboType !== comboType ?
comboType : null });
}}
>
- {title}
+ <img
src={UrlBaser.base(`/assets/${getIngestionImage(comboType)}.png`)} />
+ <p>{getIngestionTitle(comboType)}</p>
</Card>
);
}
- renderInitStep() {
- const { goToTask } = this.props;
- const { overlordModuleNeededMessage } = this.state;
+ renderWelcomeStep() {
+ const { spec } = this.state;
return (
<>
- <div className="intro">Please specify where your raw data is
located</div>
-
- <div className="cards">
- {this.renderIngestionCard('Apache Kafka', 'kafka',
'druid-kafka-indexing-service')}
- {this.renderIngestionCard('AWS Kinesis', 'kinesis',
'druid-kinesis-indexing-service')}
- {this.renderIngestionCard('HTTP(s)', 'index:http')}
- {this.renderIngestionCard('AWS S3', 'index:static-s3',
'druid-s3-extensions')}
- {this.renderIngestionCard(
- 'Google Cloud Storage',
- 'index:static-google-blobstore',
- 'druid-google-extensions',
+ <div className="main">
+ {this.renderIngestionCard('kafka')}
+ {this.renderIngestionCard('kinesis')}
+ {this.renderIngestionCard('index:static-s3')}
+ {this.renderIngestionCard('index:static-google-blobstore')}
+ {this.renderIngestionCard('hadoop')}
+ {this.renderIngestionCard('index:http')}
+ {this.renderIngestionCard('index:local')}
+ {/* this.renderIngestionCard('example') */}
+ {this.renderIngestionCard('other')}
+ </div>
+ <div className="control">
+ <Callout
className="intro">{this.renderWelcomeStepMessage()}</Callout>
+ {this.renderWelcomeStepControls()}
+ {!isEmptyIngestionSpec(spec) && (
+ <Button icon={IconNames.RESET} text="Reset spec"
onClick={this.handleResetConfirm} />
)}
- {this.renderIngestionCard('Local disk', 'index:local')}
- <Card interactive onClick={() => goToTask(null, 'supervisor')}>
- Other (streaming)
- </Card>
- <Card interactive onClick={() => goToTask(null, 'task')}>
- Other (batch)
- </Card>
</div>
-
- <Alert
- icon={IconNames.WARNING_SIGN}
- intent={Intent.WARNING}
- isOpen={Boolean(overlordModuleNeededMessage)}
- confirmButtonText="Close"
- onConfirm={() => this.setState({ overlordModuleNeededMessage: null
})}
- >
- <p>{overlordModuleNeededMessage}</p>
- </Alert>
</>
);
}
+ renderWelcomeStepMessage() {
+ const { selectedComboType } = this.state;
+
+ if (!selectedComboType) {
+ return <p>Please specify where your raw data is located</p>;
+ }
+
+ const issue = this.selectedIngestionTypeIssue();
+ if (issue) return issue;
+
+ switch (selectedComboType) {
+ case 'index:http':
+ return (
+ <>
+ <p>Load data accessible through HTTP(s).</p>
+ <p>
+ Data must be in a text format and the HTTP(s) endpoint must be
reachable by every
+ Druid process in the cluster.
+ </p>
+ </>
+ );
+
+ case 'index:local':
+ return (
+ <>
+ <p>
+ <em>Recommended only in single server deployments.</em>
+ </p>
+ <p>Load data directly from a local file.</p>
+ <p>
+ Files must be in a text format and must be accessible to all the
Druid processes in
+ the cluster.
+ </p>
+ </>
+ );
+
+ case 'index:static-s3':
+ return <p>Load text based data from Amazon S3.</p>;
+
+ case 'index:static-google-blobstore':
+ return <p>Load text based data from the Google Blobstore.</p>;
+
+ case 'kafka':
+ return <p>Load streaming data in real-time from Apache Kafka.</p>;
+
+ case 'kinesis':
+ return <p>Load streaming data in real-time from Amazon Kinesis.</p>;
+
+ case 'hadoop':
+ return (
+ <>
+ <p>
+ <em>Data loader support coming soon!</em>
+ </p>
+ <p>
+ You can not ingest data from HDFS via the data loader at this
time, however you can
+ ingest it through a Druid task.
+ </p>
+ <p>
+ Please follow{' '}
+ <ExternalLink
href="https://druid.apache.org/docs/latest/ingestion/hadoop.html";>
+ the hadoop docs
+ </ExternalLink>{' '}
+ and submit a JSON spec to start the task.
+ </p>
+ </>
+ );
+
+ case 'example':
+ return <p>Pick one of these examples to get you started.</p>;
+
+ case 'other':
+ return (
+ <p>
+ If you do not see your source of raw data here, you can try to
ingest it by submitting a{' '}
+ <ExternalLink
href="https://druid.apache.org/docs/latest/ingestion/index.html";>
+ JSON task or supervisor spec
+ </ExternalLink>
+ .
+ </p>
+ );
+
+ default:
+ return <p>Unknown ingestion type.</p>;
+ }
+ }
+
+ renderWelcomeStepControls() {
+ const { goToTask } = this.props;
+ const { spec, selectedComboType } = this.state;
+
+ const issue = this.selectedIngestionTypeIssue();
+ if (issue) return null;
+
+ switch (selectedComboType) {
+ case 'index:http':
+ case 'index:local':
+ case 'index:static-s3':
+ case 'index:static-google-blobstore':
+ case 'kafka':
+ case 'kinesis':
+ return (
+ <FormGroup>
+ <Button
+ text="Connect data"
+ rightIcon={IconNames.ARROW_RIGHT}
+ onClick={() => {
+ this.setState({
+ spec: updateIngestionType(spec, selectedComboType as any),
+ });
+ setTimeout(() => {
+ this.updateStep('connect');
+ }, 10);
+ }}
+ intent={Intent.PRIMARY}
+ />
+ </FormGroup>
+ );
+
+ case 'hadoop':
+ return (
+ <FormGroup>
+ <Button
+ text="Submit task"
+ rightIcon={IconNames.ARROW_RIGHT}
+ onClick={() => goToTask(null, 'task')}
+ intent={Intent.PRIMARY}
+ />
+ </FormGroup>
+ );
+
+ case 'example':
+ return null;
+
+ case 'other':
+ return (
+ <>
+ <FormGroup>
+ <Button
+ text="Submit supervisor"
+ rightIcon={IconNames.ARROW_RIGHT}
+ onClick={() => goToTask(null, 'supervisor')}
+ intent={Intent.PRIMARY}
+ />
+ </FormGroup>
+ <FormGroup>
+ <Button
+ text="Submit task"
+ rightIcon={IconNames.ARROW_RIGHT}
+ onClick={() => goToTask(null, 'task')}
+ intent={Intent.PRIMARY}
+ />
+ </FormGroup>
+ </>
+ );
+
+ default:
+ return null;
+ }
+ }
+
+ selectedIngestionTypeIssue(): JSX.Element | null {
+ const { selectedComboType, overlordModules } = this.state;
+ if (!selectedComboType || !overlordModules) return null;
+
+ const requiredModule = getRequiredModule(selectedComboType);
+ if (!requiredModule || overlordModules.includes(requiredModule)) return
null;
+
+ return (
+ <p>
+ {`${getIngestionTitle(selectedComboType)} ingestion requires the `}
+ <strong>{requiredModule}</strong>
+ {` extension to be loaded.`}
+ </p>
+ );
+ }
+
+ private handleResetConfirm = () => {
+ this.setState({ showResetConfirm: true });
+ };
+
renderResetConfirm() {
const { showResetConfirm } = this.state;
if (!showResetConfirm) return null;
@@ -713,8 +876,6 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
if (!inputQueryState.data) return;
this.updateSpec(fillDataSourceName(fillParser(spec,
inputQueryState.data)));
},
- prevLabel: 'Restart',
- onPrevStep: () => this.setState({ showResetConfirm: true }),
})}
</>
);
@@ -2398,8 +2559,8 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
try {
const resp = await
axios.get(`/druid/indexer/v1/supervisor/${initSupervisorId}`);
- this.updateSpec(normalizeSpecType(resp.data));
- this.updateStep('json-spec');
+ this.updateSpec(resp.data);
+ this.updateStep('spec');
} catch (e) {
AppToaster.show({
message: `Failed to get supervisor spec: ${getDruidErrorMessage(e)}`,
@@ -2413,8 +2574,8 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
try {
const resp = await axios.get(`/druid/indexer/v1/task/${initTaskId}`);
- this.updateSpec(normalizeSpecType(resp.data.payload.spec));
- this.updateStep('json-spec');
+ this.updateSpec(resp.data.payload);
+ this.updateStep('spec');
} catch (e) {
AppToaster.show({
message: `Failed to get task spec: ${getDruidErrorMessage(e)}`,
@@ -2427,7 +2588,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
return <Loader loading />;
}
- renderJsonSpecStep() {
+ renderSpecStep() {
const { goToTask } = this.props;
const { spec } = this.state;
@@ -2438,7 +2599,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
value={spec}
onChange={s => {
if (!s) return;
- this.updateSpec(normalizeSpecType(s));
+ this.updateSpec(s);
}}
height="100%"
/>
@@ -2455,6 +2616,14 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
</Callout>
</div>
<div className="next-bar">
+ {!isEmptyIngestionSpec(spec) && (
+ <Button
+ className="left"
+ icon={IconNames.RESET}
+ text="Reset spec"
+ onClick={this.handleResetConfirm}
+ />
+ )}
<Button
text="Submit"
intent={Intent.PRIMARY}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
