This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch 29.0.1
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/29.0.1 by this push:
new 0feb0d2b013 make flattenSpec location adaptive (#15946) (#16054)
0feb0d2b013 is described below
commit 0feb0d2b0135d5d92a8c31ec4fbff45dff01e795
Author: Karan Kumar <[email protected]>
AuthorDate: Sat Mar 9 03:04:59 2024 +0530
make flattenSpec location adaptive (#15946) (#16054)
Co-authored-by: Vadim Ogievetsky <[email protected]>
---
.../druid-models/ingestion-spec/ingestion-spec.tsx | 22 +++++++++++++
web-console/src/utils/sampler.ts | 3 +-
.../src/views/load-data-view/load-data-view.tsx | 36 ++++++++++------------
3 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 54a1cb241c6..e3bd4fb9e45 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -50,6 +50,7 @@ import {
getDimensionSpecName,
getDimensionSpecs,
} from '../dimension-spec/dimension-spec';
+import type { FlattenSpec } from '../flatten-spec/flatten-spec';
import type { IndexSpec } from '../index-spec/index-spec';
import { summarizeIndexSpec } from '../index-spec/index-spec';
import type { InputFormat } from '../input-format/input-format';
@@ -381,6 +382,27 @@ export function getPossibleSystemFieldsForSpec(spec:
Partial<IngestionSpec>): st
return getPossibleSystemFieldsForInputSource(inputSource);
}
+export function getFlattenSpec(spec: Partial<IngestionSpec>): FlattenSpec |
undefined {
+ const inputFormat: InputFormat | undefined = deepGet(spec,
'spec.ioConfig.inputFormat');
+ if (!inputFormat) return;
+ return (
+ (inputFormat.type === 'kafka'
+ ? inputFormat.valueFormat?.flattenSpec
+ : inputFormat.flattenSpec) || undefined
+ );
+}
+
+export function changeFlattenSpec(
+ spec: Partial<IngestionSpec>,
+ flattenSpec: FlattenSpec | undefined,
+): Partial<IngestionSpec> {
+ if (deepGet(spec, 'spec.ioConfig.inputFormat.type') === 'kafka') {
+ return deepSet(spec, 'spec.ioConfig.inputFormat.valueFormat.flattenSpec',
flattenSpec);
+ } else {
+ return deepSet(spec, 'spec.ioConfig.inputFormat.flattenSpec', flattenSpec);
+ }
+}
+
export function getPossibleSystemFieldsForInputSource(inputSource:
InputSource): string[] {
switch (inputSource.type) {
case 's3':
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 866b195d457..6c31f2bb2b6 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -36,6 +36,7 @@ import {
DETECTION_TIMESTAMP_SPEC,
getDimensionNamesFromTransforms,
getDimensionSpecName,
+ getFlattenSpec,
getSpecType,
getTimestampSchema,
isDruidSource,
@@ -157,7 +158,7 @@ export function applyCache(sampleSpec: SampleSpec,
cacheRows: CacheRows) {
data: cacheRows.map(r => JSONBig.stringify(r)).join('\n'),
});
- const flattenSpec = deepGet(sampleSpec,
'spec.ioConfig.inputFormat.flattenSpec');
+ const flattenSpec = getFlattenSpec(sampleSpec);
let inputFormat: InputFormat = { type: 'json', keepNullColumns: true };
if (flattenSpec) {
inputFormat = deepSet(inputFormat, 'flattenSpec', flattenSpec);
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 4f748ad945a..88beb76115e 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -76,6 +76,7 @@ import {
adjustForceGuaranteedRollup,
adjustId,
BATCH_INPUT_FORMAT_FIELDS,
+ changeFlattenSpec,
chooseByBestTimestamp,
cleanSpec,
computeFlattenPathsForData,
@@ -89,6 +90,7 @@ import {
FLATTEN_FIELD_FIELDS,
getArrayMode,
getDimensionSpecName,
+ getFlattenSpec,
getIngestionComboType,
getIngestionImage,
getIngestionTitle,
@@ -1459,8 +1461,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
const { columnFilter, specialColumnsOnly, parserQueryState,
selectedFlattenField } = this.state;
const spec = this.getEffectiveSpec();
const inputFormat: InputFormat = deepGet(spec,
'spec.ioConfig.inputFormat') || EMPTY_OBJECT;
- const flattenFields: FlattenField[] =
- deepGet(spec, 'spec.ioConfig.inputFormat.flattenSpec.fields') ||
EMPTY_ARRAY;
+ const flattenFields: FlattenField[] = getFlattenSpec(spec)?.fields ||
EMPTY_ARRAY;
const canHaveNestedData = inputFormatCanProduceNestedData(inputFormat);
@@ -1617,13 +1618,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
icon={IconNames.LIGHTBULB}
text={`Auto add
${pluralIfNeeded(suggestedFlattenFields.length, 'flatten spec')}`}
onClick={() => {
- this.updateSpec(
- deepSet(
- spec,
- 'spec.ioConfig.inputFormat.flattenSpec.fields',
- suggestedFlattenFields,
- ),
- );
+ this.updateSpec(changeFlattenSpec(spec, { fields:
suggestedFlattenFields }));
}}
/>
</FormGroup>
@@ -1676,24 +1671,25 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
initValue={selectedFlattenField.value}
onClose={this.resetSelected}
onDirty={this.handleDirty}
- onApply={flattenField =>
+ onApply={flattenField => {
+ const flattenSpec = getFlattenSpec(spec) || {};
this.updateSpec(
- deepSet(
+ changeFlattenSpec(
spec,
-
`spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenField.index}`,
- flattenField,
+ deepSet(flattenSpec, `fields.${selectedFlattenField.index}`,
flattenField),
),
- )
- }
+ );
+ }}
showDelete={selectedFlattenField.index !== -1}
- onDelete={() =>
+ onDelete={() => {
+ const flattenSpec = getFlattenSpec(spec) || {};
this.updateSpec(
- deepDelete(
+ changeFlattenSpec(
spec,
-
`spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenField.index}`,
+ deepDelete(flattenSpec,
`fields.${selectedFlattenField.index}`),
),
- )
- }
+ );
+ }}
/>
);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
