This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 3b62d7929c Web console: Data loader should allow for multiline JSON
messages in kafka (#13709)
3b62d7929c is described below
commit 3b62d7929c6a291acfd1b76383ddeda1e044c414
Author: Vadim Ogievetsky <[email protected]>
AuthorDate: Wed Jan 25 21:23:18 2023 -0800
Web console: Data loader should allow for multiline JSON messages in kafka
(#13709)
* stricter
* data loader should allow for mulit-line json
* add await
* kinesis also
---
web-console/lib/keywords.js | 4 ++++
.../druid-models/ingestion-spec/ingestion-spec.tsx | 20 +++++++++++++++++---
.../src/druid-models/input-format/input-format.tsx | 1 +
web-console/src/singletons/api.ts | 6 ++++--
web-console/src/utils/general.tsx | 6 +++++-
.../src/views/home-view/tasks-card/tasks-card.tsx | 2 +-
.../src/views/load-data-view/load-data-view.tsx | 2 +-
7 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/web-console/lib/keywords.js b/web-console/lib/keywords.js
index bc81153dd7..b22af5f422 100644
--- a/web-console/lib/keywords.js
+++ b/web-console/lib/keywords.js
@@ -64,6 +64,10 @@ exports.SQL_KEYWORDS = [
'OVER',
'PARTITION BY',
'WINDOW',
+ 'RANGE',
+ 'PRECEDING',
+ 'FOLLOWING',
+ 'EXTEND',
];
exports.SQL_EXPRESSION_PARTS = [
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index f144f4c2ad..b2bd24219b 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -2133,7 +2133,12 @@ export function updateIngestionType(
return newSpec;
}
-export function issueWithSampleData(sampleData: string[]): JSX.Element |
undefined {
+export function issueWithSampleData(
+ sampleData: string[],
+ spec: Partial<IngestionSpec>,
+): JSX.Element | undefined {
+ if (isStreamingSpec(spec)) return;
+
if (sampleData.length) {
const firstData = sampleData[0];
@@ -2166,14 +2171,18 @@ export function fillInputFormatIfNeeded(
sampleData: string[],
): Partial<IngestionSpec> {
if (deepGet(spec, 'spec.ioConfig.inputFormat.type')) return spec;
- return deepSet(spec, 'spec.ioConfig.inputFormat',
guessInputFormat(sampleData));
+ return deepSet(
+ spec,
+ 'spec.ioConfig.inputFormat',
+ guessInputFormat(sampleData, isStreamingSpec(spec)),
+ );
}
function noNumbers(xs: string[]): boolean {
return xs.every(x => isNaN(Number(x)));
}
-export function guessInputFormat(sampleData: string[]): InputFormat {
+export function guessInputFormat(sampleData: string[], canBeMultiLineJson =
false): InputFormat {
let sampleDatum = sampleData[0];
if (sampleDatum) {
sampleDatum = String(sampleDatum); // Really ensure it is a string
@@ -2261,6 +2270,11 @@ export function guessInputFormat(sampleData: string[]):
InputFormat {
numColumns: lineAsTsvPipe.length,
});
}
+
+ // If the object is a single json object spanning multiple lines than the
first one will just start with `{`
+ if (canBeMultiLineJson && sampleDatum.startsWith('{')) {
+ return { type: 'json', useJsonNodeReader: true };
+ }
}
return inputFormatFromType({ type: 'regex' });
diff --git a/web-console/src/druid-models/input-format/input-format.tsx
b/web-console/src/druid-models/input-format/input-format.tsx
index 1ec6e935e4..417b3c3fd8 100644
--- a/web-console/src/druid-models/input-format/input-format.tsx
+++ b/web-console/src/druid-models/input-format/input-format.tsx
@@ -114,6 +114,7 @@ function generateInputFormatFields(streaming: boolean) {
streaming
? {
name: 'useJsonNodeReader',
+ title: 'Use JSON node reader',
type: 'boolean',
defined: typeIs('json'),
disabled: (inputFormat: InputFormat) =>
inputFormat.assumeNewlineDelimited,
diff --git a/web-console/src/singletons/api.ts
b/web-console/src/singletons/api.ts
index ef312ece7c..b7340ec073 100644
--- a/web-console/src/singletons/api.ts
+++ b/web-console/src/singletons/api.ts
@@ -19,6 +19,8 @@
import axios, { AxiosError, AxiosInstance, AxiosRequestConfig } from 'axios';
import * as JSONBig from 'json-bigint-native';
+import { nonEmptyString } from '../utils';
+
export class Api {
static instance: AxiosInstance;
@@ -32,11 +34,11 @@ export class Api {
(error: AxiosError) => {
const responseData = error.response?.data;
const message = responseData?.message;
- if (typeof message === 'string') {
+ if (nonEmptyString(message)) {
return Promise.reject(new Error(message));
}
- if (error.config.method?.toLowerCase() === 'get' && typeof
responseData === 'string') {
+ if (error.config.method?.toLowerCase() === 'get' &&
nonEmptyString(responseData)) {
return Promise.reject(new Error(responseData));
}
diff --git a/web-console/src/utils/general.tsx
b/web-console/src/utils/general.tsx
index 5cf8ca0ce5..6e4ddcf6a1 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -36,7 +36,11 @@ export function isNumberLikeNaN(x: NumberLike): boolean {
return isNaN(Number(x));
}
-export function nonEmptyArray(a: any): a is unknown[] {
+export function nonEmptyString(s: unknown): s is string {
+ return typeof s === 'string' && s !== '';
+}
+
+export function nonEmptyArray(a: unknown): a is unknown[] {
return Array.isArray(a) && Boolean(a.length);
}
diff --git a/web-console/src/views/home-view/tasks-card/tasks-card.tsx
b/web-console/src/views/home-view/tasks-card/tasks-card.tsx
index 27b29320b7..b5cb5bfc0c 100644
--- a/web-console/src/views/home-view/tasks-card/tasks-card.tsx
+++ b/web-console/src/views/home-view/tasks-card/tasks-card.tsx
@@ -76,7 +76,7 @@ export interface TasksCardProps {
export const TasksCard = React.memo(function TasksCard(props: TasksCardProps) {
const [cardState] = useQueryManager<Capabilities, TaskCountsAndCapacity>({
processQuery: async capabilities => {
- const taskCounts = getTaskCounts(capabilities);
+ const taskCounts = await getTaskCounts(capabilities);
if (!capabilities.hasOverlordAccess()) return taskCounts;
const capacity = await getClusterCapacity();
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index d5f92ed246..eb2a772339 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -1364,7 +1364,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
l.input ? l.input.raw : undefined,
);
- const issue = issueWithSampleData(sampleLines);
+ const issue = issueWithSampleData(sampleLines, spec);
if (issue) {
AppToaster.show({
icon: IconNames.WARNING_SIGN,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
