This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new edb1ac1b712 Update azure console tile (#15820)
edb1ac1b712 is described below
commit edb1ac1b712dd2c9ad47b5a7cb143b3eee22f5e4
Author: George Shiqi Wu <[email protected]>
AuthorDate: Mon Feb 5 16:11:39 2024 -0500
Update azure console tile (#15820)
* Save web console changes
* Working new input type
* fix tests
---
web-console/assets/{azure.png => azureStorage.png} | Bin
.../external-config/external-config.ts | 2 +-
.../druid-models/ingestion-spec/ingestion-spec.tsx | 38 ++++++++++++++------
.../src/druid-models/input-source/input-source.tsx | 39 +++++++++++++++------
.../__snapshots__/load-data-view.spec.tsx.snap | 4 +--
.../src/views/load-data-view/load-data-view.tsx | 6 ++--
.../input-source-step/input-source-info.tsx | 2 +-
.../input-source-step/input-source-step.tsx | 2 +-
8 files changed, 63 insertions(+), 30 deletions(-)
diff --git a/web-console/assets/azure.png b/web-console/assets/azureStorage.png
similarity index 100%
rename from web-console/assets/azure.png
rename to web-console/assets/azureStorage.png
diff --git a/web-console/src/druid-models/external-config/external-config.ts
b/web-console/src/druid-models/external-config/external-config.ts
index 2cd841a1d68..d088a856f12 100644
--- a/web-console/src/druid-models/external-config/external-config.ts
+++ b/web-console/src/druid-models/external-config/external-config.ts
@@ -76,7 +76,7 @@ export function summarizeInputSource(inputSource:
InputSource, multiline: boolea
case 's3':
case 'google':
- case 'azure': {
+ case 'azureStorage': {
const possibleLines = inputSource.uris || inputSource.prefixes;
if (nonEmptyArray(possibleLines)) {
let lines: string[] = possibleLines;
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 4f4b1e9a51b..85cfe31f894 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -98,7 +98,7 @@ export type IngestionComboType =
| 'index_parallel:druid'
| 'index_parallel:inline'
| 'index_parallel:s3'
- | 'index_parallel:azure'
+ | 'index_parallel:azureStorage'
| 'index_parallel:delta'
| 'index_parallel:google'
| 'index_parallel:hdfs';
@@ -142,7 +142,7 @@ export function getIngestionComboType(
case 'druid':
case 'inline':
case 's3':
- case 'azure':
+ case 'azureStorage':
case 'google':
case 'hdfs':
return `${ioConfig.type}:${inputSource.type}` as IngestionComboType;
@@ -170,7 +170,7 @@ export function getIngestionTitle(ingestionType:
IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'Amazon S3';
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
return 'Azure Data Lake';
case 'index_parallel:delta':
@@ -231,7 +231,7 @@ export function getRequiredModule(ingestionType:
IngestionComboTypeWithExtra): s
case 'index_parallel:s3':
return 'druid-s3-extensions';
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
return 'druid-azure-extensions';
case 'index_parallel:delta':
@@ -530,7 +530,7 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
name: 'inputSource.type',
label: 'Source type',
type: 'string',
- suggestions: ['local', 'http', 'inline', 'delta', 's3', 'azure', 'google',
'hdfs'],
+ suggestions: ['local', 'http', 'inline', 'delta', 's3', 'azureStorage',
'google', 'hdfs'],
info: (
<p>
Druid connects to raw data through{' '}
@@ -844,7 +844,7 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
},
];
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
return [
inputSourceType,
{
@@ -852,7 +852,7 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
label: 'Azure URIs',
type: 'string-array',
placeholder:
- 'azure://your-container/some-file1.ext,
azure://your-container/some-file2.ext',
+
'azureStorage://your-storage-account/your-container/some-file1.ext,
azureStorage://your-storage-account/your-container/some-file2.ext',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig,
'inputSource.objects'),
@@ -870,7 +870,8 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
name: 'inputSource.prefixes',
label: 'Azure prefixes',
type: 'string-array',
- placeholder: 'azure://your-container/some-path1,
azure://your-container/some-path2',
+ placeholder:
+ 'azureStorage://your-storage-account/your-container/some-path1,
azureStorage://your-storage-account/your-container/some-path2',
required: true,
defined: ioConfig =>
!deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig,
'inputSource.objects'),
@@ -885,7 +886,7 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
name: 'inputSource.objects',
label: 'Azure objects',
type: 'json',
- placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
+ placeholder: '{"bucket":"your-storage-account",
"path":"your-container/some-file.ext"}',
required: true,
defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
info: (
@@ -902,6 +903,21 @@ export function getIoConfigFormFields(ingestionComboType:
IngestionComboType): F
),
},
inputSourceFilter,
+ {
+ name: 'inputSource.properties.sharedAccessStorageToken',
+ label: 'Shared Access Storage Token',
+ type: 'string',
+ placeholder: '(sas token)',
+ info: (
+ <>
+ <p>Shared Access Storage Token for this storage account.</p>
+ <p>
+ Note: Inlining the sas token into the ingestion spec can be
dangerous as it might
+ appear in server log files and can be seen by anyone accessing
this console.
+ </p>
+ </>
+ ),
+ },
];
case 'index_parallel:google':
@@ -1163,7 +1179,7 @@ export function getIoConfigTuningFormFields(
switch (ingestionComboType) {
case 'index_parallel:http':
case 'index_parallel:s3':
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
case 'index_parallel:google':
case 'index_parallel:delta':
case 'index_parallel:hdfs':
@@ -1440,7 +1456,7 @@ export function
guessDataSourceNameFromInputSource(inputSource: InputSource): st
}
case 's3':
- case 'azure':
+ case 'azureStorage':
case 'google': {
const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
const uriPath =
diff --git a/web-console/src/druid-models/input-source/input-source.tsx
b/web-console/src/druid-models/input-source/input-source.tsx
index 01b111b957f..6fb8fbd4b77 100644
--- a/web-console/src/druid-models/input-source/input-source.tsx
+++ b/web-console/src/druid-models/input-source/input-source.tsx
@@ -105,7 +105,7 @@ export type InputSourceDesc =
};
}
| {
- type: 'google' | 'azure';
+ type: 'google' | 'azureStorage';
uris?: string[];
prefixes?: string[];
objects?: { bucket: string; path: string }[];
@@ -154,7 +154,7 @@ export function issueWithInputSource(inputSource:
InputSource | undefined): stri
return;
case 's3':
- case 'azure':
+ case 'azureStorage':
case 'google':
if (
!nonEmptyArray(inputSource.uris) &&
@@ -188,7 +188,7 @@ const KNOWN_TYPES = [
'http',
'local',
's3',
- 'azure',
+ 'azureStorage',
'delta',
'google',
'hdfs',
@@ -349,9 +349,10 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'uris',
label: 'Azure URIs',
type: 'string-array',
- placeholder: 'azure://your-container/some-file1.ext,
azure://your-container/some-file2.ext',
+ placeholder:
+ 'azureStorage://your-storage-account/your-container/some-file1.ext,
azureStorage://your-storage-account/your-container/some-file2.ext',
defined: inputSource =>
- inputSource.type === 'azure' &&
+ inputSource.type === 'azureStorage' &&
!deepGet(inputSource, 'prefixes') &&
!deepGet(inputSource, 'objects'),
required: true,
@@ -369,9 +370,10 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'prefixes',
label: 'Azure prefixes',
type: 'string-array',
- placeholder: 'azure://your-container/some-path1,
azure://your-container/some-path2',
+ placeholder:
+ 'azureStorage://your-storage-account/your-container/some-path1,
azureStorage://your-storage-account/your-container/some-path2',
defined: inputSource =>
- inputSource.type === 'azure' &&
+ inputSource.type === 'azureStorage' &&
!deepGet(inputSource, 'uris') &&
!deepGet(inputSource, 'objects'),
required: true,
@@ -386,8 +388,8 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
name: 'objects',
label: 'Azure objects',
type: 'json',
- placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
- defined: inputSource => inputSource.type === 'azure' &&
deepGet(inputSource, 'objects'),
+ placeholder: '{"bucket":"your-storage-account",
"path":"your-container/some-file.ext"}',
+ defined: inputSource => inputSource.type === 'azureStorage' &&
deepGet(inputSource, 'objects'),
required: true,
info: (
<>
@@ -402,7 +404,22 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
</>
),
},
-
+ {
+ name: 'properties.sharedAccessStorageToken',
+ label: 'Shared Access Storage Token',
+ type: 'string',
+ placeholder: '(sas token)',
+ defined: inputSource => inputSource.type === 'azureStorage',
+ info: (
+ <>
+ <p>Shared Access Storage Token for this storage account.</p>
+ <p>
+ Note: Inlining the sas token into the ingestion spec can be
dangerous as it might appear
+ in server log files and can be seen by anyone accessing this console.
+ </p>
+ </>
+ ),
+ },
// google
{
name: 'uris',
@@ -469,7 +486,7 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
type: 'string',
suggestions: FILTER_SUGGESTIONS,
placeholder: '*',
- defined: typeIsKnown(KNOWN_TYPES, 's3', 'azure', 'google'),
+ defined: typeIsKnown(KNOWN_TYPES, 's3', 'azureStorage', 'google'),
info: (
<p>
A wildcard filter for files. See{' '}
diff --git
a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
index b0a81a1800a..3d1d2ec93ec 100644
---
a/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
+++
b/web-console/src/views/load-data-view/__snapshots__/load-data-view.spec.tsx.snap
@@ -181,8 +181,8 @@ exports[`LoadDataView matches snapshot batch 1`] = `
onClick={[Function]}
>
<img
- alt="Ingestion tile for index_parallel:azure"
- src="/some/base_url/assets/azure.png"
+ alt="Ingestion tile for index_parallel:azureStorage"
+ src="/some/base_url/assets/azurestorage.png"
/>
<p>
Azure Data Lake
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx
b/web-console/src/views/load-data-view/load-data-view.tsx
index 942e5d0aded..a0358f21d0f 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -868,7 +868,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
{mode !== 'streaming' && (
<>
{this.renderIngestionCard('index_parallel:s3')}
- {this.renderIngestionCard('index_parallel:azure')}
+ {this.renderIngestionCard('index_parallel:azureStorage')}
{this.renderIngestionCard('index_parallel:google')}
{this.renderIngestionCard('index_parallel:delta')}
{this.renderIngestionCard('index_parallel:hdfs')}
@@ -962,7 +962,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:s3':
return <p>Load text based, orc, or parquet data from Amazon S3.</p>;
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
return <p>Load text based, orc, or parquet data from Azure.</p>;
case 'index_parallel:google':
@@ -1027,7 +1027,7 @@ export class LoadDataView extends
React.PureComponent<LoadDataViewProps, LoadDat
case 'index_parallel:druid':
case 'index_parallel:inline':
case 'index_parallel:s3':
- case 'index_parallel:azure':
+ case 'index_parallel:azureStorage':
case 'index_parallel:google':
case 'index_parallel:delta':
case 'index_parallel:hdfs':
diff --git
a/web-console/src/views/workbench-view/input-source-step/input-source-info.tsx
b/web-console/src/views/workbench-view/input-source-step/input-source-info.tsx
index c5dac4c12f9..d1a7af6e8a6 100644
---
a/web-console/src/views/workbench-view/input-source-step/input-source-info.tsx
+++
b/web-console/src/views/workbench-view/input-source-step/input-source-info.tsx
@@ -63,7 +63,7 @@ export const InputSourceInfo = React.memo(function
InputSourceInfo(props: InputS
case 's3':
return <p>Load text based, avro, orc, or parquet data from Amazon
S3.</p>;
- case 'azure':
+ case 'azureStorage':
return <p>Load text based, avro, orc, or parquet data from Azure.</p>;
case 'google':
diff --git
a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
index b8ed5a74131..5cabe81d283 100644
---
a/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
+++
b/web-console/src/views/workbench-view/input-source-step/input-source-step.tsx
@@ -224,7 +224,7 @@ export const InputSourceStep = React.memo(function
InputSourceStep(props: InputS
<div className="input-source-step">
<div className="ingestion-cards">
{renderIngestionCard('s3')}
- {renderIngestionCard('azure')}
+ {renderIngestionCard('azureStorage')}
{renderIngestionCard('google')}
{renderIngestionCard('hdfs')}
{renderIngestionCard('http')}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
