This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4db8899d feat(config-ui): support skipping failed/stalled stages
(#3686)
4db8899d is described below
commit 4db8899d057ce60904e060d882d68d40a1044472
Author: 青湛 <[email protected]>
AuthorDate: Tue Nov 8 18:34:57 2022 +0800
feat(config-ui): support skipping failed/stalled stages (#3686)
---
.../blueprints/create-workflow/DataSync.jsx | 62 ++++-------
config-ui/src/components/pipelines/StageLane.jsx | 8 +-
config-ui/src/components/pipelines/StageTask.jsx | 29 +++--
config-ui/src/data/NullBlueprint.js | 3 +-
config-ui/src/data/TestBlueprintDetail.js | 1 +
config-ui/src/hooks/useBlueprintManager.jsx | 8 +-
config-ui/src/hooks/usePipelineManager.jsx | 32 +++++-
config-ui/src/images/icons/sync.svg | 11 ++
.../src/pages/blueprints/blueprint-detail.jsx | 123 ++++++++++++---------
.../src/pages/blueprints/blueprint-settings.jsx | 7 +-
.../src/pages/blueprints/create-blueprint.jsx | 8 +-
11 files changed, 177 insertions(+), 115 deletions(-)
diff --git a/config-ui/src/components/blueprints/create-workflow/DataSync.jsx
b/config-ui/src/components/blueprints/create-workflow/DataSync.jsx
index cce1a429..5dd0bdb8 100644
--- a/config-ui/src/components/blueprints/create-workflow/DataSync.jsx
+++ b/config-ui/src/components/blueprints/create-workflow/DataSync.jsx
@@ -15,10 +15,9 @@
* limitations under the License.
*
*/
-import React, { Fragment, useEffect, useState, useCallback } from 'react'
-import dayjs from '@/utils/time'
+import React from 'react'
import {
- Button,
+ Checkbox,
Icon,
Intent,
Tooltip,
@@ -33,13 +32,6 @@ import {
Card,
Colors
} from '@blueprintjs/core'
-// import {
-// Providers,
-// ProviderTypes,
-// ProviderIcons,
-// ConnectionStatus,
-// ConnectionStatusLabels
-// } from '@/data/Providers'
import InputValidationError from '@/components/validation/InputValidationError'
@@ -47,6 +39,8 @@ import CronHelp from '@/images/cron-help.png'
const DataSync = (props) => {
const {
+ skipOnFail,
+ setSkipOnFail,
activeStep,
cronConfig,
customCronConfig,
@@ -274,38 +268,22 @@ const DataSync = (props) => {
</div>
</div>
- {/* {cronConfig !== 'manual' && (
- <div>
- <Divider
- className='section-divider'
- style={{ marginTop: ' 20px' }}
- />
- <div>
- <h4 style={{ marginRight: 0, marginBottom: 0 }}>Next Run
Date</h4>
- </div>
- <div style={{ fontSize: '14px', fontWeight: 800 }}>
- {dayjs(
- createCron(
- cronConfig === 'custom' ? customCronConfig : cronConfig
- )
- .next()
- .toString()
- ).format('L LTS')}{' '}
- ·{' '}
- <span style={{ color: Colors.GRAY3 }}>
- (
- {dayjs(
- createCron(
- cronConfig === 'custom' ? customCronConfig : cronConfig
- )
- .next()
- .toString()
- ).fromNow()}
- )
- </span>
- </div>
- </div>
- )} */}
+ <h4>Running Policy</h4>
+ <div style={{ marginTop: 20 }}>
+ <Checkbox
+ label='Skip failed tasks (Recommended when collecting large volume
of data, eg. 10+ GitHub repos/Jira boards)'
+ checked={skipOnFail}
+ onChange={(e) => setSkipOnFail(e.target.checked)}
+ />
+ <p>
+ A task is a unit of a pipeline. A pipeline is an execution of a
+ blueprint. By default, when a task is failed, the whole pipeline
+ will fail and all the data that has been collected will be
+ discarded. By skipping failed tasks, the pipeline will continue to
+ run, and the data collected by other tasks will not be affected.
+ After the pipeline is finished, you can rerun these failed tasks.
+ </p>
+ </div>
</Card>
</div>
)
diff --git a/config-ui/src/components/pipelines/StageLane.jsx
b/config-ui/src/components/pipelines/StageLane.jsx
index c9ce26ad..dacfeebd 100644
--- a/config-ui/src/components/pipelines/StageLane.jsx
+++ b/config-ui/src/components/pipelines/StageLane.jsx
@@ -23,7 +23,7 @@ import StageTask from '@/components/pipelines/StageTask'
import StageLaneStatus from '@/components/pipelines/StageLaneStatus'
const StageLane = (props) => {
- const { stages = [], sK = 1, sIdx, showStageTasks = true } = props
+ const { stages = [], sK = 1, sIdx, showStageTasks = true, rerunTask } = props
const [activeStage, setActiveStage] = useState(stages[sK])
const [readyStageModules, setReadyStageModules] = useState([])
@@ -213,7 +213,11 @@ const StageLane = (props) => {
classNames='pipeline-task-fx'
// unmountOnExit
>
- <StageTask task={t} key={`stage-task-key-${tIdx}`} />
+ <StageTask
+ task={t}
+ key={`stage-task-key-${tIdx}`}
+ rerunTask={rerunTask}
+ />
</CSSTransition>
))}
{/* <StageLaneStatus
diff --git a/config-ui/src/components/pipelines/StageTask.jsx
b/config-ui/src/components/pipelines/StageTask.jsx
index 21d3a010..d62a38c4 100644
--- a/config-ui/src/components/pipelines/StageTask.jsx
+++ b/config-ui/src/components/pipelines/StageTask.jsx
@@ -21,16 +21,18 @@ import StageTaskName from
'@/components/pipelines/StageTaskName'
import StageTaskIndicator from '@/components/pipelines/StageTaskIndicator'
import StageTaskCaption from '@/components/pipelines/StageTaskCaption'
+import { ReactComponent as SyncIcon } from '@/images/icons/sync.svg'
+
const StageTask = (props) => {
- const {
- // stages = [],
- task
- // sK,
- // sIdx,
- } = props
+ const { task, rerunTask } = props
const [taskModuleOpened, setTaskModuleOpened] = useState(null)
+ const handleRerunTask = (e) => {
+ e.stopPropagation()
+ rerunTask(task.id)
+ }
+
const generateStageTaskCssClasses = () => {
return `pipeline-task-module task-${task.status
.split('_')[1]
@@ -70,11 +72,16 @@ const StageTask = (props) => {
}}
>
<div style={{ padding: '4px 2px 4px 0', width: '100%' }}>
- <StageTaskName
- task={task}
- showDetails={taskModuleOpened}
- onClose={() => setTaskModuleOpened(null)}
- />
+ <div style={{ display: 'flex', justifyContent: 'space-between' }}>
+ <StageTaskName
+ task={task}
+ showDetails={taskModuleOpened}
+ onClose={() => setTaskModuleOpened(null)}
+ />
+ <span onClick={handleRerunTask}>
+ <SyncIcon />
+ </span>
+ </div>
<StageTaskCaption task={task} options={task.options} />
</div>
</div>
diff --git a/config-ui/src/data/NullBlueprint.js
b/config-ui/src/data/NullBlueprint.js
index bc161d8e..eb0807ef 100644
--- a/config-ui/src/data/NullBlueprint.js
+++ b/config-ui/src/data/NullBlueprint.js
@@ -44,7 +44,8 @@ const NullBlueprint = {
interval: 'daily',
enable: BlueprintStatus.DISABLED,
mode: BlueprintMode.NORMAL,
- isManual: false
+ isManual: false,
+ skipOnFail: false
}
export { NullBlueprint, BlueprintMode, BlueprintStatus }
diff --git a/config-ui/src/data/TestBlueprintDetail.js
b/config-ui/src/data/TestBlueprintDetail.js
index 503a62e9..0b461f69 100644
--- a/config-ui/src/data/TestBlueprintDetail.js
+++ b/config-ui/src/data/TestBlueprintDetail.js
@@ -143,6 +143,7 @@ const TEST_BLUEPRINT_API_RESPONSE = {
enable: true,
cronConfig: '0 0 * * *',
isManual: false,
+ skipOnFail: false,
settings: {
version: '1.0.0',
connections: [
diff --git a/config-ui/src/hooks/useBlueprintManager.jsx
b/config-ui/src/hooks/useBlueprintManager.jsx
index bb48d640..0916af29 100644
--- a/config-ui/src/hooks/useBlueprintManager.jsx
+++ b/config-ui/src/hooks/useBlueprintManager.jsx
@@ -49,6 +49,7 @@ function useBlueprintManager(
const [enable, setEnable] = useState(true)
const [detectedProviderTasks, setDetectedProviderTasks] = useState([])
const [isManual, setIsManual] = useState(false)
+ const [skipOnFail, setSkipOnFail] = useState(false)
const [rawConfiguration, setRawConfiguration] = useState(
JSON.stringify([tasks], null, ' ')
)
@@ -190,6 +191,7 @@ function useBlueprintManager(
}
: NullBlueprint
)
+ setSkipOnFail(blueprintData.skipOnFail)
setErrors(b.status !== 200 ? [b.data] : [])
setTimeout(() => {
setIsFetching(false)
@@ -237,7 +239,8 @@ function useBlueprintManager(
settings,
enable: enable,
mode,
- isManual
+ isManual,
+ skipOnFail
}
console.log('>> DISPATCHING BLUEPRINT SAVE REQUEST', blueprintPayload)
const run = async () => {
@@ -299,6 +302,7 @@ function useBlueprintManager(
tasks,
enable,
isManual,
+ skipOnFail,
detectCronInterval
]
)
@@ -558,6 +562,8 @@ function useBlueprintManager(
isSaving,
isDeleting,
isManual,
+ skipOnFail,
+ setSkipOnFail,
errors
}
}
diff --git a/config-ui/src/hooks/usePipelineManager.jsx
b/config-ui/src/hooks/usePipelineManager.jsx
index 5a97c7ec..98820f17 100644
--- a/config-ui/src/hooks/usePipelineManager.jsx
+++ b/config-ui/src/hooks/usePipelineManager.jsx
@@ -246,6 +246,34 @@ function usePipelineManager(
[allowedProviders]
)
+ const rerunAllFailedTasks = async () => {
+ try {
+ const res = await request.post(
+ `${DEVLAKE_ENDPOINT}/pipelines/${activePipeline.id}/tasks`,
+ {
+ taskId: 0
+ }
+ )
+ if (res?.data?.success) {
+ fetchPipeline(activePipeline.id)
+ }
+ } catch (err) {}
+ }
+
+ const rerunTask = async (taskId) => {
+ try {
+ const res = await request.post(
+ `${DEVLAKE_ENDPOINT}/pipelines/${activePipeline.id}/tasks`,
+ {
+ taskId
+ }
+ )
+ if (res?.data?.success) {
+ fetchPipeline(activePipeline.id)
+ }
+ } catch (err) {}
+ }
+
useEffect(() => {
console.log('>> PIPELINE MANAGER - RECEIVED RUN/TASK SETTINGS', settings)
}, [settings])
@@ -284,7 +312,9 @@ function usePipelineManager(
detectPipelineProviders,
allowedProviders,
setAllowedProviders,
- getPipelineLogfile
+ getPipelineLogfile,
+ rerunAllFailedTasks,
+ rerunTask
}
}
diff --git a/config-ui/src/images/icons/sync.svg
b/config-ui/src/images/icons/sync.svg
new file mode 100644
index 00000000..f8404539
--- /dev/null
+++ b/config-ui/src/images/icons/sync.svg
@@ -0,0 +1,11 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none"
xmlns="http://www.w3.org/2000/svg";>
+ <g clip-path="url(#clip0_2610_21738)">
+ <path fill-rule="evenodd" clip-rule="evenodd" d="M12 4.33334C9.80042
4.33334 7.81848 5.25834 6.41912 6.7434C6.04037 7.14535 5.40748 7.16416 5.00553
6.7854C4.60358 6.40665 4.58478 5.77376 4.96353 5.37181C6.72503 3.50243 9.22679
2.33334 12 2.33334C16.9919 2.33334 21.1 6.1171 21.6128 10.973L22.2929
10.2929C22.6834 9.90238 23.3166 9.90238 23.7071 10.2929C24.0976 10.6834 24.0976
11.3166 23.7071 11.7071L21.3738 14.0404C20.9832 14.431 20.3501 14.431 19.9596
14.0404L17.6262 11.7071C17.2357 1 [...]
+ </g>
+ <defs>
+ <clipPath id="clip0_2610_21738">
+ <rect width="24" height="24" fill="white" />
+ </clipPath>
+ </defs>
+</svg>
+
\ No newline at end of file
diff --git a/config-ui/src/pages/blueprints/blueprint-detail.jsx
b/config-ui/src/pages/blueprints/blueprint-detail.jsx
index 701c07e4..fde5e8bb 100644
--- a/config-ui/src/pages/blueprints/blueprint-detail.jsx
+++ b/config-ui/src/pages/blueprints/blueprint-detail.jsx
@@ -151,7 +151,9 @@ const BlueprintDetail = (props) => {
// eslint-disable-next-line no-unused-vars
detectPipelineProviders,
logfile: pipelineLogFilename,
- getPipelineLogfile
+ getPipelineLogfile,
+ rerunAllFailedTasks,
+ rerunTask
} = usePipelineManager()
const {
@@ -612,67 +614,77 @@ const BlueprintDetail = (props) => {
alignItems: 'center'
}}
>
- <div style={{ display: 'block' }}>
- {/* <Button intent={Intent.PRIMARY} outlined
text='Cancel' onClick={cancelRun} /> */}
- <Popover
- key='popover-help-key-cancel-run'
- className='trigger-pipeline-cancel'
- popoverClassName='popover-pipeline-cancel'
- position={Position.BOTTOM}
- autoFocus={false}
- enforceFocus={false}
- usePortal={true}
- disabled={currentRun?.status !== 'TASK_RUNNING'}
- >
- <Button
- // icon='stop'
- text='Cancel'
- intent={Intent.PRIMARY}
- outlined
- disabled={currentRun?.status !== 'TASK_RUNNING'}
- />
- <>
- <div
- style={{
- fontSize: '12px',
- padding: '12px',
- maxWidth: '200px'
- }}
- >
- <p>
- Are you Sure you want to cancel this{' '}
- <strong>Pipeline Run</strong>?
- </p>
+ {currentRun?.status === 'TASK_RUNNING' && (
+ <div style={{ display: 'block' }}>
+ {/* <Button intent={Intent.PRIMARY} outlined
text='Cancel' onClick={cancelRun} /> */}
+ <Popover
+ key='popover-help-key-cancel-run'
+ className='trigger-pipeline-cancel'
+ popoverClassName='popover-pipeline-cancel'
+ position={Position.BOTTOM}
+ autoFocus={false}
+ enforceFocus={false}
+ usePortal={true}
+ >
+ <Button
+ // icon='stop'
+ text='Cancel'
+ intent={Intent.PRIMARY}
+ outlined
+ />
+ <>
<div
style={{
- display: 'flex',
- width: '100%',
- justifyContent: 'flex-end'
+ fontSize: '12px',
+ padding: '12px',
+ maxWidth: '200px'
}}
>
- <Button
- text='NO'
- minimal
- small
- className={Classes.POPOVER_DISMISS}
+ <p>
+ Are you Sure you want to cancel this{' '}
+ <strong>Pipeline Run</strong>?
+ </p>
+ <div
style={{
- marginLeft: 'auto',
- marginRight: '3px'
+ display: 'flex',
+ width: '100%',
+ justifyContent: 'flex-end'
}}
- />
- <Button
- className={Classes.POPOVER_DISMISS}
- text='YES'
- icon='small-tick'
- intent={Intent.DANGER}
- small
- onClick={() =>
cancelPipeline(currentRun?.id)}
- />
+ >
+ <Button
+ text='NO'
+ minimal
+ small
+ className={Classes.POPOVER_DISMISS}
+ style={{
+ marginLeft: 'auto',
+ marginRight: '3px'
+ }}
+ />
+ <Button
+ className={Classes.POPOVER_DISMISS}
+ text='YES'
+ icon='small-tick'
+ intent={Intent.DANGER}
+ small
+ onClick={() =>
+ cancelPipeline(currentRun?.id)
+ }
+ />
+ </div>
</div>
- </div>
- </>
- </Popover>
- </div>
+ </>
+ </Popover>
+ </div>
+ )}
+ {currentRun?.status === TaskStatus.COMPLETE && (
+ <Button
+ intent={Intent.PRIMARY}
+ onClick={rerunAllFailedTasks}
+ >
+ Run Failed Tasks
+ </Button>
+ )}
</div>
</div>
)}
@@ -719,6 +731,7 @@ const BlueprintDetail = (props) => {
sK={sK}
sIdx={sIdx}
showStageTasks={showCurrentRunTasks}
+ rerunTask={rerunTask}
/>
))}
</div>
diff --git a/config-ui/src/pages/blueprints/blueprint-settings.jsx
b/config-ui/src/pages/blueprints/blueprint-settings.jsx
index b04e6223..3c8e00c2 100644
--- a/config-ui/src/pages/blueprints/blueprint-settings.jsx
+++ b/config-ui/src/pages/blueprints/blueprint-settings.jsx
@@ -162,7 +162,9 @@ const BlueprintSettings = (props) => {
fetchAllBlueprints,
saveBlueprint,
saveComplete,
- errors: blueprintErrors
+ errors: blueprintErrors,
+ skipOnFail,
+ setSkipOnFail
} = useBlueprintManager()
const {
@@ -696,6 +698,7 @@ const BlueprintSettings = (props) => {
...aS,
payload: {
isManual: !!isManualCron,
+ skipOnFail,
cronConfig: isManualCron
? getCronPreset('daily').cronConfig
: isCustomCron
@@ -1193,6 +1196,8 @@ const BlueprintSettings = (props) => {
case 'cronConfig':
Settings = (
<DataSync
+ skipOnFail={skipOnFail}
+ setSkipOnFail={setSkipOnFail}
cronConfig={cronConfig}
customCronConfig={customCronConfig}
createCron={createCron}
diff --git a/config-ui/src/pages/blueprints/create-blueprint.jsx
b/config-ui/src/pages/blueprints/create-blueprint.jsx
index bb19128c..0a20f34b 100644
--- a/config-ui/src/pages/blueprints/create-blueprint.jsx
+++ b/config-ui/src/pages/blueprints/create-blueprint.jsx
@@ -194,7 +194,9 @@ const CreateBlueprint = (props) => {
deleteBlueprint,
isDeleting: isDeletingBlueprint,
isManual: isManualBlueprint,
- saveComplete: saveBlueprintComplete
+ saveComplete: saveBlueprintComplete,
+ skipOnFail,
+ setSkipOnFail
} = useBlueprintManager()
const {
@@ -928,6 +930,8 @@ const CreateBlueprint = (props) => {
{activeStep?.id === 2 && (
<DataSync
+ skipOnFail={skipOnFail}
+ setSkipOnFail={setSkipOnFail}
activeStep={activeStep}
advancedMode={advancedMode}
cronConfig={cronConfig}
@@ -1041,6 +1045,8 @@ const CreateBlueprint = (props) => {
{activeStep?.id === 4 && (
<DataSync
+ skipOnFail={skipOnFail}
+ setSkipOnFail={setSkipOnFail}
activeStep={activeStep}
advancedMode={advancedMode}
cronConfig={cronConfig}
