This is an automated email from the ASF dual-hosted git repository.
linxinyuan pushed a change to branch xinyuan-cm-for-loop-mat-dcm
in repository https://gitbox.apache.org/repos/asf/texera.git
from f158f52e78 update
add 0aa4ad4ec5 feat: add centralized LakeFS error handling for multipart
upload and dataset version operations (#4177)
add 6b90b003cf fix(backend): use dynamic DSLContext lookup to fix stale DB
connections in tests (#4179)
add 0965b80ebe refactor(scala): move SortCriteriaUnit and HyperParameters
from Java to Scala (#4186)
add 830e3090b1 feat(backend): Resumable Uploads (#4181)
add cfdad432c3 feat(backend): introduce python code template builder for
creating Python based operators (#4189)
add fa87887399 fix(dependencies): fix the version of setuptools package
(#4199)
add 39f8508742 feat: add test cases for revoking workflow access. (#4187)
add 7048ce340a chore: update release workflow to comply with Apache
release practice (#4128)
add 99348fb660 refactor(helm): unify ingress and envoy proxy as a single
gateway (#4191)
add 05bb34810e fix: test failure from testcontainer caused by docker
update raising minimum version (#4208)
add 7a6252d5a4 fix(auth): reject null password during the authentication
(#4203)
add 641fb857b5 fix(amber): fix getting uncreated region status in query
statistics handler (#4213)
add 852ebee5a4 feat(gui): Display the status for each operator during
execution (#4195)
add 3e92803d84 feat: introduce sklearn testing operator (#4211)
add 286a39626a fix: Added setup python section to frontend test to pin
python to 3.12 (#4215)
add 9bc1ebdff0 fix: prevent orphaned sessions when lakeFS abort fails
(#4197)
add 179f177d72 feat: add user registration form (#4185)
add 61726182b8 fix: Fix/macos testcase (#4222)
add 93f587fa32 feat: add Markdown editor for dataset/workflow descriptions
(#4212)
add 1409301e4a Merge branch 'main' into xinyuan-cm-for-loop-mat-dcm
No new revisions were added by this update.
Summary of changes:
.github/release/vote-email-template.md | 64 +-
.github/workflows/create-release-candidate.yml | 23 +-
.github/workflows/github-action-build.yml | 25 +-
.../service/resource/AccessControlResource.scala | 11 +-
amber/requirements.txt | 1 +
amber/src/main/python/core/models/operator.py | 45 +-
.../input_port_materialization_reader_runnable.py | 3 +-
.../controller/execution/WorkflowExecution.scala | 18 +-
.../QueryWorkerStatisticsHandler.scala | 32 +-
.../texera/web/ServletAwareConfigurator.scala | 2 +
.../apache/texera/web/auth/GuestAuthFilter.scala | 2 +-
.../apache/texera/web/auth/UserAuthenticator.scala | 1 +
.../apache/texera/web/resource/GmailResource.scala | 9 +-
.../texera/web/resource/UserConfigResource.scala | 13 +-
.../texera/web/resource/auth/AuthResource.scala | 15 +-
.../web/resource/auth/GoogleAuthResource.scala | 13 +-
.../resource/dashboard/SearchQueryBuilder.scala | 7 +-
.../resource/dashboard/UnifiedResourceSchema.scala | 7 +-
.../admin/execution/AdminExecutionResource.scala | 7 +-
.../admin/settings/AdminSettingsResource.scala | 2 +-
.../dashboard/admin/user/AdminUserResource.scala | 15 +-
.../web/resource/dashboard/hub/HubResource.scala | 14 +-
.../web/resource/dashboard/user/UserResource.scala | 65 +-
.../dataset/utils/DatasetStatisticsUtils.scala | 8 +-
.../user/project/ProjectAccessResource.scala | 20 +-
.../dashboard/user/project/ProjectResource.scala | 13 +-
.../user/project/PublicProjectResource.scala | 11 +-
.../dashboard/user/quota/UserQuotaResource.scala | 7 +-
.../user/workflow/WorkflowAccessResource.scala | 7 +-
.../user/workflow/WorkflowExecutionsResource.scala | 9 +-
.../dashboard/user/workflow/WorkflowResource.scala | 27 +-
.../user/workflow/WorkflowVersionResource.scala | 11 +-
.../service/ExecutionsMetadataPersistService.scala | 14 +-
.../user/workflow/WorkflowAccessResourceSpec.scala | 405 +++++++++
bin/k8s/Chart.yaml | 10 +-
.../access-control-service-deployment.yaml | 4 +
bin/k8s/templates/envoy-config.yaml | 147 ----
bin/k8s/templates/envoy-deployment.yaml | 49 --
...g-units-namespace.yaml => gateway-backend.yaml} | 11 +-
bin/k8s/templates/gateway-routes.yaml | 142 ++++
...y-service.yaml => gateway-security-policy.yaml} | 33 +-
bin/k8s/templates/gateway.yaml | 81 ++
bin/k8s/templates/ingress.yaml | 53 --
bin/k8s/templates/minio-ingress.yaml | 65 --
bin/k8s/values.yaml | 126 +--
build.sbt | 9 +-
.../org/apache/texera/auth/JwtAuthFilter.scala | 2 +-
.../scala/org/apache/texera/auth/JwtParser.scala | 3 +-
.../texera/auth/util/ComputingUnitAccess.scala | 7 +-
common/config/src/main/resources/kubernetes.conf | 5 +-
.../apache/texera/config/KubernetesConfig.scala | 1 +
.../scala/org/apache/texera/dao/SqlServer.scala | 14 +-
.../pybuilder}/build.sbt | 37 +-
.../texera/amber/pybuilder/BoundaryValidator.scala | 187 +++++
.../amber/pybuilder/EncodableInspector.scala | 162 ++++
.../pybuilder/EncodableStringAnnotation.java} | 19 +-
.../texera/amber/pybuilder/PythonLexerUtils.scala | 84 ++
.../amber/pybuilder/PythonTemplateBuilder.scala | 481 +++++++++++
.../amber/pybuilder/PythonLexerUtilsSpec.scala | 167 ++++
.../pybuilder/PythonTemplateBuilderSpec.scala | 598 ++++++++++++++
common/workflow-core/build.sbt | 2 +-
.../texera/service/util/S3StorageClient.scala | 1 +
common/workflow-operator/build.sbt | 2 +
.../apache/texera/amber/operator/LogicalOp.scala | 4 +-
.../HuggingFaceIrisLogisticRegressionOpDesc.scala | 23 +-
.../HuggingFaceSentimentAnalysisOpDesc.scala | 18 +-
.../HuggingFaceSpamSMSDetectionOpDesc.scala | 18 +-
.../HuggingFaceTextSummarizationOpDesc.scala | 14 +-
.../Scorer/MachineLearningScorerOpDesc.scala | 20 +-
.../sklearnAdvanced/base/HyperParameters.java | 58 --
.../sklearnAdvanced/base/HyperParameters.scala | 69 ++
.../base/SklearnAdvancedBaseDesc.scala | 62 +-
.../operator/sklearn/SklearnClassifierOpDesc.scala | 16 +-
.../sklearn/SklearnLinearRegressionOpDesc.scala | 12 +-
.../operator/sklearn/SklearnPredictionOpDesc.scala | 22 +-
.../sklearn/testing/SklearnTestingOpDesc.scala | 115 +++
.../sklearn/training/SklearnTrainingOpDesc.scala | 16 +-
.../amber/operator/sort/SortCriteriaUnit.java | 36 -
.../SortCriteriaUnit.scala} | 28 +-
.../texera/amber/operator/sort/SortOpDesc.scala | 7 +-
.../apis/reddit/RedditSearchSourceOpDesc.scala | 26 +-
.../operator/timeSeriesPlot/TimeSeriesPlot.scala | 30 +-
.../visualization/DotPlot/DotPlotOpDesc.scala | 21 +-
.../IcicleChart/IcicleChartOpDesc.scala | 31 +-
.../ImageViz/ImageVisualizerOpDesc.scala | 19 +-
.../ScatterMatrixChartOpDesc.scala | 23 +-
.../visualization/barChart/BarChartOpDesc.scala | 33 +-
.../boxViolinPlot/BoxViolinPlotOpDesc.scala | 35 +-
.../bubbleChart/BubbleChartOpDesc.scala | 39 +-
.../bulletChart/BulletChartOpDesc.scala | 22 +-
.../bulletChart/BulletChartStepDefinition.scala | 5 +-
.../candlestickChart/CandlestickChartOpDesc.scala | 26 +-
.../choroplethMap/ChoroplethMapOpDesc.scala | 31 +-
.../continuousErrorBands/BandConfig.scala | 7 +-
.../ContinuousErrorBandsOpDesc.scala | 47 +-
.../contourPlot/ContourPlotOpDesc.scala | 28 +-
.../dendrogram/DendrogramOpDesc.scala | 33 +-
.../dumbbellPlot/DumbbellDotConfig.scala | 3 +-
.../dumbbellPlot/DumbbellPlotOpDesc.scala | 49 +-
.../FigureFactoryTableConfig.scala | 3 +-
.../FigureFactoryTableOpDesc.scala | 55 +-
.../filledAreaPlot/FilledAreaPlotOpDesc.scala | 55 +-
.../funnelPlot/FunnelPlotOpDesc.scala | 40 +-
.../ganttChart/GanttChartOpDesc.scala | 41 +-
.../gaugeChart/GaugeChartOpDesc.scala | 22 +-
.../visualization/gaugeChart/GaugeChartSteps.scala | 5 +-
.../visualization/heatMap/HeatMapOpDesc.scala | 23 +-
.../hierarchychart/HierarchyChartOpDesc.scala | 31 +-
.../hierarchychart/HierarchySection.scala | 3 +-
.../histogram/HistogramChartOpDesc.scala | 45 +-
.../histogram2d/Histogram2DOpDesc.scala | 28 +-
.../visualization/lineChart/LineChartOpDesc.scala | 37 +-
.../visualization/lineChart/LineConfig.scala | 9 +-
.../nestedTable/NestedTableConfig.scala | 7 +-
.../nestedTable/NestedTableOpDesc.scala | 18 +-
.../networkGraph/NetworkGraphOpDesc.scala | 34 +-
.../visualization/pieChart/PieChartOpDesc.scala | 31 +-
.../quiverPlot/QuiverPlotOpDesc.scala | 29 +-
.../rangeSlider/RangeSliderOpDesc.scala | 39 +-
.../sankeyDiagram/SankeyDiagramOpDesc.scala | 75 +-
.../scatter3DChart/Scatter3dChartOpDesc.scala | 61 +-
.../scatterplot/ScatterplotOpDesc.scala | 51 +-
.../stripChart/StripChartOpDesc.scala | 40 +-
.../visualization/tablesChart/TablesConfig.scala | 3 +-
.../tablesChart/TablesPlotOpDesc.scala | 42 +-
.../ternaryPlot/TernaryPlotOpDesc.scala | 39 +-
.../visualization/treeplot/TreeplotOpDesc.scala | 12 +-
.../volcanoPlot/VolcanoPlotOpDesc.scala | 20 +-
.../waterfallChart/WaterfallChartOpDesc.scala | 23 +-
.../visualization/wordCloud/WordCloudOpDesc.scala | 31 +-
.../timeSeriesPlot/TimeSeriesOpDescSpec.scala | 38 +-
.../visualization/DotPlot/DotPlotOpDescSpec.scala | 3 +-
.../barChart/BarChartOpDescSpec.scala | 2 +-
.../bubbleChart/BubbleChartOpDescSpec.scala | 3 +-
.../ganttChart/GanttChartOpDescSpec.scala | 15 +-
.../hierarchychart/HierarchyChartOpDescSpec.scala | 2 -
.../texera/amber/pybuilder/DescriptorChecker.scala | 902 +++++++++++++++++++++
.../amber/pybuilder/PythonClassgraphScanner.scala | 56 ++
.../amber/pybuilder/PythonConsoleCapture.scala | 44 +
.../pybuilder/PythonRawTextReportRenderer.scala | 53 ++
.../pybuilder/PythonReflectionTextUtils.scala | 64 ++
.../amber/pybuilder/PythonReflectionUtils.scala | 65 ++
.../amber/util/PythonCodeRawInvalidTextSpec.scala | 266 ++++++
.../resource/ComputingUnitAccessResource.scala | 7 +-
.../resource/ComputingUnitManagingResource.scala | 7 +-
file-service/build.sbt | 2 +-
.../service/resource/DatasetAccessResource.scala | 7 +-
.../texera/service/resource/DatasetResource.scala | 408 +++++++---
.../service/util/LakeFSExceptionHandler.scala | 79 ++
.../service/resource/DatasetResourceSpec.scala | 637 ++++++++++++++-
frontend/.yarnrc.yml | 1 +
frontend/src/app/app.module.ts | 6 +
.../src/app/common/service/gmail/gmail.service.ts | 2 +-
.../src/app/common/service/user/auth.service.ts | 104 ++-
.../registration-request-modal.component.html | 54 ++
.../registration-request-modal.component.scss} | 19 +-
.../registration-request-modal.component.ts} | 40 +-
.../app/common/service/user/stub-user.service.ts | 2 +
.../src/app/common/service/user/user.service.ts | 32 -
frontend/src/app/common/type/user.ts | 1 +
.../component/admin/user/admin-user.component.html | 6 +
.../component/admin/user/admin-user.component.ts | 5 +
.../dashboard/component/dashboard.component.html | 32 -
.../app/dashboard/component/dashboard.component.ts | 67 +-
.../conflicting-file-modal-content.component.html} | 26 +-
.../conflicting-file-modal-content.component.scss} | 3 +
.../conflicting-file-modal-content.component.ts} | 33 +-
.../files-uploader/files-uploader.component.ts | 214 ++++-
.../user/list-item/list-item.component.html | 15 +-
.../user/list-item/list-item.component.scss | 1 +
.../user/list-item/list-item.component.ts | 49 +-
.../markdown-description.component.html | 77 ++
.../markdown-description.component.scss} | 103 +--
.../markdown-description.component.ts | 117 +++
.../dataset-detail.component.html | 4 +-
.../dataset-detail.component.ts | 31 +-
.../service/user/dataset/dataset.service.ts | 91 +--
.../app/dashboard/type/dashboard-file.interface.ts | 1 +
.../detail/hub-workflow-detail.component.html | 2 +-
.../coeditor-user-icon.component.ts | 10 +-
.../workspace/component/menu/menu.component.html | 8 +
.../app/workspace/component/menu/menu.component.ts | 18 +
.../workflow-editor/workflow-editor.component.scss | 4 +
.../workflow-editor/workflow-editor.component.ts | 1 +
.../workspace/service/joint-ui/joint-ui.service.ts | 19 +
.../src/assets/operator_images/SklearnTesting.png | Bin 0 -> 843070 bytes
sql/texera_ddl.sql | 6 +-
sql/updates/{15.sql => 20.sql} | 21 +-
sql/updates/{16.sql => 21.sql} | 2 +-
sql/updates/{18.sql => 22.sql} | 7 +-
190 files changed, 7159 insertions(+), 2161 deletions(-)
create mode 100644
amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowAccessResourceSpec.scala
delete mode 100644 bin/k8s/templates/envoy-config.yaml
delete mode 100644 bin/k8s/templates/envoy-deployment.yaml
copy bin/k8s/templates/{workflow-computing-units-namespace.yaml =>
gateway-backend.yaml} (82%)
create mode 100644 bin/k8s/templates/gateway-routes.yaml
rename bin/k8s/templates/{envoy-service.yaml => gateway-security-policy.yaml}
(60%)
create mode 100644 bin/k8s/templates/gateway.yaml
delete mode 100644 bin/k8s/templates/ingress.yaml
delete mode 100644 bin/k8s/templates/minio-ingress.yaml
copy {access-control-service => common/pybuilder}/build.sbt (73%)
create mode 100644
common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/BoundaryValidator.scala
create mode 100644
common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/EncodableInspector.scala
copy
common/{workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/annotations/BatchByColumn.java
=>
pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/EncodableStringAnnotation.java}
(69%)
create mode 100644
common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonLexerUtils.scala
create mode 100644
common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilder.scala
create mode 100644
common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonLexerUtilsSpec.scala
create mode 100644
common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonTemplateBuilderSpec.scala
delete mode 100644
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/HyperParameters.java
create mode 100644
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/sklearnAdvanced/base/HyperParameters.scala
create mode 100644
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/testing/SklearnTestingOpDesc.scala
delete mode 100644
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sort/SortCriteriaUnit.java
copy
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/{visualization/dumbbellPlot/DumbbellDotConfig.scala
=> sort/SortCriteriaUnit.scala} (63%)
copy
amber/src/main/scala/org/apache/texera/amber/engine/common/CheckpointState.scala
=>
common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/timeSeriesPlot/TimeSeriesOpDescSpec.scala
(54%)
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/DescriptorChecker.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/PythonClassgraphScanner.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/PythonConsoleCapture.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/PythonRawTextReportRenderer.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/PythonReflectionTextUtils.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/pybuilder/PythonReflectionUtils.scala
create mode 100644
common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonCodeRawInvalidTextSpec.scala
create mode 100644
file-service/src/main/scala/org/apache/texera/service/util/LakeFSExceptionHandler.scala
create mode 100644
frontend/src/app/common/service/user/registration-request-modal/registration-request-modal.component.html
copy
frontend/src/app/{dashboard/component/user/search-bar/search-bar.component.scss
=>
common/service/user/registration-request-modal/registration-request-modal.component.scss}
(83%)
copy
frontend/src/app/common/service/{notification/notification.service.spec.ts =>
user/registration-request-modal/registration-request-modal.component.ts} (51%)
copy
frontend/src/app/dashboard/component/user/{user-avatar/user-avatar.component.html
=>
files-uploader/conflicting-file-modal-content/conflicting-file-modal-content.component.html}
(67%)
copy
frontend/src/app/{workspace/component/menu/coeditor-user-icon/coeditor-user-icon.component.css
=>
dashboard/component/user/files-uploader/conflicting-file-modal-content/conflicting-file-modal-content.component.scss}
(96%)
copy
frontend/src/app/{workspace/component/code-editor-dialog/annotation-suggestion.component.ts
=>
dashboard/component/user/files-uploader/conflicting-file-modal-content/conflicting-file-modal-content.component.ts}
(58%)
create mode 100644
frontend/src/app/dashboard/component/user/markdown-description/markdown-description.component.html
copy
frontend/src/app/{workspace/component/file-selection/file-selection.component.scss
=>
dashboard/component/user/markdown-description/markdown-description.component.scss}
(50%)
create mode 100644
frontend/src/app/dashboard/component/user/markdown-description/markdown-description.component.ts
create mode 100644 frontend/src/assets/operator_images/SklearnTesting.png
copy sql/updates/{15.sql => 20.sql} (71%)
copy sql/updates/{16.sql => 21.sql} (93%)
copy sql/updates/{18.sql => 22.sql} (83%)
