This is an automated email from the ASF dual-hosted git repository.
atul pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
from 176934e849 Web console: expose in the UI that a query is stuck waiting
for task slots (#13291)
add e5ad24ff9f Support for middle manager less druid, tasks launch as k8s
jobs (#13156)
No new revisions were added by this update.
Summary of changes:
.../org/apache/druid/indexer/TaskLocation.java | 14 +
.../java/org/apache/druid/indexer/TaskStatus.java | 5 +
.../org/apache/druid/indexer/TaskLocationTest.java | 11 +
.../org/apache/druid/indexer/TaskStatusTest.java | 5 +
distribution/docker/Dockerfile | 1 +
distribution/docker/peon.sh | 155 ++++++
distribution/pom.xml | 2 +
docs/development/extensions-contrib/k8s-jobs.md | 129 +++++
docs/development/extensions.md | 1 +
.../kubernetes-overlord-extensions/pom.xml | 238 +++++++++
.../druid/k8s/overlord/K8sOverlordConfig.java | 5 +-
.../druid/k8s/overlord/K8sOverlordModule.java | 82 +++
.../druid/k8s/overlord/KubernetesTaskRunner.java | 549 +++++++++++++++++++++
.../k8s/overlord/KubernetesTaskRunnerConfig.java | 110 +++++
.../k8s/overlord/KubernetesTaskRunnerFactory.java | 112 +++++
.../k8s/overlord/common/Base64Compression.java | 51 ++
.../k8s/overlord/common/DruidK8sConstants.java | 18 +-
.../k8s/overlord/common/DruidKubernetesClient.java | 29 +-
.../overlord/common/DruidKubernetesPeonClient.java | 274 ++++++++++
.../druid/k8s/overlord/common/JobResponse.java | 78 +++
.../druid/k8s/overlord/common/K8sTaskAdapter.java | 278 +++++++++++
.../druid/k8s/overlord/common/K8sTaskId.java | 82 +++
.../k8s/overlord/common/KubernetesClientApi.java | 7 +-
.../k8s/overlord/common/KubernetesExecutor.java | 9 +-
.../k8s/overlord/common/KubernetesPeonClient.java | 62 +++
.../KubernetesResourceNotFoundException.java | 9 +-
.../overlord/common/MultiContainerTaskAdapter.java | 151 ++++++
.../k8s/overlord/common/PeonCommandContext.java | 73 +++
.../druid/k8s/overlord/common/PeonPhase.java | 62 +++
.../common/SingleContainerTaskAdapter.java | 68 +++
.../druid/k8s/overlord/common/TaskAdapter.java | 14 +-
.../org.apache.druid.initialization.DruidModule | 16 +
.../k8s/overlord/KubernetesTaskRunnerTest.java | 476 ++++++++++++++++++
.../k8s/overlord/common/Base64CompressionTest.java | 48 ++
.../common/DruidKubernetesPeonClientTest.java | 151 ++++++
.../common/DruidPeonClientIntegrationTest.java | 152 ++++++
.../druid/k8s/overlord/common/JobResponseTest.java | 78 +++
.../k8s/overlord/common/K8sTaskAdapterTest.java | 163 ++++++
.../druid/k8s/overlord/common/K8sTaskIdTest.java | 26 +-
.../druid/k8s/overlord/common/K8sTestUtils.java | 145 ++++++
.../common/MultiContainerTaskAdapterTest.java | 107 ++++
.../druid/k8s/overlord/common/PeonPhaseTest.java | 24 +-
.../common/SingleContainerTaskAdapterTest.java | 110 +++++
.../k8s/overlord/common/TestKubernetesClient.java | 20 +-
.../resources/expectedMultiContainerOutput.yaml | 108 ++++
.../src/test/resources/expectedPodSpec.yaml | 66 +++
.../resources/expectedSingleiContainerOutput.yaml | 60 +++
.../src/test/resources/multiContainerPodSpec.yaml | 26 +
.../src/test/resources/podSpec.yaml | 15 +
.../druid/indexing/kafka/KafkaIndexTaskTest.java | 7 +-
.../indexing/kinesis/KinesisIndexTaskTest.java | 7 +-
.../druid/msq/indexing/MSQControllerTask.java | 2 +-
.../apache/druid/msq/indexing/MSQWorkerTask.java | 2 +-
.../apache/druid/indexing/common/TaskToolbox.java | 39 +-
.../druid/indexing/common/TaskToolboxFactory.java | 12 +-
.../druid/indexing/common/actions/TaskAction.java | 4 +-
.../indexing/common/actions/TaskActionToolbox.java | 19 +
.../common/actions/UpdateLocationAction.java | 81 +++
.../common/actions/UpdateStatusAction.java | 85 ++++
.../druid/indexing/common/config/TaskConfig.java | 14 +-
.../common/task/AbstractBatchIndexTask.java | 13 +-
.../druid/indexing/common/task/AbstractTask.java | 76 +++
.../task/AppenderatorDriverRealtimeIndexTask.java | 2 +-
.../druid/indexing/common/task/ArchiveTask.java | 2 +-
.../common/task/KillUnusedSegmentsTask.java | 2 +-
.../druid/indexing/common/task/MoveTask.java | 2 +-
.../druid/indexing/common/task/NoopTask.java | 2 +-
.../indexing/common/task/RealtimeIndexTask.java | 2 +-
.../druid/indexing/common/task/RestoreTask.java | 2 +-
.../task/batch/parallel/SinglePhaseSubTask.java | 2 +-
.../druid/indexing/overlord/ForkingTaskRunner.java | 88 ++--
.../overlord/ForkingTaskRunnerFactory.java | 10 +-
.../overlord/QuotableWhiteSpaceSplitter.java | 62 +++
.../indexing/overlord/RemoteTaskRunnerFactory.java | 10 +-
.../apache/druid/indexing/overlord/TaskRunner.java | 20 +
.../druid/indexing/overlord/TaskRunnerFactory.java | 2 +
.../overlord/hrtr/HttpRemoteTaskRunnerFactory.java | 10 +-
.../indexing/overlord/http/OverlordResource.java | 31 +-
.../seekablestream/SeekableStreamIndexTask.java | 2 +-
.../worker/executor/ExecutorLifecycle.java | 88 ++--
.../worker/executor/ExecutorLifecycleConfig.java | 6 +
.../druid/indexing/common/TaskToolboxTest.java | 7 +-
.../druid/indexing/common/TestRealtimeTask.java | 2 +-
.../apache/druid/indexing/common/TestTasks.java | 4 +-
.../common/actions/TaskActionToolboxTest.java | 44 ++
.../common/actions/UpdateLocationActionTest.java | 71 +++
.../common/actions/UpdateStatusActionTest.java | 76 +++
.../indexing/common/task/AbstractTaskTest.java | 111 +++++
.../AppenderatorDriverRealtimeIndexTaskTest.java | 7 +-
.../common/task/BatchAppenderatorsTest.java | 6 +-
.../common/task/CompactionTaskRunTest.java | 5 +-
.../indexing/common/task/CompactionTaskTest.java | 7 +-
.../druid/indexing/common/task/HadoopTaskTest.java | 3 +-
.../indexing/common/task/IngestionTestBase.java | 6 +-
.../common/task/RealtimeIndexTaskTest.java | 7 +-
.../AbstractParallelIndexSupervisorTaskTest.java | 11 +-
.../parallel/ParallelIndexPhaseRunnerTest.java | 4 +-
.../ParallelIndexSupervisorTaskKillTest.java | 2 +-
.../ParallelIndexSupervisorTaskResourceTest.java | 2 +-
.../task/batch/parallel/TaskMonitorTest.java | 18 +-
.../indexing/overlord/ForkingTaskRunnerTest.java | 90 +++-
.../druid/indexing/overlord/RealtimeishTask.java | 2 +-
.../overlord/SingleTaskBackgroundRunnerTest.java | 57 ++-
.../druid/indexing/overlord/TaskLifecycleTest.java | 15 +-
.../druid/indexing/overlord/TaskLockboxTest.java | 2 +-
.../druid/indexing/overlord/TaskQueueTest.java | 13 +
.../indexing/overlord/ThreadingTaskRunnerTest.java | 5 +-
.../druid/indexing/overlord/http/OverlordTest.java | 20 +-
.../indexing/worker/WorkerTaskManagerTest.java | 9 +-
.../indexing/worker/WorkerTaskMonitorTest.java | 7 +-
.../executor/ExecutorLifecycleConfigTest.java} | 16 +-
...ocalIntermediaryDataManagerAutoCleanupTest.java | 3 +-
...ermediaryDataManagerManualAddAndDeleteTest.java | 3 +-
.../shuffle/ShuffleDataSegmentPusherTest.java | 3 +-
.../worker/shuffle/ShuffleResourceTest.java | 3 +-
licenses.yaml | 10 +
pom.xml | 1 +
.../client/indexing/HttpIndexingServiceClient.java | 1 +
.../apache/druid/guice/annotations/AttemptId.java | 15 +-
.../main/java/org/apache/druid/cli/CliIndexer.java | 3 +
.../java/org/apache/druid/cli/CliOverlord.java | 19 +
.../main/java/org/apache/druid/cli/CliPeon.java | 44 +-
.../java/org/apache/druid/cli/CliPeonTest.java | 90 ++++
.../test/java/org/apache/druid/cli/MainTest.java | 3 -
website/.spelling | 7 +
125 files changed, 5755 insertions(+), 260 deletions(-)
create mode 100755 distribution/docker/peon.sh
create mode 100644 docs/development/extensions-contrib/k8s-jobs.md
create mode 100644 extensions-contrib/kubernetes-overlord-extensions/pom.xml
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/K8sOverlordConfig.java
(87%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/K8sOverlordModule.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunner.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerConfig.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerFactory.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/Base64Compression.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidK8sConstants.java
(51%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesClient.java
(53%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClient.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/JobResponse.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskAdapter.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/K8sTaskId.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesClientApi.java
(75%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesExecutor.java
(76%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesPeonClient.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/KubernetesResourceNotFoundException.java
(79%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/MultiContainerTaskAdapter.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonCommandContext.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/PeonPhase.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapter.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/common/TaskAdapter.java
(74%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/KubernetesTaskRunnerTest.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/Base64CompressionTest.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/DruidKubernetesPeonClientTest.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/DruidPeonClientIntegrationTest.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/JobResponseTest.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTaskAdapterTest.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTaskIdTest.java
(52%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/K8sTestUtils.java
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/MultiContainerTaskAdapterTest.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/PeonPhaseTest.java
(53%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/SingleContainerTaskAdapterTest.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
extensions-contrib/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/common/TestKubernetesClient.java
(63%)
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/resources/expectedSingleiContainerOutput.yaml
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/resources/multiContainerPodSpec.yaml
create mode 100644
extensions-contrib/kubernetes-overlord-extensions/src/test/resources/podSpec.yaml
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateLocationAction.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/actions/UpdateStatusAction.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/overlord/QuotableWhiteSpaceSplitter.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionToolboxTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/actions/UpdateLocationActionTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/actions/UpdateStatusActionTest.java
copy
indexing-service/src/{main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=>
test/java/org/apache/druid/indexing/worker/executor/ExecutorLifecycleConfigTest.java}
(68%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/overlord/TaskRunnerFactory.java
=> server/src/main/java/org/apache/druid/guice/annotations/AttemptId.java (66%)
create mode 100644 services/src/test/java/org/apache/druid/cli/CliPeonTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
