This is an automated email from the ASF dual-hosted git repository. xvrl pushed a commit to branch dependabot/maven/org.postgresql-postgresql-42.6.0 in repository https://gitbox.apache.org/repos/asf/druid.git
commit 164aabca4e19e5bcbadb509fe26cb981446d7955 Merge: 3095dfe5f8 afe22907a5 Author: Xavier Léauté <[email protected]> AuthorDate: Sat Aug 12 10:31:28 2023 -0400 Merge remote-tracking branch 'origin/master' into dependabot/maven/org.postgresql-postgresql-42.6.0 .github/dependabot.yml | 3 + .github/scripts/analyze_dependencies_script.sh | 2 +- .github/scripts/license_checks_script.sh | 2 +- .github/workflows/cron-job-its.yml | 6 +- .github/workflows/stale.yml | 43 +- .github/workflows/static-checks.yml | 7 - LICENSE | 4 + .../benchmark/FrontCodedIndexedBenchmark.java | 18 +- .../query/CachingClusteredClientBenchmark.java | 9 +- .../coordinator/BalancerStrategyBenchmark.java | 4 +- .../org/apache/druid/common/aws/AWSClientUtil.java | 54 +- .../apache/druid/common/aws/AWSClientUtilTest.java | 16 + distribution/asf-release-process-guide.md | 2 +- distribution/bin/find-missing-backports.py | 75 +- distribution/pom.xml | 185 -- docs/api-reference/json-querying-api.md | 874 +++++- docs/api-reference/sql-api.md | 813 ++++- docs/api-reference/sql-jdbc.md | 55 +- docs/api-reference/supervisor-api.md | 3136 +++++++++++++++++++- docs/api-reference/tasks-api.md | 363 ++- .../tutorial-query-deepstorage-retention-rule.png | Bin 0 -> 99106 bytes docs/assets/web-console-0.7-tasks.png | Bin 0 -> 83141 bytes docs/assets/web-console-01-home-view.png | Bin 74166 -> 87581 bytes docs/assets/web-console-02-data-loader-1.png | Bin 99786 -> 80007 bytes docs/assets/web-console-03-data-loader-2.png | Bin 275998 -> 268671 bytes docs/assets/web-console-04-datasources.png | Bin 112268 -> 97639 bytes docs/assets/web-console-05-retention.png | Bin 83062 -> 86941 bytes docs/assets/web-console-06-segments.png | Bin 79223 -> 103333 bytes docs/assets/web-console-07-supervisors.png | Bin 136709 -> 51193 bytes docs/assets/web-console-08-supervisor-status.png | Bin 93077 -> 89501 bytes docs/assets/web-console-09-task-status.png | Bin 195995 -> 201632 bytes docs/assets/web-console-10-servers.png | Bin 93976 -> 87782 bytes docs/assets/web-console-13-lookups.png | Bin 71735 -> 66934 bytes docs/configuration/index.md | 46 +- docs/data-management/delete.md | 13 +- docs/design/architecture.md | 23 +- docs/design/deep-storage.md | 26 +- docs/development/extensions-contrib/k8s-jobs.md | 268 +- docs/development/extensions-core/hdfs.md | 4 +- .../extensions-core/kafka-supervisor-reference.md | 2 - .../extensions-core/kinesis-ingestion.md | 24 +- docs/development/modules.md | 4 +- docs/ingestion/hadoop.md | 2 +- docs/ingestion/ingestion-spec.md | 2 +- docs/ingestion/schema-design.md | 4 +- docs/multi-stage-query/concepts.md | 4 +- docs/multi-stage-query/reference.md | 66 +- docs/operations/durable-storage.md | 86 + docs/operations/metrics.md | 155 +- docs/operations/other-hadoop.md | 2 +- docs/operations/rule-configuration.md | 10 +- docs/operations/web-console.md | 46 +- docs/querying/aggregations.md | 404 ++- docs/querying/datasourcemetadataquery.md | 2 +- docs/querying/filters.md | 738 +++-- docs/querying/multitenancy.md | 2 +- docs/querying/query-from-deep-storage.md | 195 ++ docs/querying/querying.md | 2 +- docs/querying/searchquery.md | 2 +- docs/querying/sql-aggregations.md | 4 +- docs/querying/sql-query-context.md | 1 + docs/tutorials/tutorial-batch-hadoop.md | 15 +- docs/tutorials/tutorial-kafka.md | 3 +- docs/tutorials/tutorial-query-deep-storage.md | 293 ++ .../druid/auto/_common/common.runtime.properties | 2 +- .../cluster/_common/common.runtime.properties | 2 +- .../large/_common/common.runtime.properties | 2 +- .../medium/_common/common.runtime.properties | 2 +- .../_common/common.runtime.properties | 2 +- .../_common/common.runtime.properties | 2 +- .../small/_common/common.runtime.properties | 2 +- .../xlarge/_common/common.runtime.properties | 2 +- examples/quickstart/jupyter-notebooks/Dockerfile | 14 +- .../docker-jupyter/docker-compose-local.yaml | 10 + .../docker-jupyter/docker-compose.yaml | 10 + .../jupyter-notebooks/docker-jupyter/environment | 4 +- .../jupyter-notebooks/druidapi/druidapi/display.py | 34 + .../jupyter-notebooks/druidapi/druidapi/sql.py | 11 +- .../jupyter-notebooks/druidapi/druidapi/tasks.py | 17 +- .../notebooks/01-introduction/00-START-HERE.ipynb | 3 +- .../01-druidapi-package-intro.ipynb | 4 +- .../01-introduction/02-datagen-intro.ipynb | 642 ++++ .../02-ingestion/01-streaming-from-kafka.ipynb | 255 +- .../notebooks/02-ingestion/DruidDataDriver.py | 1133 ------- .../02-ingestion/kafka_docker_config.json | 90 - examples/quickstart/releases/Druid26.ipynb | 416 +++ .../quickstart/releases/README.md | 19 +- .../quickstart/tutorial/hadoop/docker/Dockerfile | 46 +- .../quickstart/tutorial/hadoop/docker/bootstrap.sh | 13 +- .../quickstart/tutorial/hadoop3/docker/Dockerfile | 143 - .../tutorial/hadoop3/docker/bootstrap.sh | 41 - .../tutorial/hadoop3/docker/core-site.xml.template | 24 - .../tutorial/hadoop3/docker/hdfs-site.xml | 32 - .../tutorial/hadoop3/docker/mapred-site.xml | 24 - .../quickstart/tutorial/hadoop3/docker/ssh_config | 20 - .../tutorial/hadoop3/docker/yarn-site.xml | 65 - .../tutorial/wikipedia-index-hadoop.json | 79 - extensions-contrib/cloudfiles-extensions/pom.xml | 9 +- .../cloudfiles/CloudFilesByteSourceTest.java | 50 +- .../CloudFilesDataSegmentPusherTest.java | 24 +- .../cloudfiles/CloudFilesObjectApiProxyTest.java | 28 +- extensions-contrib/compressed-bigdecimal/pom.xml | 3 - .../DistinctCountGroupByQueryTest.java | 14 +- .../distinctcount/DistinctCountTopNQueryTest.java | 2 +- extensions-contrib/kafka-emitter/pom.xml | 5 + .../druid/emitter/kafka/KafkaEmitterTest.java | 5 +- .../kubernetes-overlord-extensions/README.md | 22 +- .../k8s/overlord/KubernetesPeonLifecycle.java | 58 +- .../overlord/KubernetesPeonLifecycleFactory.java | 5 +- .../druid/k8s/overlord/KubernetesTaskRunner.java | 105 +- .../k8s/overlord/KubernetesTaskRunnerFactory.java | 9 +- .../druid/k8s/overlord/KubernetesWorkItem.java | 5 + .../druid/k8s/overlord/PeonLifecycleFactory.java | 2 +- .../k8s/overlord/common/DruidK8sConstants.java | 1 + .../taskadapter/PodTemplateTaskAdapter.java | 6 +- .../k8s/overlord/KubernetesPeonLifecycleTest.java | 260 +- .../overlord/KubernetesTaskRunnerFactoryTest.java | 27 +- .../k8s/overlord/KubernetesTaskRunnerTest.java | 43 +- .../druid/k8s/overlord/KubernetesWorkItemTest.java | 14 + .../k8s/overlord/TestPeonLifecycleFactory.java | 2 +- .../taskadapter/PodTemplateTaskAdapterTest.java | 38 + .../src/test/resources/expectedNoopJob.yaml | 2 + .../src/test/resources/expectedNoopJobLongIds.yaml | 2 + .../test/resources/expectedNoopJobTlsEnabled.yaml | 2 + .../MaterializedViewSupervisorSpec.java | 3 +- .../main/resources/defaultMetricDimensions.json | 6 +- extensions-contrib/thrift-extensions/pom.xml | 28 - .../druid/segment/MapVirtualColumnTopNTest.java | 2 +- extensions-core/avro-extensions/pom.xml | 45 - extensions-core/azure-extensions/pom.xml | 13 +- .../druid/data/input/azure/AzureInputSource.java | 2 +- .../druid/storage/azure/AzureByteSource.java | 2 +- .../storage/azure/AzureDataSegmentPusher.java | 2 +- .../apache/druid/storage/azure/AzureStorage.java | 141 +- .../storage/azure/AzureStorageDruidModule.java | 2 +- .../apache/druid/storage/azure/AzureTaskLogs.java | 8 +- .../storage/azure/output/AzureInputRange.java | 99 + .../storage/azure/output/AzureOutputConfig.java | 171 ++ .../azure/output/AzureStorageConnector.java | 219 ++ .../azure/output/AzureStorageConnectorModule.java | 13 +- .../output/AzureStorageConnectorProvider.java | 59 + .../org.apache.druid.initialization.DruidModule | 1 + .../druid/storage/azure/AzureByteSourceTest.java | 6 +- .../storage/azure/AzureDataSegmentPullerTest.java | 8 +- .../storage/azure/AzureDataSegmentPusherTest.java | 12 +- .../druid/storage/azure/AzureStorageTest.java | 71 + .../druid/storage/azure/AzureTaskLogsTest.java | 60 +- .../storage/azure/output/AzureInputRangeTest.java | 17 +- .../azure/output/AzureOutputConfigTest.java | 85 + .../storage/azure/output/AzureOutputSerdeTest.java | 140 + .../output/AzureStorageConnectorProviderTest.java | 148 + .../azure/output/AzureStorageConnectorTest.java | 202 ++ .../DoublesSketchApproxQuantileSqlAggregator.java | 14 +- ...DoublesSketchListArgBaseOperatorConversion.java | 5 +- .../sql/DoublesSketchSqlAggregatorTest.java | 7 +- extensions-core/druid-ranger-security/pom.xml | 232 -- extensions-core/hdfs-storage/pom.xml | 289 -- .../ApproximateHistogramGroupByQueryTest.java | 10 +- .../ApproximateHistogramTopNQueryTest.java | 216 +- .../FixedBucketsHistogramGroupByQueryTest.java | 11 +- .../FixedBucketsHistogramTopNQueryTest.java | 36 +- ...dBucketsHistogramQuantileSqlAggregatorTest.java | 7 +- .../histogram/sql/QuantileSqlAggregatorTest.java | 7 +- .../kafka/KafkaIndexTaskClientFactory.java | 4 +- .../kafka/supervisor/KafkaSupervisorIOConfig.java | 6 +- .../supervisor/KafkaSupervisorTuningConfig.java | 34 - .../indexing/kafka/KafkaIndexTaskClientTest.java | 1122 ------- .../kafka/KafkaIndexTaskTuningConfigTest.java | 2 - .../druid/indexing/kafka/KafkaSamplerSpecTest.java | 5 + .../supervisor/KafkaSupervisorIOConfigTest.java | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 72 +- .../KafkaSupervisorTuningConfigTest.java | 3 - .../kinesis/KinesisIndexTaskClientFactory.java | 4 +- .../supervisor/KinesisSupervisorIOConfig.java | 3 +- .../supervisor/KinesisSupervisorTuningConfig.java | 34 - .../kinesis/KinesisIndexTaskClientTest.java | 1121 ------- .../kinesis/KinesisIndexTaskTuningConfigTest.java | 2 - .../kinesis/supervisor/KinesisSupervisorTest.java | 48 +- .../KinesisSupervisorTuningConfigTest.java | 3 - .../org/apache/druid/msq/exec/ControllerImpl.java | 2 +- .../druid/msq/exec/TaskDataSegmentProvider.java | 6 +- .../java/org/apache/druid/msq/exec/WorkerImpl.java | 13 +- .../druid/msq/guice/MSQServiceClientModule.java | 86 - .../druid/msq/indexing/IndexerWorkerContext.java | 5 +- .../msq/indexing/report/MSQResultsReport.java | 2 +- .../input/external/ExternalInputSpecSlicer.java | 25 +- .../apache/druid/msq/kernel/StageDefinition.java | 2 +- .../msq/querykit/groupby/GroupByQueryKit.java | 6 +- .../msq/rpc/CoordinatorServiceClientImpl.java | 90 - .../msq/sql/resources/SqlStatementResource.java | 155 +- .../ClusterByStatisticsCollectorImpl.java | 55 +- .../msq/statistics/DelegateOrMinKeyCollector.java | 19 +- .../druid/msq/statistics/DistinctKeyCollector.java | 17 + .../apache/druid/msq/statistics/KeyCollector.java | 10 +- .../statistics/QuantilesSketchKeyCollector.java | 19 +- .../druid/msq/util/SqlStatementResourceHelper.java | 11 +- .../org.apache.druid.initialization.DruidModule | 1 - .../apache/druid/msq/TestArrayStorageAdapter.java | 56 - .../org/apache/druid/msq/exec/MSQInsertTest.java | 38 +- .../org/apache/druid/msq/exec/MSQReplaceTest.java | 3 +- .../org/apache/druid/msq/exec/MSQSelectTest.java | 26 +- .../msq/exec/TaskDataSegmentProviderTest.java | 13 +- .../DataSourceMSQDestinationTest.java | 3 +- .../msq/sql/entity/SqlStatementResultTest.java | 2 +- .../SqlMSQStatementResourcePostTest.java | 172 +- .../{ => resources}/SqlStatementResourceTest.java | 128 +- .../druid/msq/test/MSQTestControllerContext.java | 14 +- extensions-core/orc-extensions/pom.xml | 259 -- extensions-core/parquet-extensions/pom.xml | 251 -- .../jackson/databind/PropertyNamingStrategies.java | 465 +++ .../druid/storage/s3/S3DataSegmentKiller.java | 140 +- .../org/apache/druid/storage/s3/S3TaskLogs.java | 56 +- .../storage/s3/output/S3StorageConnector.java | 202 +- .../druid/storage/s3/S3DataSegmentKillerTest.java | 236 ++ .../apache/druid/storage/s3/S3TaskLogsTest.java | 40 + .../variance/VarianceTopNQueryTest.java | 63 +- .../variance/sql/VarianceSqlAggregatorTest.java | 34 +- indexing-hadoop/pom.xml | 58 - .../druid/indexer/DeterminePartitionsJob.java | 46 +- .../indexer/DeterminePartitionsJobSampler.java | 70 + .../apache/druid/indexer/HadoopTuningConfig.java | 37 +- .../druid/indexer/BatchDeltaIngestionTest.java | 3 +- .../indexer/DetermineHashedPartitionsJobTest.java | 3 +- .../indexer/DeterminePartitionsJobSamplerTest.java | 102 + .../druid/indexer/DeterminePartitionsJobTest.java | 3 +- .../indexer/DetermineRangePartitionsJobTest.java | 3 +- .../indexer/HadoopDruidIndexerConfigTest.java | 3 +- .../druid/indexer/HadoopTuningConfigTest.java | 3 +- .../druid/indexer/IndexGeneratorJobTest.java | 3 +- .../org/apache/druid/indexer/JobHelperTest.java | 3 +- .../indexer/path/GranularityPathSpecTest.java | 3 +- indexing-service/pom.xml | 47 - .../druid/indexing/common/IndexTaskClient.java | 518 ---- .../druid/indexing/common/TaskToolboxFactory.java | 8 +- .../actions/RetrieveUnusedSegmentsAction.java | 20 +- .../druid/indexing/common/task/ArchiveTask.java | 2 +- .../druid/indexing/common/task/CompactionTask.java | 48 +- .../druid/indexing/common/task/IndexTask.java | 31 +- .../common/task/KillUnusedSegmentsTask.java | 158 +- .../druid/indexing/common/task/MoveTask.java | 2 +- .../druid/indexing/common/task/RestoreTask.java | 2 +- .../apache/druid/indexing/common/task/Task.java | 2 +- .../InputSourceSplitParallelIndexTaskRunner.java | 2 +- .../parallel/ParallelIndexSupervisorTask.java | 3 + .../parallel/PartialDimensionCardinalityTask.java | 2 +- .../parallel/PartialDimensionDistributionTask.java | 2 +- .../batch/parallel/PartialSegmentGenerateTask.java | 2 +- .../SinglePhaseParallelIndexTaskRunner.java | 2 +- .../task/batch/parallel/SinglePhaseSubTask.java | 2 +- .../druid/indexing/input/DruidInputSource.java | 141 +- .../druid/indexing/input/TaskInputSource.java | 16 +- .../SeekableStreamIndexTaskClientAsyncImpl.java | 24 +- .../SeekableStreamIndexTaskClientFactory.java | 60 - .../SeekableStreamIndexTaskClientSyncImpl.java | 442 --- .../supervisor/SeekableStreamSupervisor.java | 116 +- .../SeekableStreamSupervisorIOConfig.java | 13 +- .../SeekableStreamSupervisorTuningConfig.java | 6 - .../druid/indexing/common/IndexTaskClientTest.java | 349 --- .../druid/indexing/common/TaskToolboxTest.java | 3 +- .../apache/druid/indexing/common/TestUtils.java | 2 - .../actions/RetrieveSegmentsActionsTest.java | 2 +- .../AppenderatorDriverRealtimeIndexTaskTest.java | 3 +- .../task/ClientCompactionTaskQuerySerdeTest.java | 7 +- ...ClientKillUnusedSegmentsTaskQuerySerdeTest.java | 34 +- .../common/task/CompactionTaskParallelRunTest.java | 28 +- .../common/task/CompactionTaskRunTest.java | 14 +- .../indexing/common/task/CompactionTaskTest.java | 101 +- .../common/task/KillUnusedSegmentsTaskTest.java | 212 +- .../common/task/RealtimeIndexTaskTest.java | 3 +- .../AbstractParallelIndexSupervisorTaskTest.java | 31 +- .../druid/indexing/input/DruidInputSourceTest.java | 6 - .../overlord/SingleTaskBackgroundRunnerTest.java | 3 +- .../druid/indexing/overlord/TaskLifecycleTest.java | 106 +- .../indexing/overlord/TaskRunnerUtilsTest.java | 2 +- .../indexing/overlord/TestTaskToolboxFactory.java | 6 +- .../druid/indexing/overlord/http/OverlordTest.java | 2 +- .../SeekableStreamIndexTaskTestBase.java | 3 +- .../SeekableStreamSamplerSpecTest.java | 3 +- .../SeekableStreamSupervisorSpecTest.java | 18 +- .../SeekableStreamSupervisorStateTest.java | 17 +- .../TestIndexerMetadataStorageCoordinator.java | 24 + .../indexing/worker/WorkerTaskManagerTest.java | 11 +- .../indexing/worker/WorkerTaskMonitorTest.java | 3 +- .../druid/testing/tools/CliCustomNodeRole.java | 3 - integration-tests/README.md | 2 +- integration-tests/build_run_cluster.sh | 5 - .../override-examples/hadoop/azure_to_azure | 2 - .../override-examples/hadoop/azure_to_hdfs | 2 - .../override-examples/hadoop/s3_to_hdfs | 2 - .../override-examples/hadoop/s3_to_s3 | 2 - .../docker/environment-configs/router | 2 +- .../test-groups/custom-coordinator-duties | 4 +- integration-tests/pom.xml | 7 - .../script/copy_resources_template.sh | 13 +- .../org/apache/druid/cli/CliCustomNodeRole.java | 3 - .../apache/druid/tests/query/ITJdbcQueryTest.java | 2 +- licenses.yaml | 541 +--- owasp-dependency-check-suppressions.xml | 61 - pom.xml | 61 +- .../druid/data/input/FilePerSplitHintSpec.java | 53 + .../data/input/impl/CloudObjectInputSource.java | 21 +- .../org/apache/druid/error/DruidException.java | 11 + .../main/java/org/apache/druid/error/NotFound.java | 64 + .../org/apache/druid/frame/field/FieldWriters.java | 5 +- .../druid/frame/field/StringArrayFieldWriter.java | 7 +- .../druid/frame/field/StringFieldReader.java | 99 +- .../druid/frame/field/StringFieldWriter.java | 38 +- .../org/apache/druid/frame/key/RowKeyReader.java | 2 +- .../read/columnar/StringFrameColumnReader.java | 465 +-- .../apache/druid/frame/write/FrameWriterUtils.java | 15 +- .../frame/write/columnar/FrameColumnWriters.java | 9 +- .../write/columnar/StringFrameColumnWriter.java | 87 +- .../druid/guice/GuiceAnnotationIntrospector.java | 82 +- .../org/apache/druid/indexer/TaskLocation.java | 32 +- .../java/org/apache/druid/math/expr/ExprEval.java | 71 +- .../druid/math/expr/ExpressionTypeConversion.java | 2 +- .../java/org/apache/druid/query/DataSource.java | 3 +- .../java/org/apache/druid/query/DruidMetrics.java | 3 + ...nestDataSource.java => FilteredDataSource.java} | 122 +- .../druid/query/FinalizeResultsQueryRunner.java | 12 +- .../org/apache/druid/query/FluentQueryRunner.java | 129 + .../druid/query/FluentQueryRunnerBuilder.java | 106 - .../org/apache/druid/query/QueryToolChest.java | 90 +- .../org/apache/druid/query/UnnestDataSource.java | 10 +- .../druid/query/aggregation/AggregatorUtil.java | 25 +- .../aggregation/SimpleLongAggregatorFactory.java | 1 - .../aggregation/first/StringFirstLastUtils.java | 4 +- .../last/DoubleLastAggregatorFactory.java | 5 +- .../last/FloatLastAggregatorFactory.java | 5 +- .../last/LongLastAggregatorFactory.java | 3 +- .../last/StringLastAggregatorFactory.java | 4 +- .../last/StringLastVectorAggregator.java | 6 +- .../apache/druid/query/filter/EqualityFilter.java | 76 +- .../druid/query/filter/FallbackPredicate.java | 66 + .../org/apache/druid/query/filter/RangeFilter.java | 154 +- .../filter/vector/ArrayVectorValueMatcher.java | 2 +- .../filter/vector/DoubleVectorValueMatcher.java | 13 +- .../filter/vector/FloatVectorValueMatcher.java | 13 +- .../filter/vector/LongVectorValueMatcher.java | 13 +- .../vector/MultiValueStringVectorValueMatcher.java | 11 +- .../filter/vector/ObjectVectorValueMatcher.java | 4 +- .../SingleValueStringVectorValueMatcher.java | 11 +- .../vector/StringObjectVectorValueMatcher.java | 11 +- .../filter/vector/VectorValueMatcherFactory.java | 10 +- .../query/topn/TopNLexicographicResultBuilder.java | 2 +- .../druid/query/topn/TopNNumericResultBuilder.java | 2 +- .../org/apache/druid/query/topn/TopNQuery.java | 6 +- .../druid/query/topn/TopNQueryQueryToolChest.java | 245 +- .../apache/druid/query/topn/TopNResultValue.java | 53 +- .../druid/segment/AutoTypeColumnIndexer.java | 32 +- .../segment/BaseObjectColumnValueSelector.java | 32 + .../apache/druid/segment/DimensionSelector.java | 7 +- .../druid/segment/FilteredStorageAdapter.java | 145 + .../druid/segment/UnnestSegmentReference.java | 120 - .../apache/druid/segment/UnnestStorageAdapter.java | 12 +- .../segment/VectorColumnProcessorFactory.java | 5 + .../druid/segment/WrappedSegmentReference.java | 88 + .../org/apache/druid/segment/column/Types.java | 20 + .../druid/segment/data/FrontCodedIndexed.java | 668 ++--- .../segment/data/FrontCodedIndexedWriter.java | 14 +- .../druid/segment/loading/DataSegmentKiller.java | 24 + .../nested/CompressedNestedDataComplexColumn.java | 19 +- .../nested/NestedFieldColumnIndexSupplier.java | 20 +- .../nested/NestedFieldDictionaryEncodedColumn.java | 211 +- .../nested/ScalarDoubleColumnAndIndexSupplier.java | 12 +- .../nested/ScalarLongColumnAndIndexSupplier.java | 37 +- .../apache/druid/segment/nested/VariantColumn.java | 234 +- .../nested/VariantColumnAndIndexSupplier.java | 31 +- .../serde/NestedCommonFormatColumnPartSerde.java | 12 +- .../vector/VectorColumnSelectorFactory.java | 2 +- .../druid/segment/vector/VectorObjectSelector.java | 30 +- .../druid/segment/virtual/ExpressionPlan.java | 2 +- .../druid/segment/virtual/ExpressionSelectors.java | 9 +- .../segment/virtual/NestedFieldVirtualColumn.java | 117 +- .../druid/storage/StorageConnectorModule.java | 6 +- .../storage/remote/ChunkingStorageConnector.java | 215 ++ .../remote/ChunkingStorageConnectorParameters.java | 259 ++ .../input/impl/CloudObjectInputSourceTest.java | 20 +- .../java/org/apache/druid/error/NotFoundTest.java | 61 + .../druid/frame/TestArrayStorageAdapter.java | 186 +- ...erTest.java => StringArrayFieldWriterTest.java} | 89 +- .../druid/frame/field/StringFieldReaderTest.java | 55 +- .../druid/frame/field/StringFieldWriterTest.java | 13 +- .../apache/druid/frame/key/RowKeyReaderTest.java | 28 +- .../apache/druid/frame/testutil/FrameTestUtil.java | 56 +- .../apache/druid/frame/write/FrameWriterTest.java | 2 - .../druid/frame/write/FrameWriterTestData.java | 28 +- .../org/apache/druid/indexer/TaskLocationTest.java | 24 +- .../org/apache/druid/indexer/TaskStatusTest.java | 2 +- .../druid/java/util/metrics/MetricsVerifier.java | 3 +- .../java/org/apache/druid/math/expr/EvalTest.java | 214 +- .../org/apache/druid/math/expr/ExprEvalTest.java | 316 +- .../apache/druid/query/FilteredDataSourceTest.java | 153 + .../druid/query/MultiValuedDimensionTest.java | 6 +- .../apache/druid/query/QueryRunnerTestHelper.java | 237 +- .../query/groupby/GroupByQueryRunnerTest.java | 2 +- .../groupby/GroupByQueryRunnerTestHelper.java | 1 - .../groupby/GroupByTimeseriesQueryRunnerTest.java | 29 +- .../query/groupby/NestedGroupByArrayQueryTest.java | 7 - .../metadata/SegmentMetadataUnionQueryTest.java | 37 +- .../druid/query/scan/NestedDataScanQueryTest.java | 20 +- .../query/scan/UnnestScanQueryRunnerTest.java | 68 + .../druid/query/search/SearchQueryRunnerTest.java | 7 +- .../timeseries/NestedDataTimeseriesQueryTest.java | 695 +++++ .../timeseries/TimeSeriesUnionQueryRunnerTest.java | 2 +- .../timeseries/TimeseriesQueryRunnerTest.java | 17 +- .../druid/query/topn/TopNBinaryFnBenchmark.java | 4 +- .../apache/druid/query/topn/TopNBinaryFnTest.java | 32 +- .../query/topn/TopNQueryQueryToolChestTest.java | 14 +- .../druid/query/topn/TopNQueryRunnerTest.java | 252 +- .../query/topn/TopNQueryRunnerTestHelper.java | 2 +- .../druid/query/topn/TopNUnionQueryTest.java | 30 +- .../query/topn/UnnestTopNQueryRunnerTest.java | 8 +- .../druid/segment/SchemalessTestFullTest.java | 38 +- .../druid/segment/SchemalessTestSimpleTest.java | 2 +- .../java/org/apache/druid/segment/TestHelper.java | 15 +- .../druid/segment/UnnestStorageAdapterTest.java | 58 + .../druid/segment/filter/BaseFilterTest.java | 20 +- .../druid/segment/filter/EqualityFilterTests.java | 116 + .../druid/segment/filter/RangeFilterTests.java | 352 +++ .../join/table/FrameBasedIndexedTableTest.java | 14 +- .../nested/NestedFieldColumnSelectorsTest.java | 17 +- .../virtual/DummyStringVirtualColumnTest.java | 2 +- .../virtual/ExpressionVectorSelectorsTest.java | 3 +- .../ChunkingStorageConnectorParametersTest.java | 66 + .../remote/ChunkingStorageConnectorTest.java | 87 + .../druid/storage/remote/TestStorageConnector.java | 135 + .../test/resources/nested-all-types-test-data.json | 14 +- .../org/apache/druid/client/BrokerServerView.java | 2 +- .../apache/druid/client/CoordinatorServerView.java | 2 +- .../druid/client/HttpServerInventoryView.java | 4 +- .../client/coordinator/CoordinatorClient.java | 177 +- .../client/coordinator/CoordinatorClientImpl.java | 114 + .../ClientKillUnusedSegmentsTaskQuery.java | 31 +- .../apache/druid/discovery/DruidLeaderClient.java | 6 +- .../IndexerMetadataStorageCoordinator.java | 19 +- .../IndexerSQLMetadataStorageCoordinator.java | 12 +- .../druid/metadata/SQLMetadataConnector.java | 26 +- .../druid/metadata/SqlSegmentsMetadataManager.java | 2 +- .../druid/metadata/SqlSegmentsMetadataQuery.java | 17 +- .../metadata/storage/derby/DerbyConnector.java | 18 + .../org/apache/druid/rpc/StandardRetryPolicy.java | 24 + .../druid/rpc/guice/ServiceClientModule.java | 34 +- .../apache/druid/rpc/indexing/OverlordClient.java | 33 +- .../druid/rpc/indexing/OverlordClientImpl.java | 2 +- .../CoordinatorBasedSegmentHandoffNotifier.java | 54 +- .../segment/loading/OmniDataSegmentKiller.java | 18 + .../druid/server/ClientQuerySegmentWalker.java | 24 +- .../druid/server/LocalQuerySegmentWalker.java | 6 +- .../coordination/SegmentLoadDropHandler.java | 17 +- .../coordinator/CoordinatorDynamicConfig.java | 67 + .../druid/server/coordinator/DruidCluster.java | 9 +- .../druid/server/coordinator/DruidCoordinator.java | 12 +- .../coordinator/DruidCoordinatorRuntimeParams.java | 20 - .../druid/server/coordinator/ServerHolder.java | 5 + .../duty/CollectSegmentAndServerStats.java | 4 - .../server/coordinator/duty/CompactSegments.java | 73 +- .../coordinator/duty/CoordinatorDutyUtils.java | 129 + .../server/coordinator/duty/KillAuditLog.java | 57 +- .../coordinator/duty/KillCompactionConfig.java | 160 +- .../coordinator/duty/KillDatasourceMetadata.java | 92 +- .../druid/server/coordinator/duty/KillRules.java | 62 +- .../server/coordinator/duty/KillSupervisors.java | 61 +- .../duty/KillSupervisorsCustomDuty.java | 72 +- .../coordinator/duty/KillUnusedSegments.java | 156 +- .../duty/MarkOvershadowedSegmentsAsUnused.java | 17 +- .../coordinator/duty/MetadataCleanupDuty.java | 135 + .../druid/server/coordinator/duty/RunRules.java | 88 +- .../coordinator/duty/SegmentDeleteHandler.java | 12 +- .../coordinator/duty/UnloadUnusedSegments.java | 96 +- .../loading/StrategicSegmentAssigner.java | 71 +- .../druid/server/coordinator/stats/Stats.java | 20 + .../druid/server/http/DataSourcesResource.java | 2 +- .../apache/druid/server/http/MetadataResource.java | 7 +- .../druid/server/metrics/ServiceStatusMonitor.java | 10 +- .../metrics/WorkerTaskCountStatsMonitor.java | 5 +- .../druid/client/CachingClusteredClientTest.java | 44 +- .../druid/client/CachingQueryRunnerTest.java | 2 +- .../druid/client/HttpServerInventoryViewTest.java | 4 +- .../client/cache/BackgroundCachePopulatorTest.java | 3 +- .../apache/druid/client/cache/CacheConfigTest.java | 8 +- .../coordinator/CoordinatorClientImplTest.java | 146 + .../client/coordinator/NoopCoordinatorClient.java | 42 +- .../ClientKillUnusedSegmentsTaskQueryTest.java | 25 +- .../druid/discovery/DruidLeaderClientTest.java | 4 +- .../IndexerSQLMetadataStorageCoordinatorTest.java | 16 + .../druid/rpc/indexing/OverlordClientImplTest.java | 13 +- ...CoordinatorBasedSegmentHandoffNotifierTest.java | 189 +- .../segment/loading/OmniDataSegmentKillerTest.java | 45 + .../apache/druid/server/QuerySchedulerTest.java | 23 +- .../coordination/SegmentLoadDropHandlerTest.java | 57 +- .../coordinator/BalanceSegmentsProfiler.java | 12 +- .../druid/server/coordinator/DruidClusterTest.java | 10 +- .../server/coordinator/DruidCoordinatorTest.java | 6 +- .../druid/server/coordinator/ServerHolderTest.java | 23 +- .../coordinator/balancer/BalancerStrategyTest.java | 10 +- .../balancer/CachingCostBalancerStrategyTest.java | 4 +- .../balancer/CostBalancerStrategyTest.java | 10 +- .../DiskNormalizedCostBalancerStrategyTest.java | 6 +- .../balancer/ReservoirSegmentSamplerTest.java | 6 +- .../coordinator/duty/BalanceSegmentsTest.java | 4 +- .../server/coordinator/duty/KillAuditLogTest.java | 49 +- .../coordinator/duty/KillCompactionConfigTest.java | 88 +- .../duty/KillDatasourceMetadataTest.java | 72 +- .../server/coordinator/duty/KillRulesTest.java | 47 +- .../duty/KillSupervisorsCustomDutyTest.java | 63 +- .../coordinator/duty/KillSupervisorsTest.java | 49 +- .../coordinator/duty/KillUnusedSegmentsTest.java | 138 +- .../duty/MarkOvershadowedSegmentsAsUnusedTest.java | 94 +- .../server/coordinator/duty/RunRulesTest.java | 58 +- .../coordinator/duty/UnloadUnusedSegmentsTest.java | 18 +- .../coordinator/loading/LoadQueuePeonTester.java | 61 - .../RoundRobinServerSelectorTest.java | 8 +- .../coordinator/loading/TestLoadQueuePeon.java | 124 + .../rules/BroadcastDistributionRuleTest.java | 526 ++-- .../server/coordinator/rules/LoadRuleTest.java | 372 +-- .../simulate/CoordinatorSimulation.java | 6 + .../simulate/CoordinatorSimulationBaseTest.java | 20 + .../simulate/CoordinatorSimulationBuilder.java | 11 + .../coordinator/simulate/SegmentLoadingTest.java | 27 + .../simulate/TestSegmentsMetadataManager.java | 11 +- .../server/http/CoordinatorDynamicConfigTest.java | 88 + .../druid/server/http/DataSourcesResourceTest.java | 2 +- .../druid/server/http/MetadataResourceTest.java | 4 +- .../java/org/apache/druid/cli/CliCoordinator.java | 3 +- .../main/java/org/apache/druid/cli/CliIndexer.java | 2 +- .../org/apache/druid/cli/CliMiddleManager.java | 18 + .../java/org/apache/druid/cli/CliOverlord.java | 3 +- .../main/java/org/apache/druid/cli/CliPeon.java | 28 +- .../org/apache/druid/cli/PullDependencies.java | 2 +- sql/pom.xml | 17 +- sql/src/main/codegen/config.fmpp | 375 +-- sql/src/main/codegen/includes/common.ftl | 10 +- sql/src/main/codegen/includes/from.ftl | 605 ++-- sql/src/main/codegen/includes/insert.ftl | 3 +- sql/src/main/codegen/includes/replace.ftl | 5 +- .../calcite/prepare/BaseDruidSqlValidator.java | 6 +- .../sql/avatica/AbstractDruidJdbcStatement.java | 25 +- .../builtin/EarliestLatestBySqlAggregator.java | 27 +- .../aggregation/builtin/LiteralSqlAggregator.java | 87 + .../aggregation/builtin/StringSqlAggregator.java | 33 +- .../aggregation/builtin/SumSqlAggregator.java | 89 +- .../expression/BasicOperandTypeChecker.java | 229 ++ .../druid/sql/calcite/expression/Expressions.java | 3 +- .../calcite/expression/OperatorConversions.java | 31 +- .../builtin/NestedDataOperatorConversions.java | 116 +- .../builtin/SearchOperatorConversion.java | 89 + .../external/BaseUserDefinedTableMacro.java | 124 +- .../sql/calcite/external/DruidTableMacro.java | 18 +- .../external/DruidUserDefinedTableMacro.java | 1 - .../druid/sql/calcite/external/Externals.java | 45 +- .../external/SchemaAwareUserDefinedTableMacro.java | 28 +- .../sql/calcite/parser/DruidSqlParserUtils.java | 55 +- .../druid/sql/calcite/planner/CalcitePlanner.java | 295 +- .../sql/calcite/planner/CalciteRulesManager.java | 227 +- .../apache/druid/sql/calcite/planner/Calcites.java | 23 +- .../sql/calcite/planner/DruidOperatorTable.java | 21 +- .../druid/sql/calcite/planner/DruidPlanner.java | 63 +- .../sql/calcite/planner/DruidSqlValidator.java | 6 +- .../druid/sql/calcite/planner/DruidTypeSystem.java | 18 +- .../druid/sql/calcite/planner/PlannerContext.java | 8 + .../druid/sql/calcite/planner/PlannerFactory.java | 36 +- .../druid/sql/calcite/planner/QueryHandler.java | 2 +- .../calcite/planner/RelParameterizerShuttle.java | 14 + .../calcite/planner/SqlParameterizerShuttle.java | 2 +- .../sql/calcite/rel/DruidCorrelateUnnestRel.java | 204 +- .../druid/sql/calcite/rel/DruidJoinQueryRel.java | 6 - .../apache/druid/sql/calcite/rel/DruidQuery.java | 8 +- .../druid/sql/calcite/rel/DruidUnnestRel.java | 19 +- .../org/apache/druid/sql/calcite/rel/Grouping.java | 2 +- .../druid/sql/calcite/rel/PartialDruidQuery.java | 1 + .../apache/druid/sql/calcite/rel/Projection.java | 6 +- .../rule/CorrelateFilterLTransposeRule.java | 67 - .../sql/calcite/rule/DruidCorrelateUnnestRule.java | 119 +- .../sql/calcite/rule/DruidFilterUnnestRule.java | 6 +- .../druid/sql/calcite/rule/DruidJoinRule.java | 2 +- .../sql/calcite/rule/DruidRelToDruidRule.java | 6 +- .../apache/druid/sql/calcite/rule/DruidRules.java | 5 +- .../rule/FilterJoinExcludePushToChildRule.java | 40 +- .../druid/sql/calcite/rule/GroupByRules.java | 7 +- .../rule/ProjectAggregatePruneUnusedCallRule.java | 6 +- .../druid/sql/calcite/rule/SortCollapseRule.java | 2 +- .../logical/DruidAggregateCaseToFilterRule.java | 2 +- .../sql/calcite/schema/SegmentMetadataCache.java | 6 +- .../druid/sql/calcite/table/DatasourceTable.java | 2 +- .../apache/druid/sql/calcite/table/DruidTable.java | 35 + .../druid/sql/calcite/table/InlineTable.java | 2 +- .../druid/sql/calcite/table/LookupTable.java | 2 +- .../druid/sql/calcite/view/DruidViewMacro.java | 2 +- .../org/apache/druid/sql/http/ResultFormat.java | 2 + .../java/org/apache/druid/sql/http/SqlQuery.java | 2 +- .../org/apache/druid/sql/SqlStatementTest.java | 6 +- .../druid/sql/avatica/DruidAvaticaHandlerTest.java | 12 +- .../druid/sql/avatica/DruidStatementTest.java | 8 +- .../druid/sql/calcite/CalciteArraysQueryTest.java | 340 ++- .../druid/sql/calcite/CalciteJoinQueryTest.java | 154 +- .../sql/calcite/CalciteNestedDataQueryTest.java | 126 +- .../sql/calcite/CalciteParameterQueryTest.java | 12 +- .../apache/druid/sql/calcite/CalciteQueryTest.java | 343 ++- .../druid/sql/calcite/CalciteReplaceDmlTest.java | 5 +- .../druid/sql/calcite/CalciteWindowQueryTest.java | 8 + .../calcite/DecoupledPlanningCalciteQueryTest.java | 30 + .../expression/OperatorConversionsTest.java | 15 +- .../calcite/parser/DruidSqlParserUtilsTest.java | 207 +- .../sql/calcite/parser/DruidSqlUnparseTest.java | 12 +- .../calcite/schema/SegmentMetadataCacheTest.java | 4 +- .../org/apache/druid/sql/http/SqlResourceTest.java | 2 +- ...mExternalWithoutSecuritySupport-logicalPlan.txt | 2 +- .../expected/ingest/httpExtern-logicalPlan.txt | 2 +- .../ingest/insertFromExternal-logicalPlan.txt | 2 +- .../ingest/insertWithClusteredBy-logicalPlan.txt | 2 +- .../ingest/insertWithPartitionedBy-logicalPlan.txt | 2 +- .../expected/ingest/localExtern-logicalPlan.txt | 2 +- .../calcite/tests/window/simpleSum.sqlTest | 2 +- .../wikipediaAggregationsMultipleOrdering.sqlTest | 2 +- .../window/wikipediaCumulativeOrdered.sqlTest | 7 +- .../window/wikipediaFramedAggregations.sqlTest | 2 +- .../tests/window/wikipediaSimplePartition.sqlTest | 32 +- .../wikipediaSimplePartitionInitialSort.sqlTest | 16 +- web-console/package-lock.json | 12 +- web-console/src/bootstrap/react-table-defaults.tsx | 7 +- .../src/components/json-input/json-input.tsx | 14 +- .../src/components/menu-checkbox/menu-checkbox.tsx | 5 +- .../src/components/rule-editor/rule-editor.tsx | 43 +- .../segment-timeline/segment-timeline.tsx | 3 +- .../compaction-config-dialog.tsx | 15 +- .../coordinator-dynamic-config-dialog.tsx | 15 +- .../index-spec-dialog/index-spec-dialog.tsx | 17 +- .../lookup-edit-dialog/lookup-edit-dialog.tsx | 15 +- .../overlord-dynamic-config-dialog.tsx | 15 +- .../__snapshots__/retention-dialog.spec.tsx.snap | 326 +- .../dialogs/retention-dialog/retention-dialog.scss | 4 - .../dialogs/retention-dialog/retention-dialog.tsx | 57 +- .../src/druid-models/async-query/async-query.ts | 8 +- .../execution/execution-ingest-complete.mock.ts | 240 +- .../execution/execution-ingest-error.mock.ts | 48 +- .../src/druid-models/execution/execution.spec.ts | 62 +- .../src/druid-models/execution/execution.ts | 145 +- .../druid-models/ingestion-spec/ingestion-spec.tsx | 9 - .../druid-models/query-context/query-context.tsx | 3 +- web-console/src/druid-models/task/task.ts | 10 + .../workbench-query/workbench-query.ts | 2 +- .../src/helpers/execution/sql-task-execution.ts | 33 +- web-console/src/react-table/react-table-utils.ts | 2 +- .../src/utils/download-query-detail-archive.ts | 20 +- web-console/src/utils/download.ts | 17 + web-console/src/utils/druid-query.ts | 2 +- web-console/src/utils/general.tsx | 15 +- web-console/src/views/tasks-view/tasks-view.tsx | 16 +- .../__snapshots__/column-tree.spec.tsx.snap | 24 +- .../__snapshots__/complex-menu-items.spec.tsx.snap | 74 + .../complex-menu-items.spec.tsx} | 40 +- .../complex-menu-items/complex-menu-items.tsx | 83 + .../column-tree/column-tree-menu/index.ts | 1 + .../number-menu-items/number-menu-items.tsx | 12 +- .../string-menu-items/string-menu-items.tsx | 14 +- .../time-menu-items/time-menu-items.tsx | 11 +- .../workbench-view/column-tree/column-tree.tsx | 29 +- .../destination-pages-dialog.scss} | 11 +- .../destination-pages-dialog.tsx} | 35 +- .../destination-pages-pane.tsx | 190 ++ .../execution-details-pane.spec.tsx.snap | 247 +- .../execution-details-pane.tsx | 29 +- .../execution-error-pane.spec.tsx.snap | 2 +- .../execution-stages-pane.spec.tsx.snap | 17 +- .../execution-submit-dialog.tsx | 15 +- .../execution-summary-panel.tsx | 70 +- .../flexible-query-input/flexible-query-input.tsx | 3 +- .../ingest-success-pane/ingest-success-pane.tsx | 2 +- .../recent-query-task-panel.tsx | 6 +- .../views/workbench-view/run-panel/run-panel.tsx | 80 +- website/.spelling | 22 +- website/package-lock.json | 124 +- website/sidebars.json | 3 + 674 files changed, 26684 insertions(+), 16741 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
