This is an automated email from the ASF dual-hosted git repository. wave pushed a commit to branch dave2wave-branch-14693 in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 9d8ec43ac5e657243f2ec0cfadbf52c4ce91f9e8 Merge: 42945e5 b80680e Author: Dave Fisher <[email protected]> AuthorDate: Fri Feb 11 13:39:49 2022 -0800 Merge branch 'master' of https://github.com/apache/pulsar .github/ISSUE_TEMPLATE/flaky_test.md | 13 +- .../ci-integration-backwards-compatibility.yaml | 1 + .github/workflows/ci-integration-cli.yaml | 1 + .github/workflows/ci-integration-function.yaml | 1 + .github/workflows/ci-integration-messaging.yaml | 1 + .github/workflows/ci-integration-process.yaml | 1 + .../workflows/ci-integration-pulsar-io-ora.yaml | 1 + .github/workflows/ci-integration-pulsar-io.yaml | 1 + .github/workflows/ci-integration-schema.yaml | 1 + .github/workflows/ci-integration-sql.yaml | 1 + .github/workflows/ci-integration-standalone.yaml | 1 + .github/workflows/ci-integration-thread.yaml | 1 + .../ci-integration-tiered-filesystem.yaml | 1 + .../workflows/ci-integration-tiered-jcloud.yaml | 1 + .github/workflows/ci-integration-transaction.yaml | 1 + .github/workflows/ci-shade-test.yaml | 1 + README.md | 2 +- SECURITY.md | 3 + buildtools/pom.xml | 19 + conf/broker.conf | 46 +- conf/functions_worker.yml | 27 +- conf/proxy.conf | 35 +- conf/standalone.conf | 48 +- conf/websocket.conf | 17 +- distribution/server/src/assemble/LICENSE.bin.txt | 4 +- docker/pulsar/Dockerfile | 16 +- docker/pulsar/pom.xml | 1 + docker/pulsar/scripts/pulsar-zookeeper-ruok.sh | 2 +- .../apache/bookkeeper/mledger/ManagedLedger.java | 6 + .../bookkeeper/mledger/ManagedLedgerConfig.java | 16 + .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 20 +- .../mledger/impl/ManagedLedgerOfflineBacklog.java | 4 +- .../bookkeeper/mledger/offload/OffloaderUtils.java | 16 +- .../bookkeeper/mledger/impl/ManagedLedgerTest.java | 25 + .../mledger/offload/OffloadersCacheTest.java | 37 +- pom.xml | 28 +- .../ProxySaslAuthenticationTest.java | 1 + .../apache/pulsar/broker/ServiceConfiguration.java | 88 +- .../broker/authorization/AuthorizationService.java | 28 +- .../authorization/PulsarAuthorizationProvider.java | 2 +- .../resources/DynamicConfigurationResources.java | 10 + .../broker/resources/NamespaceResources.java | 15 +- .../web/plugin/servlet/AdditionalServletUtils.java | 19 +- .../plugin/servlet/AdditionalServletUtilsTest.java | 71 +- .../PulsarConfigurationLoaderTest.java | 67 ++ .../apache/pulsar/PulsarClusterMetadataSetup.java | 19 +- .../org/apache/pulsar/PulsarStandaloneStarter.java | 12 +- .../org/apache/pulsar/broker/PulsarService.java | 28 +- .../apache/pulsar/broker/admin/AdminResource.java | 52 +- .../pulsar/broker/admin/impl/BrokersBase.java | 302 +++-- .../pulsar/broker/admin/impl/NamespacesBase.java | 6 +- .../broker/admin/impl/PersistentTopicsBase.java | 400 ++++--- .../broker/admin/impl/ResourceQuotasBase.java | 10 +- .../broker/admin/v1/NonPersistentTopics.java | 11 +- .../pulsar/broker/admin/v1/PersistentTopics.java | 60 +- .../broker/admin/v2/NonPersistentTopics.java | 10 +- .../pulsar/broker/admin/v2/PersistentTopics.java | 127 ++- .../pulsar/broker/admin/v3/PersistentTopics.java | 93 -- .../broker/intercept/BrokerInterceptorUtils.java | 18 +- .../pulsar/broker/lookup/TopicLookupBase.java | 9 +- .../pulsar/broker/namespace/NamespaceService.java | 39 +- .../pulsar/broker/namespace/OwnedBundle.java | 8 +- .../pulsar/broker/namespace/OwnershipCache.java | 3 +- .../broker/protocol/ProtocolHandlerUtils.java | 18 +- .../broker/service/AbstractBaseDispatcher.java | 4 + .../pulsar/broker/service/BacklogQuotaManager.java | 20 +- .../pulsar/broker/service/BrokerService.java | 36 +- .../org/apache/pulsar/broker/service/Consumer.java | 22 +- ...ashRangeAutoSplitStickyKeyConsumerSelector.java | 4 +- .../org/apache/pulsar/broker/service/Producer.java | 17 +- .../apache/pulsar/broker/service/PulsarStats.java | 2 + .../apache/pulsar/broker/service/ServerCnx.java | 12 +- .../service/nonpersistent/NonPersistentTopic.java | 2 +- ...istentStickyKeyDispatcherMultipleConsumers.java | 2 +- .../service/persistent/PersistentSubscription.java | 83 -- .../broker/service/persistent/PersistentTopic.java | 12 +- .../broker/service/plugin/EntryFilterProvider.java | 19 +- .../buffer/impl/TopicTransactionBuffer.java | 38 +- .../buffer/impl/TransactionBufferHandlerImpl.java | 32 - .../pendingack/impl/PendingAckHandleImpl.java | 11 +- .../pulsar/broker/web/PulsarWebResource.java | 79 +- .../common/naming/NamespaceBundleFactory.java | 24 +- .../apache/pulsar/compaction/CompactorTool.java | 2 +- .../broker/BookKeeperClientFactoryImplTest.java | 84 +- .../apache/pulsar/broker/admin/AdminApi2Test.java | 118 +- .../broker/admin/AdminApiMultiBrokersTest.java | 54 + .../org/apache/pulsar/broker/admin/AdminTest.java | 2 +- .../pulsar/broker/admin/PersistentTopicsTest.java | 30 +- .../intercept/BrokerInterceptorUtilsTest.java | 70 +- .../broker/namespace/NamespaceServiceTest.java | 16 +- .../broker/protocol/ProtocolHandlerUtilsTest.java | 102 +- .../broker/service/BacklogQuotaManagerTest.java | 10 +- .../pulsar/broker/service/BatchMessageTest.java | 92 ++ .../BatchMessageWithBatchIndexLevelTest.java | 70 +- .../pulsar/broker/service/BkEnsemblesTestBase.java | 2 +- .../pulsar/broker/service/BrokerServiceTest.java | 40 + .../PersistentDispatcherFailoverConsumerTest.java | 5 +- .../pulsar/broker/service/PersistentTopicTest.java | 39 +- .../pulsar/broker/service/ReplicatorTest.java | 1 - .../pulsar/broker/service/ServerCnxTest.java | 5 +- .../service/TransactionMarkerDeleteTest.java | 183 ++- ...ntStickyKeyDispatcherMultipleConsumersTest.java | 36 +- ...ntStickyKeyDispatcherMultipleConsumersTest.java | 100 +- .../persistent/PersistentSubscriptionTest.java | 5 +- .../pulsar/broker/transaction/TransactionTest.java | 25 + .../broker/transaction/TransactionTestBase.java | 3 +- .../buffer/TransactionBufferClientTest.java | 11 - .../buffer/TransactionStablePositionTest.java | 96 +- .../pendingack/PendingAckInMemoryDeleteTest.java | 1 + .../AdditionalServletWithPulsarServiceTest.java | 43 +- .../broker/zookeeper/ClusterMetadataSetupTest.java | 190 ++- .../MultiBrokerMetadataConsistencyTest.java | 82 ++ .../api/AuthorizationProducerConsumerTest.java | 89 ++ .../pulsar/client/api/BrokerServiceLookupTest.java | 5 +- .../client/api/ConsumerBatchReceiveTest.java | 22 + ...MultiListenersWithInternalListenerNameTest.java | 34 +- .../client/api/SimpleProducerConsumerTest.java | 12 +- .../pulsar/client/impl/MessageChunkingTest.java | 2 +- .../pulsar/client/impl/PulsarTestClient.java | 6 +- .../apache/pulsar/client/impl/TableViewTest.java | 161 +++ .../client/impl/TransactionEndToEndTest.java | 11 +- ...ransactionEndToEndWithoutBatchIndexAckTest.java | 46 + .../common/naming/ServiceConfigurationTest.java | 10 +- .../worker/PulsarFunctionLocalRunTest.java | 7 +- .../pulsar_broker_test_standalone.conf | 4 +- .../org/apache/pulsar/client/admin/Brokers.java | 9 + .../org/apache/pulsar/client/admin/Topics.java | 69 ++ .../org/apache/pulsar/common/io/SinkConfig.java | 1 + .../common/partition/PartitionedTopicMetadata.java | 1 + .../common/policies/data/PublisherStats.java | 3 + .../pulsar/client/admin/internal/BrokersImpl.java | 9 + .../pulsar/client/admin/internal/TopicsImpl.java | 46 +- .../apache/pulsar/client/api/ConsumerBuilder.java | 4 +- .../apache/pulsar/admin/cli/CmdFunctionsTest.java | 24 - .../pulsar/admin/cli/PulsarAdminToolTest.java | 11 +- .../org/apache/pulsar/admin/cli/CmdBrokers.java | 20 + .../java/org/apache/pulsar/admin/cli/CmdSinks.java | 7 + .../org/apache/pulsar/admin/cli/CmdTopics.java | 15 +- .../org/apache/pulsar/admin/cli/TestCmdSinks.java | 18 +- .../apache/pulsar/admin/cli/TestCmdSources.java | 19 +- .../org/apache/pulsar/admin/cli/TestCmdTopics.java | 18 +- .../pulsar/client/impl/ConsumerBuilderImpl.java | 1 - .../apache/pulsar/client/impl/MessagesImpl.java | 4 + .../client/impl/PartitionedProducerImpl.java | 86 +- .../pulsar/client/impl/ProducerBuilderImpl.java | 1 - .../apache/pulsar/client/impl/ProducerImpl.java | 6 +- .../pulsar/client/impl/PulsarClientImpl.java | 8 +- .../client/impl/TransactionMetaStoreHandler.java | 20 +- .../impl/transaction/TransactionBuilderImpl.java | 4 +- .../client/impl/transaction/TransactionImpl.java | 6 + .../client/impl/BinaryProtoLookupServiceTest.java | 12 +- .../client/impl/ConsumerBuilderImplTest.java | 2 +- .../client/impl/PartitionedProducerImplTest.java | 7 +- .../client/impl/ProducerBuilderImplTest.java | 2 +- .../apache/pulsar/common/nar/NarClassLoader.java | 18 +- .../pulsar/common/nar/NarClassLoaderBuilder.java | 75 ++ .../common/policies/data/TopicOperation.java | 3 - .../data/stats/NonPersistentTopicStatsImpl.java | 71 +- .../policies/data/stats/PublisherStatsImpl.java | 3 + .../common/policies/data/stats/TopicStatsImpl.java | 67 +- .../apache/pulsar/common/protocol/Commands.java | 1 + .../pulsar/common/protocol/PulsarHandler.java | 2 +- .../util/NettyClientSslContextRefresher.java | 10 +- .../ConcurrentOpenLongPairRangeSet.java | 2 +- pulsar-common/src/main/proto/PulsarApi.proto | 1 + .../policies/data/PartitionedTopicStatsTest.java | 4 +- .../policies/data/PersistentTopicStatsTest.java | 137 ++- .../common/policies/data/PublisherStatsTest.java | 3 +- .../common/util/FileModifiedTimeUpdaterTest.java | 53 + .../functions/runtime/thread/ThreadRuntime.java | 10 +- .../pulsar/functions/worker/WorkerConfig.java | 62 +- .../pulsar/functions/config/TestWorkerConfig.java | 71 ++ .../runtime/thread/ThreadRuntimeFactoryTest.java | 90 +- .../EnvironmentBasedSecretsProviderTest.java | 37 +- .../pulsar/functions/utils/FunctionCommon.java | 7 +- .../pulsar/functions/utils/SinkConfigUtils.java | 62 +- .../pulsar/functions/utils/SourceConfigUtils.java | 15 +- .../utils/functioncache/FunctionCacheEntry.java | 10 +- .../functions/utils/functions/FunctionUtils.java | 5 +- .../pulsar/functions/utils/io/ConnectorUtils.java | 6 +- .../functions/utils/SinkConfigUtilsTest.java | 74 +- .../functions/utils/SourceConfigUtilsTest.java | 33 +- .../functions/worker/FunctionRuntimeManager.java | 6 +- .../org/apache/pulsar/functions/worker/Worker.java | 2 +- .../pulsar/functions/worker/WorkerUtils.java | 37 +- .../functions/worker/rest/api/ComponentImpl.java | 12 +- .../worker/service/WorkerServiceLoader.java | 18 +- .../worker/ClusterServiceCoordinatorTest.java | 22 +- .../worker/FunctionRuntimeManagerTest.java | 1206 ++++++++++---------- .../worker/rest/api/FunctionsImplTest.java | 10 - .../rest/api/v2/FunctionApiV2ResourceTest.java | 208 ++-- .../rest/api/v3/FunctionApiV3ResourceTest.java | 193 ++-- .../worker/rest/api/v3/SinkApiV3ResourceTest.java | 459 ++++---- .../rest/api/v3/SourceApiV3ResourceTest.java | 367 +++--- pulsar-io/data-generator/pom.xml | 4 +- pulsar-io/debezium/postgres/pom.xml | 7 + pulsar-io/flume/pom.xml | 6 - .../flume/node/TestEnvVarResolverProperties.java | 32 +- .../bookkeeper/AbstractMetadataDriver.java | 3 +- .../bookkeeper/PulsarLedgerIdGenerator.java | 19 +- .../metadata/cache/impl/MetadataCacheImpl.java | 1 + .../metadata/impl/AbstractMetadataStore.java | 2 + .../metadata/impl/MetadataStoreFactoryImpl.java | 22 + .../pulsar/metadata/impl/ZKMetadataStore.java | 24 +- .../pulsar/metadata/BaseMetadataStoreTest.java | 22 +- .../apache/pulsar/metadata/LockManagerTest.java | 3 +- .../org/apache/pulsar/metadata/TestZKServer.java | 1 + .../org/apache/pulsar/metadata/ZKSessionTest.java | 4 +- pulsar-proxy/pom.xml | 17 +- .../proxy/extensions/ProxyExtensionsUtils.java | 18 +- .../proxy/server/BrokerDiscoveryProvider.java | 2 +- .../pulsar/proxy/server/BrokerProxyValidator.java | 181 +++ .../pulsar/proxy/server/DirectProxyHandler.java | 156 +-- .../pulsar/proxy/server/ProxyConfiguration.java | 127 ++- .../pulsar/proxy/server/ProxyConnection.java | 99 +- .../apache/pulsar/proxy/server/ProxyService.java | 27 +- .../pulsar/proxy/server/ProxyServiceStarter.java | 45 +- .../proxy/server/ServiceChannelInitializer.java | 8 + .../proxy/server/TargetAddressDeniedException.java | 44 +- .../proxy/extensions/ProxyExtensionUtilsTest.java | 92 +- .../extensions/SimpleProxyExtensionTestBase.java | 5 +- .../proxy/server/AuthedAdminProxyHandlerTest.java | 1 + .../proxy/server/BrokerProxyValidatorTest.java | 102 ++ .../proxy/server/ProxyAdditionalServletTest.java | 5 +- .../ProxyAuthenticatedProducerConsumerTest.java | 5 +- .../proxy/server/ProxyAuthenticationTest.java | 1 + .../proxy/server/ProxyConfigurationTest.java | 56 + .../pulsar/proxy/server/ProxyConnectionTest.java | 60 +- .../server/ProxyConnectionThrottlingTest.java | 5 +- .../server/ProxyEnableHAProxyProtocolTest.java | 5 +- .../proxy/server/ProxyForwardAuthDataTest.java | 1 + .../proxy/server/ProxyKeyStoreTlsTestWithAuth.java | 5 +- .../server/ProxyKeyStoreTlsTestWithoutAuth.java | 5 +- .../proxy/server/ProxyLookupThrottlingTest.java | 5 +- .../pulsar/proxy/server/ProxyParserTest.java | 5 +- .../proxy/server/ProxyRolesEnforcementTest.java | 1 + .../proxy/server/ProxyServiceStarterTest.java | 5 +- .../proxy/server/ProxyServiceTlsStarterTest.java | 15 +- .../apache/pulsar/proxy/server/ProxyStatsTest.java | 5 +- .../org/apache/pulsar/proxy/server/ProxyTest.java | 5 +- .../apache/pulsar/proxy/server/ProxyTlsTest.java | 5 +- .../pulsar/proxy/server/ProxyTlsTestWithAuth.java | 5 +- .../server/ProxyWithAuthorizationNegTest.java | 1 + .../proxy/server/ProxyWithAuthorizationTest.java | 2 + .../server/ProxyWithJwtAuthorizationTest.java | 1 + .../server/ProxyWithoutServiceDiscoveryTest.java | 1 + .../SuperUserAuthedAdminProxyHandlerTest.java | 1 + .../server/UnauthedAdminProxyHandlerTest.java | 5 +- pulsar-proxy/src/test/resources/proxy.conf | 18 +- pulsar-sql/presto-distribution/LICENSE | 2 +- pulsar-testclient/pom.xml | 6 + .../pulsar/testclient/ManagedLedgerWriter.java | 24 +- .../pulsar/testclient/PerformanceProducerTest.java | 51 +- .../apache/pulsar/websocket/WebSocketService.java | 2 +- .../service/WebSocketProxyConfiguration.java | 32 +- .../websocket/WebSocketProxyConfigurationTest.java | 90 ++ site2/docs/admin-api-clusters.md | 4 +- site2/docs/administration-proxy.md | 30 +- site2/docs/client-libraries-cpp.md | 225 ++-- site2/docs/client-libraries-dotnet.md | 5 +- site2/docs/client-libraries-websocket.md | 4 +- site2/docs/concepts-architecture-overview.md | 5 +- site2/docs/deploy-bare-metal-multi-cluster.md | 10 +- site2/docs/deploy-bare-metal.md | 10 +- site2/docs/developing-binary-protocol.md | 7 +- site2/docs/reference-cli-tools.md | 10 +- site2/docs/reference-configuration.md | 91 +- site2/docs/security-tls-keystore.md | 1 - site2/docs/security-versioning-policy.md | 67 ++ site2/website-next/docs/administration-proxy.md | 5 +- .../docs/concepts-architecture-overview.md | 6 +- site2/website-next/docs/reference-cli-tools.md | 8 +- .../docs/security-versioning-policy.md | 67 ++ site2/website-next/sidebars.json | 1 + site2/website/sidebars.json | 1 + tests/bc_2_0_0/pom.xml | 1 + tests/bc_2_0_1/pom.xml | 1 + tests/bc_2_6_0/pom.xml | 1 + tests/docker-images/java-test-image/Dockerfile | 6 +- tests/docker-images/java-test-image/pom.xml | 1 + .../docker-images/latest-version-image/Dockerfile | 2 +- tests/integration/pom.xml | 8 +- .../io/sources/AvroKafkaSourceTest.java | 39 +- tests/pulsar-client-admin-shade-test/pom.xml | 1 + tests/pulsar-client-all-shade-test/pom.xml | 1 + tests/pulsar-client-shade-test/pom.xml | 1 + tiered-storage/jcloud/pom.xml | 4 +- .../offload/jcloud/impl/MockManagedLedger.java | 5 + 288 files changed, 6583 insertions(+), 3730 deletions(-)
