This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push: new 0f970e3 Removed internal shading (#1816) 0f970e3 is described below commit 0f970e3055dc1ec129da1ccc4e3155415be94724 Author: Matteo Merli <mme...@apache.org> AuthorDate: Wed May 23 21:43:44 2018 -0700 Removed internal shading (#1816) * Removed internal shading * Fixed handling of IllegalArgumentException in ZK client wrapper * Fixed tests with real ZK server --- all/pom.xml | 12 +- all/src/assemble/bin.xml | 3 - bin/pulsar | 10 +- managed-ledger/pom.xml | 2 +- pom.xml | 13 +- pulsar-broker/pom.xml | 6 - .../pulsar/broker/lookup/TopicLookupBase.java | 19 +-- .../s3offload/impl/OffloadIndexBlockImpl.java | 5 +- .../client/api/MessageDispatchThrottlingTest.java | 4 +- pulsar-client-admin-shaded-for-functions/pom.xml | 180 --------------------- pulsar-client-admin/pom.xml | 10 +- .../org/apache/pulsar/client/admin/Functions.java | 8 +- .../client/admin/internal/FunctionsImpl.java | 42 ++--- .../apache/pulsar/admin/cli/CmdFunctionsTest.java | 11 +- .../org/apache/pulsar/admin/cli/CmdFunctions.java | 68 ++++---- .../java/org/apache/pulsar/admin/cli/CmdSinks.java | 39 +++-- .../org/apache/pulsar/admin/cli/CmdSources.java | 36 ++--- pulsar-functions/instance/pom.xml | 12 ++ .../pulsar/functions/sink/PulsarSinkTest.java | 2 +- pulsar-functions/pom.xml | 1 - pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime-shaded/pom.xml | 7 +- pulsar-functions/runtime/pom.xml | 1 - pulsar-functions/utils/pom.xml | 89 +--------- pulsar-functions/worker/pom.xml | 71 +------- .../org/apache/pulsar/functions/worker/Utils.java | 4 +- .../functions/worker/rest/api/FunctionsImpl.java | 1 - pulsar-zookeeper-utils/pom.xml | 2 +- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 3 +- .../zookeeper/ZookeeperClientFactoryImpl.java | 2 +- .../pulsar/zookeeper/ZookeeperServerTest.java | 2 + 31 files changed, 158 insertions(+), 509 deletions(-) diff --git a/all/pom.xml b/all/pom.xml index 7ac9f54..0a9d289 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -73,12 +73,12 @@ <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> </dependency> - + <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-core</artifactId> </dependency> - + <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-web</artifactId> @@ -88,7 +88,7 @@ <groupId>com.fasterxml.jackson.dataformat</groupId> <artifactId>jackson-dataformat-yaml</artifactId> </dependency> - + <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-slf4j-impl</artifactId> @@ -114,6 +114,12 @@ <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-functions-worker</artifactId> + <version>${project.version}</version> + </dependency> + <!-- runtime-all --> <dependency> <groupId>org.apache.pulsar</groupId> diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml index f3cfa9b..317982a 100644 --- a/all/src/assemble/bin.xml +++ b/all/src/assemble/bin.xml @@ -121,9 +121,6 @@ <!-- Already included in pulsar-zookeeper instrumented jar --> <exclude>org.apache.zookeeper:zookeeper</exclude> - <!-- Already shaded in runtime-shaded jar --> - <exclude>io.grpc:*</exclude> - <!-- Explicitely remove JUnit which is getting pulled in even though it's set to the scope 'test' --> <exclude>junit:junit</exclude> diff --git a/bin/pulsar b/bin/pulsar index 4a58d30..e096d27 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -251,6 +251,8 @@ OPTS="$OPTS -Dpulsar.functions.process.container.log.dir=$PULSAR_LOG_DIR" OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" +ZK_OPTS=" -Dzookeeper.4lw.commands.whitelist=*" + #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" if [ $COMMAND == "broker" ]; then @@ -261,16 +263,16 @@ elif [ $COMMAND == "bookie" ]; then exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.proto.BookieServer --conf $PULSAR_BOOKKEEPER_CONF $@ elif [ $COMMAND == "zookeeper" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"zookeeper.log"} - exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.zookeeper.ZooKeeperStarter $PULSAR_ZK_CONF $@ + exec $JAVA ${ZK_OPTS} $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.zookeeper.ZooKeeperStarter $PULSAR_ZK_CONF $@ elif [ $COMMAND == "global-zookeeper" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"global-zookeeper.log"} # Allow global ZK to turn into read-only mode when it cannot reach the quorum - OPTS="${OPTS} -Dreadonlymode.enabled=true" + OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true" exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.zookeeper.ConfigurationStoreStarter $PULSAR_GLOBAL_ZK_CONF $@ elif [ $COMMAND == "configuration-store" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"configuration-store.log"} # Allow global ZK to turn into read-only mode when it cannot reach the quorum - OPTS="${OPTS} -Dreadonlymode.enabled=true" + OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true" exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.zookeeper.ConfigurationStoreStarter $PULSAR_CONFIGURATION_STORE_CONF $@ elif [ $COMMAND == "discovery" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"discovery.log"} @@ -286,7 +288,7 @@ elif [ $COMMAND == "functions-worker" ]; then exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@ elif [ $COMMAND == "standalone" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"} - exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@ + exec $JAVA $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@ elif [ $COMMAND == "initialize-cluster-metadata" ]; then exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup $@ elif [ $COMMAND == "zookeeper-shell" ]; then diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 7b0f701..9ee8570 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -35,7 +35,7 @@ <dependencies> <dependency> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-server-shaded</artifactId> + <artifactId>bookkeeper-server</artifactId> </dependency> <dependency> diff --git a/pom.xml b/pom.xml index 28e769b..90fa1fa 100644 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,6 @@ flexible messaging model and an intuitive client API.</description> <module>pulsar-client-shaded</module> <module>pulsar-client-admin</module> <module>pulsar-client-admin-shaded</module> - <module>pulsar-client-admin-shaded-for-functions</module> <module>pulsar-client-tools</module> <module>pulsar-client-tools-test</module> <module>pulsar-websocket</module> @@ -123,7 +122,7 @@ flexible messaging model and an intuitive client API.</description> <testRetryCount>1</testRetryCount> <bookkeeper.version>4.7.0</bookkeeper.version> - <zookeeper.version>3.4.10</zookeeper.version> + <zookeeper.version>3.5.4-beta</zookeeper.version> <netty.version>4.1.21.Final</netty.version> <storm.version>1.0.5</storm.version> <jetty.version>9.3.11.v20160721</jetty.version> @@ -141,7 +140,7 @@ flexible messaging model and an intuitive client API.</description> <typetools.version>0.5.0</typetools.version> <jboss-reflect.version>2.2.1.SP1</jboss-reflect.version> <protobuf2.version>2.4.1</protobuf2.version> - <protobuf3.version>3.5.1</protobuf3.version> + <protobuf3.version>3.4.0</protobuf3.version> <grpc.version>1.5.0</grpc.version> <protoc-gen-grpc-java.version>1.0.0</protoc-gen-grpc-java.version> <gson.version>2.8.2</gson.version> @@ -252,7 +251,7 @@ flexible messaging model and an intuitive client API.</description> <dependency> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-server-shaded</artifactId> + <artifactId>bookkeeper-server</artifactId> <version>${bookkeeper.version}</version> <exclusions> <exclusion> @@ -722,10 +721,9 @@ flexible messaging model and an intuitive client API.</description> <scope>import</scope> </dependency> - <!-- use shaded dependency util pulsar bump zookeeper version to 3.5 --> <dependency> <groupId>org.apache.distributedlog</groupId> - <artifactId>distributedlog-core-shaded</artifactId> + <artifactId>distributedlog-core</artifactId> <version>${bookkeeper.version}</version> </dependency> @@ -781,9 +779,10 @@ flexible messaging model and an intuitive client API.</description> <dependency> <!-- We use MockedBookKeeper in many unit tests --> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-server-tests-shaded</artifactId> + <artifactId>bookkeeper-server</artifactId> <version>${bookkeeper.version}</version> <scope>test</scope> + <classifier>tests</classifier> <exclusions> <exclusion> <groupId>org.slf4j</groupId> diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 7ed7407..a23e76f 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -111,12 +111,6 @@ <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-common</artifactId> - <version>${project.version}</version> - </dependency> - - <dependency> - <groupId>${project.groupId}</groupId> <artifactId>pulsar-broker-common</artifactId> <version>${project.version}</version> </dependency> diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index 3c2544b..288e14c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -22,30 +22,22 @@ import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.pulsar.common.api.Commands.newLookupErrorResponse; import static org.apache.pulsar.common.api.Commands.newLookupResponse; +import io.netty.buffer.ByteBuf; + import java.net.URI; import java.net.URISyntaxException; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import javax.ws.rs.DefaultValue; import javax.ws.rs.Encoded; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.container.AsyncResponse; -import javax.ws.rs.container.Suspended; -import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; -import org.apache.pulsar.broker.lookup.LookupResult; -import org.apache.pulsar.broker.web.NoSwaggerDocumentation; import org.apache.pulsar.broker.web.PulsarWebResource; import org.apache.pulsar.broker.web.RestException; import org.apache.pulsar.common.api.proto.PulsarApi.CommandLookupTopicResponse.LookupType; @@ -58,11 +50,6 @@ import org.apache.pulsar.common.util.Codec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import dlshade.org.apache.commons.lang3.StringUtils; -import io.netty.buffer.ByteBuf; -import io.swagger.annotations.ApiResponse; -import io.swagger.annotations.ApiResponses; - public class TopicLookupBase extends PulsarWebResource { private static final String LOOKUP_PATH_V1 = "/lookup/v2/destination/"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java index 3c5f337..db758d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java @@ -21,11 +21,14 @@ package org.apache.pulsar.broker.s3offload.impl; import static com.google.common.base.Preconditions.checkState; import com.google.common.collect.Maps; +import com.google.protobuf.ByteString; + import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufInputStream; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; + import java.io.DataInputStream; import java.io.IOException; import java.io.FilterInputStream; @@ -36,6 +39,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; import java.util.stream.Collectors; + import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.api.DigestType; import org.apache.bookkeeper.client.api.LedgerMetadata; @@ -43,7 +47,6 @@ import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.proto.DataFormats; import org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat; import org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.State; -import org.apache.bookkeeper.shaded.com.google.protobuf.ByteString; import org.apache.pulsar.broker.s3offload.OffloadIndexBlock; import org.apache.pulsar.broker.s3offload.OffloadIndexEntry; import org.slf4j.Logger; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java index 0c2a37e..72f5147 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.client.api; +import com.google.common.collect.Sets; + import java.lang.reflect.Field; import java.util.Arrays; import java.util.LinkedList; @@ -39,8 +41,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import dlshade.com.google.common.collect.Sets; - public class MessageDispatchThrottlingTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(MessageDispatchThrottlingTest.class); diff --git a/pulsar-client-admin-shaded-for-functions/pom.xml b/pulsar-client-admin-shaded-for-functions/pom.xml deleted file mode 100644 index e36681c..0000000 --- a/pulsar-client-admin-shaded-for-functions/pom.xml +++ /dev/null @@ -1,180 +0,0 @@ -<!-- - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar</artifactId> - <version>2.1.0-incubating-SNAPSHOT</version> - <relativePath>..</relativePath> - </parent> - - <artifactId>pulsar-client-admin-shaded-for-functions</artifactId> - <name>Pulsar Client Admin Shaded (for functions)</name> - - <dependencies> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>pulsar-client-admin-original</artifactId> - <version>${project.parent.version}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <!-- Shade all the dependencies to avoid conflicts --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <createDependencyReducedPom>true</createDependencyReducedPom> - <promoteTransitiveDependencies>true</promoteTransitiveDependencies> - <minimizeJar>false</minimizeJar> - - <artifactSet> - <includes> - <include>org.apache.pulsar:pulsar-common</include> - <include>org.apache.bookkeeper:circe-checksum</include> - <include>org.apache.pulsar:pulsar-io-core</include> - <include>org.apache.pulsar:pulsar-client-original</include> - <include>org.apache.pulsar:pulsar-client-admin-original</include> - <!-- client dependencies as below --> - <include>org.apache.commons:commons-lang3</include> - <include>commons-codec:commons-codec</include> - <include>commons-collections:commons-collections</include> - <include>org.asynchttpclient:*</include> - <include>io.netty:netty</include> - <include>io.netty:netty-all</include> - <include>io.netty:netty-codec-http</include> - <include>io.netty:netty-tcnative-boringssl-static</include> - <include>io.netty:netty-transport-native-epoll</include> - <include>org.reactivestreams:reactive-streams</include> - <include>com.typesafe.netty:netty-reactive-streams</include> - <include>org.javassist:javassist</include> - <include>com.google.protobuf:protobuf-java</include> - <include>com.google.guava:guava</include> - <include>com.google.code.gson:gson</include> - <include>com.fasterxml.jackson.core</include> - <include>com.fasterxml.jackson.module</include> - <include>net.jpountz.lz4:lz4</include> - <include>com.yahoo.datasketches:sketches-core</include> - <include>org.apache.httpcomponents:httpclient</include> - <include>commons-logging:commons-logging</include> - <include>org.apache.httpcomponents:httpcore</include> - </includes> - </artifactSet> - <filters> - <filter> - <artifact>net.jpountz.lz4:lz4</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>org.apache.pulsar:pulsar-common</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>org.apache.bookkeeper:circe-checksum</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>org.apache.pulsar:pulsar-io-core</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>org.apache.pulsar:pulsar-client-original</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>org.apache.pulsar:pulsar-client-admin-original</artifact> - <includes> - <include>**</include> - </includes> - </filter> - <filter> - <artifact>commons-logging:commons-logging</artifact> - <includes> - <include>**</include> - </includes> - </filter> - </filters> - <relocations> - <relocation> - <pattern>org.asynchttpclient</pattern> - <shadedPattern>org.apache.pulsar.shade.org.asynchttpclient</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.http</pattern> - <shadedPattern>org.apache.pulsar.shade.org.apache.http</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.commons</pattern> - <shadedPattern>org.apache.pulsar.shade.org.apache.commons</shadedPattern> - </relocation> - <relocation> - <pattern>com.google</pattern> - <shadedPattern>org.apache.pulsar.shade.com.google</shadedPattern> - </relocation> - <relocation> - <pattern>io.netty</pattern> - <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern> - </relocation> - <relocation> - <pattern>com.scurrilous.circe</pattern> - <shadedPattern>org.apache.pulsar.shade.com.scurrilous.circe</shadedPattern> - </relocation> - <relocation> - <pattern>net.jpountz</pattern> - <shadedPattern>org.apache.pulsar.shade.net.jpountz</shadedPattern> - </relocation> - <relocation> - <pattern>com.yahoo.datasketches</pattern> - <shadedPattern>org.apache.pulsar.shade.com.yahoo.datasketches</shadedPattern> - </relocation> - <relocation> - <pattern>com.yahoo.sketches</pattern> - <shadedPattern>org.apache.pulsar.shade.com.yahoo.sketches</shadedPattern> - </relocation> - </relocations> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 702a29e..4ea2814 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -48,16 +48,10 @@ <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-functions-proto-shaded</artifactId> + <artifactId>pulsar-functions-proto</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>*</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> </dependency> - + <dependency> <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-client</artifactId> diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java index e5bb716..7c2d907 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/Functions.java @@ -18,13 +18,13 @@ */ package org.apache.pulsar.client.admin; +import java.util.List; + import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException; import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException; import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList; - -import java.util.List; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatusList; /** * Admin interface for function management. diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index 66da497..1e2ae22 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -18,20 +18,15 @@ */ package org.apache.pulsar.client.admin.internal; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.api.Authentication; -import org.apache.pulsar.common.policies.data.*; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.proto.InstanceCommunication.FunctionStatusList; -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; -import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; +import com.google.protobuf.AbstractMessage.Builder; +import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.util.JsonFormat; -import org.apache.pulsar.functions.shaded.com.google.protobuf.MessageOrBuilder; -import org.apache.pulsar.functions.shaded.com.google.protobuf.AbstractMessage.Builder; -import org.apache.pulsar.functions.shaded.com.google.protobuf.util.JsonFormat; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.StandardCopyOption; +import java.util.List; import javax.ws.rs.ClientErrorException; import javax.ws.rs.client.Entity; @@ -39,11 +34,18 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.StandardCopyOption; -import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.pulsar.client.admin.Functions; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.common.policies.data.ErrorData; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatusList; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; @Slf4j public class FunctionsImpl extends BaseResource implements Functions { @@ -199,11 +201,11 @@ public class FunctionsImpl extends BaseResource implements Functions { throw getApiException(e); } } - + public static void mergeJson(String json, Builder builder) throws IOException { JsonFormat.parser().merge(json, builder); } - + public static String printJson(MessageOrBuilder msg) throws IOException { return JsonFormat.printer().print(msg); } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index 83220d0..3d0bb2b 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -31,6 +31,11 @@ import static org.powermock.api.mockito.PowerMockito.mockStatic; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; +import com.google.gson.Gson; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; + import java.io.File; import java.util.HashMap; import java.util.Map; @@ -53,9 +58,7 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.utils.DefaultSerDe; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf; -import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.utils.Reflections; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; @@ -65,8 +68,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test; -import com.google.gson.Gson; - /** * Unit test of {@link CmdFunctions}. */ diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index eedbbea..21de29b 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -18,6 +18,11 @@ */ package org.apache.pulsar.admin.cli; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.isNull; +import static org.apache.bookkeeper.common.concurrent.FutureUtils.result; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.converters.StringConverter; @@ -28,9 +33,31 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParser; import com.google.gson.reflect.TypeToken; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; + import org.apache.bookkeeper.api.StorageClient; import org.apache.bookkeeper.api.kv.Table; import org.apache.bookkeeper.api.kv.result.KeyValue; @@ -46,19 +73,14 @@ import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.api.utils.DefaultSerDe; import org.apache.pulsar.functions.instance.InstanceConfig; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.Resources; +import org.apache.pulsar.functions.proto.Function.SinkSpec; +import org.apache.pulsar.functions.proto.Function.SourceSpec; +import org.apache.pulsar.functions.proto.Function.SubscriptionType; import org.apache.pulsar.functions.runtime.ProcessRuntimeFactory; import org.apache.pulsar.functions.runtime.RuntimeSpawner; -import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBuf; -import org.apache.pulsar.functions.shaded.io.netty.buffer.ByteBufUtil; -import org.apache.pulsar.functions.shaded.io.netty.buffer.Unpooled; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.shaded.proto.Function.Resources; -import org.apache.pulsar.functions.shaded.proto.Function.SinkSpec; -import org.apache.pulsar.functions.shaded.proto.Function.SourceSpec; -import org.apache.pulsar.functions.shaded.proto.Function.SubscriptionType; -import org.apache.pulsar.functions.sink.PulsarSink; -import org.apache.pulsar.functions.source.PulsarSource; import org.apache.pulsar.functions.utils.FunctionConfig; import org.apache.pulsar.functions.utils.Reflections; import org.apache.pulsar.functions.utils.Utils; @@ -66,27 +88,7 @@ import org.apache.pulsar.functions.utils.WindowConfig; import org.apache.pulsar.functions.windowing.WindowFunctionExecutor; import org.apache.pulsar.functions.windowing.WindowUtils; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import static com.google.common.base.Preconditions.checkNotNull; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.isNull; -import static org.apache.bookkeeper.common.concurrent.FutureUtils.result; +import net.jodah.typetools.TypeResolver; @Slf4j @Parameters(commandDescription = "Interface for managing Pulsar Functions (lightweight, Lambda-style compute processes that work with Pulsar)") diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java index ed1cfa7..fa99091 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java @@ -25,38 +25,37 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; + import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.internal.FunctionsImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.utils.IdentityFunction; -import org.apache.pulsar.functions.shaded.proto.Function; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.proto.Function.Resources; -import org.apache.pulsar.functions.shaded.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.shaded.proto.Function.SinkSpec; -import org.apache.pulsar.functions.shaded.proto.Function.SourceSpec; -import org.apache.pulsar.functions.sink.PulsarSink; -import org.apache.pulsar.functions.source.PulsarSource; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.Resources; +import org.apache.pulsar.functions.proto.Function.SinkSpec; +import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.utils.FunctionConfig; import org.apache.pulsar.functions.utils.Reflections; import org.apache.pulsar.functions.utils.SinkConfig; import org.apache.pulsar.functions.utils.Utils; import org.apache.pulsar.io.core.Sink; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; +import net.jodah.typetools.TypeResolver; -@Slf4j @Getter @Parameters(commandDescription = "Interface for managing Pulsar Sinks (Egress data from Pulsar)") public class CmdSinks extends CmdBase { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java index dbcdef7..6cdc37c 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java @@ -25,39 +25,31 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.util.Map; + import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; + import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.internal.FunctionsImpl; -import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.utils.IdentityFunction; -import org.apache.pulsar.functions.shaded.proto.Function; -import org.apache.pulsar.functions.shaded.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.shaded.proto.Function.Resources; -import org.apache.pulsar.functions.shaded.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.shaded.proto.Function.SinkSpec; -import org.apache.pulsar.functions.shaded.proto.Function.SourceSpec; -import org.apache.pulsar.functions.sink.PulsarSink; -import org.apache.pulsar.functions.source.PulsarSource; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; +import org.apache.pulsar.functions.proto.Function.Resources; +import org.apache.pulsar.functions.proto.Function.SinkSpec; +import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.utils.FunctionConfig; import org.apache.pulsar.functions.utils.Reflections; import org.apache.pulsar.functions.utils.SourceConfig; import org.apache.pulsar.functions.utils.Utils; -import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; +import net.jodah.typetools.TypeResolver; -@Slf4j @Getter @Parameters(commandDescription = "Interface for managing Pulsar Source (Ingress data to Pulsar)") public class CmdSources extends CmdBase { diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index b6dfb74..e5cc988 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -60,6 +60,18 @@ </dependency> <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-functions-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-client-original</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>org.apache.bookkeeper</groupId> <artifactId>stream-storage-java-client</artifactId> </dependency> diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java index 0f826ac..86ea562 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java @@ -24,7 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException;; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.functions.api.SerDe; import org.apache.pulsar.functions.api.utils.DefaultSerDe; import org.apache.pulsar.functions.utils.FunctionConfig; diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 99c6947..02ab2e4 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -43,7 +43,6 @@ <module>runtime-shaded</module> <module>runtime-all</module> <module>worker</module> - <module>worker-shaded</module> </modules> <dependencyManagement> diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 3d9201c..813f4c6 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -36,7 +36,7 @@ <dependencies> <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-functions-runtime-shaded</artifactId> + <artifactId>pulsar-functions-runtime</artifactId> <version>${project.parent.version}</version> <exclusions> <exclusion> diff --git a/pulsar-functions/runtime-shaded/pom.xml b/pulsar-functions/runtime-shaded/pom.xml index f50e3d9..f7130ac 100644 --- a/pulsar-functions/runtime-shaded/pom.xml +++ b/pulsar-functions/runtime-shaded/pom.xml @@ -39,11 +39,6 @@ <artifactId>pulsar-functions-runtime</artifactId> <version>${project.parent.version}</version> <exclusions> - <!-- exclude `pulsar-client-admin-shaded-for-functions` here, this allows worker and runtime to use unshaded clients --> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-client-admin-shaded-for-functions</artifactId> - </exclusion> <exclusion> <groupId>io.netty</groupId> <artifactId>netty-common</artifactId> @@ -89,7 +84,7 @@ <dependency> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-server-shaded</artifactId> + <artifactId>bookkeeper-server</artifactId> </dependency> <dependency> diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 6feecd9..419d77b 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -59,7 +59,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> - <version>1.8</version> <executions> <execution> <phase>package</phase> diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index e831ca9..b278ed7 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -35,95 +35,8 @@ <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-client-admin-shaded-for-functions</artifactId> + <artifactId>pulsar-client-admin-original</artifactId> <version>${project.version}</version> - <!-- exclude shaded dependencies to avoid conflicts --> - <exclusions> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-client-admin-original</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-proto</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-io-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-client-original</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </exclusion> - <exclusion> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </exclusion> - <exclusion> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </exclusion> - <exclusion> - <groupId>org.asynchttpclient</groupId> - <artifactId>async-http-client</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-all</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-codec-http</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-tcnative-boringssl-static</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-transport-native-epoll</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </exclusion> - <exclusion> - <groupId>net.jpountz.lz4</groupId> - <artifactId>lz4</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcomponents-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcomponents-client</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.datasketches</groupId> - <artifactId>sketches-core</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 22137ba..7208861 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -36,74 +36,8 @@ <dependency> <groupId>${project.groupId}</groupId> - <artifactId>pulsar-functions-runtime-shaded</artifactId> + <artifactId>pulsar-functions-runtime</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-all</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-lite</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf.nano</groupId> - <artifactId>protobuf-javanano</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java-util</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf-lite</artifactId> - </exclusion> - <exclusion> - <groupId>io.grpc</groupId> - <artifactId>grpc-protobuf-nano</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-metrics</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.pulsar</groupId> - <artifactId>pulsar-functions-instance</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-transport</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-resolver</artifactId> - </exclusion> - <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty-codec-socks</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> @@ -159,10 +93,9 @@ <artifactId>jetty-servlet</artifactId> </dependency> - <!-- use shaded dependency util pulsar bump zookeeper version to 3.5 --> <dependency> <groupId>org.apache.distributedlog</groupId> - <artifactId>distributedlog-core-shaded</artifactId> + <artifactId>distributedlog-core</artifactId> </dependency> </dependencies> diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java index 494368f..b64a7fa 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Utils.java @@ -39,12 +39,10 @@ import org.apache.distributedlog.impl.metadata.BKDLConfig; import org.apache.distributedlog.metadata.DLMetadata; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.worker.dlog.DLInputStream; import org.apache.pulsar.functions.worker.dlog.DLOutputStream; - -import dlshade.org.apache.zookeeper.KeeperException.Code; +import org.apache.zookeeper.KeeperException.Code; @Slf4j public final class Utils { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index d306efa..6d5cd1a 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -55,7 +55,6 @@ import org.apache.pulsar.functions.proto.Function.FunctionMetaData; import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; -import org.apache.pulsar.functions.source.PulsarSource; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; import org.apache.pulsar.functions.worker.MembershipManager; diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index f340c3c..2e40ce1 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -41,7 +41,7 @@ <dependency> <groupId>org.apache.bookkeeper</groupId> - <artifactId>bookkeeper-server-shaded</artifactId> + <artifactId>bookkeeper-server</artifactId> </dependency> <dependency> diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java index 09a75d0..05fe468 100644 --- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java +++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java @@ -118,12 +118,13 @@ public class LocalBookkeeperEnsemble { } try { + // Allow all commands on ZK control port + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); zks = new ZooKeeperServer(zkDataDir, zkDataDir, ZooKeeperServer.DEFAULT_TICK_TIME); serverFactory = new NIOServerCnxnFactory(); serverFactory.configure(new InetSocketAddress(ZooKeeperDefaultPort), maxCC); serverFactory.startup(zks); } catch (Exception e) { - // TODO Auto-generated catch block LOG.error("Exception while instantiating ZooKeeper", e); } diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZookeeperClientFactoryImpl.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZookeeperClientFactoryImpl.java index 18ab9d8..9d8b098 100644 --- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZookeeperClientFactoryImpl.java +++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZookeeperClientFactoryImpl.java @@ -75,7 +75,7 @@ public class ZookeeperClientFactoryImpl implements ZooKeeperClientFactory { return null; }); - } catch (IOException e) { + } catch (IllegalArgumentException | IOException e) { future.completeExceptionally(e); } diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java index c53e1bc..20de01a 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperServerTest.java @@ -48,6 +48,8 @@ public class ZookeeperServerTest implements Closeable { public void start() throws IOException { try { + // Allow all commands on ZK control port + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperServer.DEFAULT_TICK_TIME); zks.setMaxSessionTimeout(20000); serverFactory = new NIOServerCnxnFactory(); -- To stop receiving notification emails like this one, please contact si...@apache.org.