This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 8a82d3db7850e866f9153ef4de381669a933d633 Author: Volodymyr Vysotskyi <[email protected]> AuthorDate: Mon Apr 13 18:04:28 2020 +0300 DRILL-7702: Update ZooKeeper and Curator, exclude org.codehaus.jackson --- contrib/format-maprdb/pom.xml | 17 ++++++- .../store/mapr/db/binary/BinaryTableGroupScan.java | 2 +- .../store/mapr/db/json/JsonTableGroupScan.java | 2 +- contrib/storage-hbase/pom.xml | 30 +++++++++++- contrib/storage-hive/core/pom.xml | 16 +++++++ contrib/storage-hive/hive-exec-shade/pom.xml | 16 +++++++ distribution/src/assemble/component.xml | 1 + .../apache/drill/yarn/zk/ZKClusterCoordinator.java | 2 +- .../drill/exec/coord/zk/ZKClusterCoordinator.java | 2 +- .../org/apache/drill/exec/work/QueryWorkUnit.java | 2 +- .../apache/drill/exec/work/foreman/Foreman.java | 28 ++++------- exec/jdbc-all/pom.xml | 4 +- pom.xml | 55 ++++++++++++++++++---- 13 files changed, 140 insertions(+), 37 deletions(-) diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml index d308970..35011a8 100644 --- a/contrib/format-maprdb/pom.xml +++ b/contrib/format-maprdb/pom.xml @@ -59,7 +59,10 @@ <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </exclusion> - + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> </exclusions> </dependency> </dependencies> @@ -144,6 +147,18 @@ <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java index 24addd0..7ebb1a7 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java @@ -22,6 +22,7 @@ import static org.apache.drill.exec.store.mapr.db.util.CommonFns.isNullOrEmpty; import java.util.List; import java.util.TreeMap; +import com.fasterxml.jackson.annotation.JsonCreator; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rex.RexNode; import org.apache.drill.common.exceptions.DrillRuntimeException; @@ -56,7 +57,6 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.RegionLocator; -import org.codehaus.jackson.annotate.JsonCreator; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java index bb4abc0..b93a62e 100644 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java +++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import com.fasterxml.jackson.annotation.JsonCreator; import com.mapr.db.impl.ConditionImpl; import com.mapr.db.impl.ConditionNode.RowkeyRange; @@ -67,7 +68,6 @@ import org.apache.drill.exec.util.Utilities; import org.apache.drill.exec.metastore.store.FileSystemMetadataProviderManager; import org.apache.drill.exec.metastore.MetadataProviderManager; import org.apache.drill.metastore.metadata.TableMetadataProvider; -import org.codehaus.jackson.annotate.JsonCreator; import org.ojai.store.QueryCondition; import com.fasterxml.jackson.annotation.JacksonInject; diff --git a/contrib/storage-hbase/pom.xml b/contrib/storage-hbase/pom.xml index 7aa69bb..5f08276 100644 --- a/contrib/storage-hbase/pom.xml +++ b/contrib/storage-hbase/pom.xml @@ -200,6 +200,7 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-testing-util</artifactId> + <scope>test</scope> <classifier>tests</classifier> <exclusions> <exclusion> @@ -210,8 +211,23 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> </exclusions> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -257,6 +273,10 @@ <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -273,6 +293,14 @@ <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 1c18176..ecfaf01 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -228,6 +228,22 @@ <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> </exclusions> </dependency> diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml index 6380eb3..778ba68 100644 --- a/contrib/storage-hive/hive-exec-shade/pom.xml +++ b/contrib/storage-hive/hive-exec-shade/pom.xml @@ -104,6 +104,22 @@ <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> </exclusions> </dependency> </dependencies> diff --git a/distribution/src/assemble/component.xml b/distribution/src/assemble/component.xml index fefbac4..f37f195 100644 --- a/distribution/src/assemble/component.xml +++ b/distribution/src/assemble/component.xml @@ -159,6 +159,7 @@ <useProjectArtifact>false</useProjectArtifact> <includes> <include>org.apache.zookeeper:zookeeper</include> + <include>org.apache.zookeeper:zookeeper-jute</include> </includes> <scope>runtime</scope> </dependencySet> diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/zk/ZKClusterCoordinator.java b/drill-yarn/src/main/java/org/apache/drill/yarn/zk/ZKClusterCoordinator.java index e798312..fd54a43 100644 --- a/drill-yarn/src/main/java/org/apache/drill/yarn/zk/ZKClusterCoordinator.java +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/zk/ZKClusterCoordinator.java @@ -174,7 +174,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator { // serviceCache before discovery to prevent // double releasing and disallowing jvm to spit bothering warnings. simply // put, we are great! - AutoCloseables.close(serviceCache, discovery, curator, factory); + AutoCloseables.close(serviceCache, discovery, factory, curator); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java index 74dd01c..aeb0dd0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/ZKClusterCoordinator.java @@ -173,7 +173,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator { // discovery attempts to close its caches(ie serviceCache) already. however, being good citizens we make sure to // explicitly close serviceCache. Not only that we make sure to close serviceCache before discovery to prevent // double releasing and disallowing jvm to spit bothering warnings. simply put, we are great! - AutoCloseables.close(serviceCache, discovery, curator, factory); + AutoCloseables.close(serviceCache, discovery, factory, curator); } @Override diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/QueryWorkUnit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/QueryWorkUnit.java index 699b1d7..2e3a0d0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/QueryWorkUnit.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/QueryWorkUnit.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.work; import java.util.ArrayList; import java.util.List; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.drill.exec.physical.base.FragmentRoot; import org.apache.drill.exec.planner.PhysicalPlanReader; import org.apache.drill.exec.planner.fragment.Wrapper; @@ -31,7 +32,6 @@ import org.apache.drill.exec.work.foreman.ForemanSetupException; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; -import org.codehaus.jackson.map.ObjectMapper; public class QueryWorkUnit { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 1cecbd3..38b85b6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.work.foreman; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.drill.exec.work.filter.RuntimeFilterRouter; import org.apache.drill.shaded.guava.com.google.common.base.Preconditions; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; @@ -64,7 +65,6 @@ import org.apache.drill.exec.work.WorkManager.WorkerBee; import org.apache.drill.exec.work.foreman.rm.QueryQueue.QueueTimeoutException; import org.apache.drill.exec.work.foreman.rm.QueryQueue.QueryQueueException; import org.apache.drill.exec.work.foreman.rm.QueryResourceManager; -import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,7 +115,7 @@ public class Foreman implements Runnable { private final QueryResourceManager queryRM; private final ResponseSendListener responseListener = new ResponseSendListener(); - private final ConnectionClosedListener closeListener = new ConnectionClosedListener(); + private final GenericFutureListener<Future<Void>> closeListener = future -> cancel(); private final ChannelFuture closeFuture; private final FragmentsRunner fragmentsRunner; private final QueryStateProcessor queryStateProcessor; @@ -602,6 +602,10 @@ public class Foreman implements Runnable { new BasicOptimizer.BasicOptimizationContext(queryContext), plan); } + public RuntimeFilterRouter getRuntimeFilterRouter() { + return runtimeFilterRouter; + } + /** * Manages the end-state processing for Foreman. * @@ -763,12 +767,7 @@ public class Foreman implements Runnable { * We only need to do this if the resultState differs from the last recorded state */ if (resultState != queryStateProcessor.getState()) { - suppressingClose(new AutoCloseable() { - @Override - public void close() throws Exception { - queryStateProcessor.recordNewState(resultState); - } - }); + suppressingClose(() -> queryStateProcessor.recordNewState(resultState)); } // set query end time before writing final profile @@ -853,17 +852,10 @@ public class Foreman implements Runnable { } } - private class ConnectionClosedListener implements GenericFutureListener<Future<Void>> { - @Override - public void operationComplete(Future<Void> future) throws Exception { - cancel(); - } - } - /** * Listens for the status of the RPC response sent to the user for the query. */ - private class ResponseSendListener extends BaseRpcOutcomeListener<Ack> { + private static class ResponseSendListener extends BaseRpcOutcomeListener<Ack> { @Override public void failed(final RpcException ex) { logger.info("Failure while trying communicate query result to initiating client. " + @@ -876,8 +868,4 @@ public class Foreman implements Runnable { } } - public RuntimeFilterRouter getRuntimeFilterRouter() { - return runtimeFilterRouter; - } - } diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 2aaae61..80a2d75 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -536,7 +536,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>43000000</maxsize> + <maxsize>45000000</maxsize> <minsize>15000000</minsize> <files> <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file> @@ -596,7 +596,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>44000000</maxsize> + <maxsize>46000000</maxsize> <minsize>15000000</minsize> <files> <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file> diff --git a/pom.xml b/pom.xml index 0ca5c5f..0c8028c 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ <sqlline.version>1.9.0</sqlline.version> <jackson.version>2.9.9</jackson.version> <jackson.databind.version>2.9.9</jackson.databind.version> - <zookeeper.version>3.4.12</zookeeper.version> + <zookeeper.version>3.5.7</zookeeper.version> <mapr.release.version>6.1.0-mapr</mapr.release.version> <ojai.version>3.0-mapr-1808</ojai.version> <kerby.version>1.0.0</kerby.version> @@ -72,7 +72,7 @@ <commons.io.version>2.4</commons.io.version> <hamcrest.core.version>1.3</hamcrest.core.version> <maven.embedder.version>3.5.3</maven.embedder.version> - <curator.version>2.7.1</curator.version> + <curator.version>4.3.0</curator.version> <wiremock.standalone.version>2.23.2</wiremock.standalone.version> <jmockit.version>1.47</jmockit.version> <logback.version>1.2.3</logback.version> @@ -2293,6 +2293,14 @@ <groupId>io.netty</groupId> <artifactId>netty</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> </exclusions> </dependency> @@ -2656,8 +2664,8 @@ <scope>test</scope> <exclusions> <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> </exclusion> <exclusion> <groupId>io.netty</groupId> @@ -2815,6 +2823,9 @@ <hbase.version>1.1.1-mapr-1602-m7-5.2.0</hbase.version> <hadoop.version>2.7.0-mapr-1808</hadoop.version> <zookeeper.version>3.4.11-mapr-1808</zookeeper.version> + <!-- TODO: Remove this property and use a common dependency of curator-test after ZooKeeper 3.5+ + for MapR profile is released. See problem description in https://issues.apache.org/jira/browse/CURATOR-428 --> + <curator.test.version>2.13.0</curator.test.version> </properties> <dependencyManagement> <dependencies> @@ -3148,8 +3159,16 @@ <artifactId>netty-all</artifactId> </exclusion> <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> </exclusion> </exclusions> </dependency> @@ -3183,8 +3202,8 @@ <version>${hbase.version}</version> <exclusions> <exclusion> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> </exclusion> <exclusion> <groupId>io.netty</groupId> @@ -3226,6 +3245,14 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-client</artifactId> </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -3353,6 +3380,18 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>${curator.test.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-annotations</artifactId> <version>${hbase.version}</version>
