This is an automated email from the ASF dual-hosted git repository. volodymyr pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit db6488238a912fefb5cc5d9e83f5575df51a74e9 Author: Anton Gozhiy <[email protected]> AuthorDate: Mon Nov 4 14:08:22 2019 +0200 DRILL-6540: Updated Hadoop and HBase libraries to the latest versions Hadoop: 3.2.1 HBase: 2.2.2 closes #1895 --- .../org/apache/drill/common/util/GuavaPatcher.java | 17 +++- distribution/pom.xml | 79 +++++++++++++++---- distribution/src/assemble/component.xml | 81 +++++++++---------- distribution/src/{ => main}/resources/LICENSE | 0 distribution/src/{ => main}/resources/NOTICE | 0 distribution/src/{ => main}/resources/README.md | 0 .../src/{ => main}/resources/auto-setup.sh | 0 .../src/{ => main}/resources/core-site-example.xml | 0 .../src/{ => main}/resources/distrib-env.sh | 0 .../src/{ => main}/resources/distrib-setup.sh | 0 .../src/{ => main}/resources/drill-am-log.xml | 4 +- distribution/src/{ => main}/resources/drill-am.sh | 0 distribution/src/{ => main}/resources/drill-conf | 0 .../src/{ => main}/resources/drill-config.sh | 0 .../src/{ => main}/resources/drill-embedded | 0 .../src/{ => main}/resources/drill-embedded.bat | 0 distribution/src/{ => main}/resources/drill-env.sh | 0 .../src/{ => main}/resources/drill-localhost | 0 .../resources/drill-on-yarn-example.conf | 0 .../src/{ => main}/resources/drill-on-yarn.sh | 0 .../resources/drill-override-example.conf | 48 ++++++------ .../src/{ => main}/resources/drill-override.conf | 4 +- .../src/{ => main}/resources/drill-setup.sh | 0 .../resources/drill-sqlline-override-example.conf | 0 distribution/src/{ => main}/resources/drillbit | 0 distribution/src/{ => main}/resources/drillbit.sh | 0 distribution/src/{ => main}/resources/dumpcat | 0 .../src/{ => main}/resources/hadoop-excludes.txt | 0 distribution/src/{ => main}/resources/logback.xml | 8 +- distribution/src/{ => main}/resources/runbit | 0 .../src/{ => main}/resources/saffron.properties | 0 distribution/src/{ => main}/resources/sqlline | 0 distribution/src/{ => main}/resources/sqlline.bat | 1 + .../storage-plugins-override-example.conf | 0 distribution/src/{ => main}/resources/submit_plan | 0 .../src/main/resources/winutils/hadoop.dll | Bin 0 -> 85504 bytes .../src/main/resources/winutils/winutils.exe | Bin 0 -> 112640 bytes .../src/{ => main}/resources/yarn-client-log.xml | 2 +- .../src/{ => main}/resources/yarn-drillbit.sh | 0 docs/dev/HadoopWinutils.md | 11 +++ exec/java-exec/pom.xml | 20 +---- .../apache/commons/logging/impl/Log4JLogger.java | 25 ++++++ .../drill/exec/store/LocalSyncableFileSystem.java | 8 +- .../exec/physical/unit/TestOutputBatchSize.java | 5 +- .../org/apache/drill/exec/work/batch/FileTest.java | 4 +- exec/jdbc-all/pom.xml | 2 +- .../org/apache/drill/jdbc/DrillbitClassLoader.java | 40 +--------- .../org/apache/drill/jdbc/ITTestShadedJar.java | 1 - pom.xml | 86 +++++++++++++++++---- 49 files changed, 269 insertions(+), 177 deletions(-) diff --git a/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java b/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java index 8eed12d..5924df8 100644 --- a/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java +++ b/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java @@ -135,14 +135,29 @@ public class GuavaPatcher { + " throw new IllegalArgumentException(format(errorMessageTemplate, new Object[] { new Integer(arg1) }));\n" + " }\n" + " }", + "public static void checkArgument(boolean expression, String errorMessageTemplate, long arg1) {\n" + + " if (!expression) {\n" + + " throw new IllegalArgumentException(format(errorMessageTemplate, new Object[] { new Long(arg1) }));\n" + + " }\n" + + " }", + "public static void checkArgument(boolean expression, String errorMessageTemplate, long arg1, long arg2) {\n" + + " if (!expression) {\n" + + " throw new IllegalArgumentException(format(errorMessageTemplate, new Object[] { new Long(arg1), new Long(arg2)}));\n" + + " }\n" + + " }", "public static Object checkNotNull(Object reference, String errorMessageTemplate, int arg1) {\n" + " if (reference == null) {\n" + " throw new NullPointerException(format(errorMessageTemplate, new Object[] { new Integer(arg1) }));\n" + " } else {\n" + " return reference;\n" + " }\n" + + " }", + "public static void checkState(boolean expression, String errorMessageTemplate, int arg1) {\n" + + " if (!expression) {\n" + + " throw new IllegalStateException(format(errorMessageTemplate, new Object[] { new Integer(arg1) }));\n" + + " }\n" + " }" - ); + ); List<String> newMethods = IntStream.rangeClosed(startIndex, endIndex) .mapToObj( diff --git a/distribution/pom.xml b/distribution/pom.xml index 10113bd..2c80d4b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -335,15 +335,30 @@ <name>!alt-hadoop</name> </property> </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-winutils</artifactId> - <version>2.7.1</version> - <type>zip</type> - </dependency> - </dependencies> <build> + <plugins> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>copy-winutils</id> + <phase>process-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/winutils</outputDirectory> + <resources> + <resource> + <directory>src/main/resources/winutils</directory> + <filtering>false</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> </build> </profile> <profile> @@ -361,12 +376,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-winutils</artifactId> - <version>2.7.0-mapr-1506</version> - <type>zip</type> - </dependency> - <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> </dependency> @@ -392,6 +401,44 @@ </dependency> </dependencies> <build> + <plugins> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>copy-winutils</id> + <phase>none</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.1.1</version> + <executions> + <execution> + <id>unpack-winutils</id> + <goals> + <goal>unpack</goal> + </goals> + <phase>process-resources</phase> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-winutils</artifactId> + <version>2.7.0-mapr-1506</version> + <type>zip</type> + <overWrite>true</overWrite> + <outputDirectory>${project.build.directory}/winutils</outputDirectory> + <excludes>**/*.pdb,**/*.lib,**/*.exp</excludes> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + </plugins> </build> </profile> <profile> @@ -494,7 +541,7 @@ <directory>/etc/init.d/</directory> <sources> <source> - <location>src/resources/drillbit</location> + <location>src/main/resources/drillbit</location> </source> </sources> <directoryIncluded>false</directoryIncluded> @@ -599,7 +646,7 @@ </mapper> </data> <data> - <src>src/resources/drillbit</src> + <src>src/main/resources/drillbit</src> <dst>/etc/init.d/drillbit</dst> <type>file</type> <mapper> diff --git a/distribution/src/assemble/component.xml b/distribution/src/assemble/component.xml index b8001c0..5dabc89 100644 --- a/distribution/src/assemble/component.xml +++ b/distribution/src/assemble/component.xml @@ -196,21 +196,6 @@ </includes> <scope>test</scope> </dependencySet> - <dependencySet> - <outputDirectory>winutils/bin</outputDirectory> - <unpack>true</unpack> - <unpackOptions> - <excludes> - <exclude>**/*.pdb</exclude> - <exclude>**/*.lib</exclude> - <exclude>**/*.exp</exclude> - </excludes> - </unpackOptions> - <useProjectArtifact>false</useProjectArtifact> - <includes> - <include>org.apache.hadoop:hadoop-winutils</include> - </includes> - </dependencySet> </dependencySets> <fileSets> @@ -218,6 +203,10 @@ <directory>../sample-data</directory> <outputDirectory>sample-data</outputDirectory> </fileSet> + <fileSet> + <directory>${project.build.directory}/winutils</directory> + <outputDirectory>winutils/bin</outputDirectory> + </fileSet> </fileSets> <files> @@ -226,11 +215,11 @@ <outputDirectory /> </file> <file> - <source>src/resources/LICENSE</source> + <source>src/main/resources/LICENSE</source> <outputDirectory /> </file> <file> - <source>src/resources/README.md</source> + <source>src/main/resources/README.md</source> <outputDirectory /> </file> <file> @@ -242,146 +231,146 @@ <outputDirectory /> </file> <file> - <source>src/resources/runbit</source> + <source>src/main/resources/runbit</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/hadoop-excludes.txt</source> + <source>src/main/resources/hadoop-excludes.txt</source> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drillbit.sh</source> + <source>src/main/resources/drillbit.sh</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-conf</source> + <source>src/main/resources/drill-conf</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-embedded</source> + <source>src/main/resources/drill-embedded</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-embedded.bat</source> + <source>src/main/resources/drill-embedded.bat</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-localhost</source> + <source>src/main/resources/drill-localhost</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-config.sh</source> + <source>src/main/resources/drill-config.sh</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/sqlline</source> + <source>src/main/resources/sqlline</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/sqlline.bat</source> + <source>src/main/resources/sqlline.bat</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-on-yarn.sh</source> + <source>src/main/resources/drill-on-yarn.sh</source> <fileMode>0750</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-am.sh</source> + <source>src/main/resources/drill-am.sh</source> <fileMode>0750</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/yarn-drillbit.sh</source> + <source>src/main/resources/yarn-drillbit.sh</source> <fileMode>0750</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/submit_plan</source> + <source>src/main/resources/submit_plan</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-override.conf</source> + <source>src/main/resources/drill-override.conf</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/logback.xml</source> + <source>src/main/resources/logback.xml</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/yarn-client-log.xml</source> + <source>src/main/resources/yarn-client-log.xml</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/drill-am-log.xml</source> + <source>src/main/resources/drill-am-log.xml</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/drill-env.sh</source> + <source>src/main/resources/drill-env.sh</source> <fileMode>0750</fileMode> <outputDirectory>conf</outputDirectory> </file> <file> - <source>src/resources/distrib-env.sh</source> + <source>src/main/resources/distrib-env.sh</source> <fileMode>0750</fileMode> <outputDirectory>conf</outputDirectory> </file> <file> - <source>src/resources/auto-setup.sh</source> + <source>src/main/resources/auto-setup.sh</source> <fileMode>0755</fileMode> <outputDirectory>bin</outputDirectory> </file> <file> - <source>src/resources/drill-setup.sh</source> + <source>src/main/resources/drill-setup.sh</source> <fileMode>0750</fileMode> <outputDirectory>conf</outputDirectory> </file> <file> - <source>src/resources/distrib-setup.sh</source> + <source>src/main/resources/distrib-setup.sh</source> <fileMode>0750</fileMode> <outputDirectory>conf</outputDirectory> </file> <file> - <source>src/resources/drill-override-example.conf</source> + <source>src/main/resources/drill-override-example.conf</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/core-site-example.xml</source> + <source>src/main/resources/core-site-example.xml</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/saffron.properties</source> + <source>src/main/resources/saffron.properties</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/drill-on-yarn-example.conf</source> + <source>src/main/resources/drill-on-yarn-example.conf</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/storage-plugins-override-example.conf</source> + <source>src/main/resources/storage-plugins-override-example.conf</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> <file> - <source>src/resources/drill-sqlline-override-example.conf</source> + <source>src/main/resources/drill-sqlline-override-example.conf</source> <outputDirectory>conf</outputDirectory> <fileMode>0640</fileMode> </file> diff --git a/distribution/src/resources/LICENSE b/distribution/src/main/resources/LICENSE similarity index 100% rename from distribution/src/resources/LICENSE rename to distribution/src/main/resources/LICENSE diff --git a/distribution/src/resources/NOTICE b/distribution/src/main/resources/NOTICE similarity index 100% rename from distribution/src/resources/NOTICE rename to distribution/src/main/resources/NOTICE diff --git a/distribution/src/resources/README.md b/distribution/src/main/resources/README.md similarity index 100% rename from distribution/src/resources/README.md rename to distribution/src/main/resources/README.md diff --git a/distribution/src/resources/auto-setup.sh b/distribution/src/main/resources/auto-setup.sh similarity index 100% rename from distribution/src/resources/auto-setup.sh rename to distribution/src/main/resources/auto-setup.sh diff --git a/distribution/src/resources/core-site-example.xml b/distribution/src/main/resources/core-site-example.xml similarity index 100% rename from distribution/src/resources/core-site-example.xml rename to distribution/src/main/resources/core-site-example.xml diff --git a/distribution/src/resources/distrib-env.sh b/distribution/src/main/resources/distrib-env.sh similarity index 100% rename from distribution/src/resources/distrib-env.sh rename to distribution/src/main/resources/distrib-env.sh diff --git a/distribution/src/resources/distrib-setup.sh b/distribution/src/main/resources/distrib-setup.sh similarity index 100% rename from distribution/src/resources/distrib-setup.sh rename to distribution/src/main/resources/distrib-setup.sh diff --git a/distribution/src/resources/drill-am-log.xml b/distribution/src/main/resources/drill-am-log.xml similarity index 99% rename from distribution/src/resources/drill-am-log.xml rename to distribution/src/main/resources/drill-am-log.xml index 187ca9d..3d1a9a3 100644 --- a/distribution/src/resources/drill-am-log.xml +++ b/distribution/src/main/resources/drill-am-log.xml @@ -27,14 +27,14 @@ and from there into the YARN-provided output log directory. --> <configuration> - + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <encoder> <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n </pattern> </encoder> </appender> - + <logger name="org.apache.drill" additivity="false"> <level value="info" /> <appender-ref ref="STDOUT" /> diff --git a/distribution/src/resources/drill-am.sh b/distribution/src/main/resources/drill-am.sh similarity index 100% rename from distribution/src/resources/drill-am.sh rename to distribution/src/main/resources/drill-am.sh diff --git a/distribution/src/resources/drill-conf b/distribution/src/main/resources/drill-conf similarity index 100% rename from distribution/src/resources/drill-conf rename to distribution/src/main/resources/drill-conf diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/main/resources/drill-config.sh similarity index 100% rename from distribution/src/resources/drill-config.sh rename to distribution/src/main/resources/drill-config.sh diff --git a/distribution/src/resources/drill-embedded b/distribution/src/main/resources/drill-embedded similarity index 100% rename from distribution/src/resources/drill-embedded rename to distribution/src/main/resources/drill-embedded diff --git a/distribution/src/resources/drill-embedded.bat b/distribution/src/main/resources/drill-embedded.bat similarity index 100% rename from distribution/src/resources/drill-embedded.bat rename to distribution/src/main/resources/drill-embedded.bat diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/main/resources/drill-env.sh similarity index 100% rename from distribution/src/resources/drill-env.sh rename to distribution/src/main/resources/drill-env.sh diff --git a/distribution/src/resources/drill-localhost b/distribution/src/main/resources/drill-localhost similarity index 100% rename from distribution/src/resources/drill-localhost rename to distribution/src/main/resources/drill-localhost diff --git a/distribution/src/resources/drill-on-yarn-example.conf b/distribution/src/main/resources/drill-on-yarn-example.conf similarity index 100% rename from distribution/src/resources/drill-on-yarn-example.conf rename to distribution/src/main/resources/drill-on-yarn-example.conf diff --git a/distribution/src/resources/drill-on-yarn.sh b/distribution/src/main/resources/drill-on-yarn.sh similarity index 100% rename from distribution/src/resources/drill-on-yarn.sh rename to distribution/src/main/resources/drill-on-yarn.sh diff --git a/distribution/src/resources/drill-override-example.conf b/distribution/src/main/resources/drill-override-example.conf similarity index 88% rename from distribution/src/resources/drill-override-example.conf rename to distribution/src/main/resources/drill-override-example.conf index 5aa45a9..e72396c 100644 --- a/distribution/src/resources/drill-override-example.conf +++ b/distribution/src/main/resources/drill-override-example.conf @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This file tells Drill to consider this module when class path scanning. -# This file can also include any supplementary configuration information. +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. # This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. drill.logical.function.packages += "org.apache.drill.exec.expr.fn.impl" @@ -41,7 +41,7 @@ drill.exec: { threads: 1 } }, - use.ip : false + use.ip : false }, operator: { packages += "org.apache.drill.exec.physical.config" @@ -64,28 +64,28 @@ drill.exec: { action_on_plugins_override_file: "none" }, zk: { - connect: "localhost:2181", - root: "drill", - refresh: 500, - timeout: 5000, - retry: { - count: 7200, - delay: 500 - } - # This option controls whether Drill specifies ACLs when it creates znodes. - # If this is 'false', then anyone has all privileges for all Drill znodes. - # This corresponds to ZOO_OPEN_ACL_UNSAFE. - # Setting this flag to 'true' enables the provider specified in "acl_provider" - apply_secure_acl: false, + connect: "localhost:2181", + root: "drill", + refresh: 500, + timeout: 5000, + retry: { + count: 7200, + delay: 500 + } + # This option controls whether Drill specifies ACLs when it creates znodes. + # If this is 'false', then anyone has all privileges for all Drill znodes. + # This corresponds to ZOO_OPEN_ACL_UNSAFE. + # Setting this flag to 'true' enables the provider specified in "acl_provider" + apply_secure_acl: false, - # This option specified the ACL provider to be used by Drill. - # Custom ACL providers can be provided in the Drillbit classpath and Drill can be made to pick them - # by changing this option. - # Note: This option has no effect if "apply_secure_acl" is 'false' - # - # The default "creator-all" will setup ACLs such that - # - Only the Drillbit user will have all privileges(create, delete, read, write, admin). Same as ZOO_CREATOR_ALL_ACL - # - Other users will only be able to read the cluster-discovery(list of Drillbits in the cluster) znodes. + # This option specified the ACL provider to be used by Drill. + # Custom ACL providers can be provided in the Drillbit classpath and Drill can be made to pick them + # by changing this option. + # Note: This option has no effect if "apply_secure_acl" is 'false' + # + # The default "creator-all" will setup ACLs such that + # - Only the Drillbit user will have all privileges(create, delete, read, write, admin). Same as ZOO_CREATOR_ALL_ACL + # - Other users will only be able to read the cluster-discovery(list of Drillbits in the cluster) znodes. # acl_provider: "creator-all" }, diff --git a/distribution/src/resources/drill-override.conf b/distribution/src/main/resources/drill-override.conf similarity index 97% rename from distribution/src/resources/drill-override.conf rename to distribution/src/main/resources/drill-override.conf index b484ea3..9f07e19 100644 --- a/distribution/src/resources/drill-override.conf +++ b/distribution/src/main/resources/drill-override.conf @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This file tells Drill to consider this module when class path scanning. -# This file can also include any supplementary configuration information. +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. # This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. # See 'drill-override-example.conf' for example configurations diff --git a/distribution/src/resources/drill-setup.sh b/distribution/src/main/resources/drill-setup.sh similarity index 100% rename from distribution/src/resources/drill-setup.sh rename to distribution/src/main/resources/drill-setup.sh diff --git a/distribution/src/resources/drill-sqlline-override-example.conf b/distribution/src/main/resources/drill-sqlline-override-example.conf similarity index 100% rename from distribution/src/resources/drill-sqlline-override-example.conf rename to distribution/src/main/resources/drill-sqlline-override-example.conf diff --git a/distribution/src/resources/drillbit b/distribution/src/main/resources/drillbit similarity index 100% rename from distribution/src/resources/drillbit rename to distribution/src/main/resources/drillbit diff --git a/distribution/src/resources/drillbit.sh b/distribution/src/main/resources/drillbit.sh similarity index 100% rename from distribution/src/resources/drillbit.sh rename to distribution/src/main/resources/drillbit.sh diff --git a/distribution/src/resources/dumpcat b/distribution/src/main/resources/dumpcat similarity index 100% rename from distribution/src/resources/dumpcat rename to distribution/src/main/resources/dumpcat diff --git a/distribution/src/resources/hadoop-excludes.txt b/distribution/src/main/resources/hadoop-excludes.txt similarity index 100% rename from distribution/src/resources/hadoop-excludes.txt rename to distribution/src/main/resources/hadoop-excludes.txt diff --git a/distribution/src/resources/logback.xml b/distribution/src/main/resources/logback.xml similarity index 99% rename from distribution/src/resources/logback.xml rename to distribution/src/main/resources/logback.xml index 7d95063..720826e 100644 --- a/distribution/src/resources/logback.xml +++ b/distribution/src/main/resources/logback.xml @@ -27,7 +27,7 @@ <RemoteHosts>${LILITH_HOSTNAME:-localhost}</RemoteHosts> </appender> --> - + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <encoder> <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n @@ -50,8 +50,8 @@ <pattern>%msg%n</pattern> </encoder> </appender> - - + + <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <file>${log.path}</file> <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> @@ -80,7 +80,7 @@ <!-- <appender-ref ref="SOCKET" /> --> </logger> - <!-- + <!-- <logger name="org.apache.drill" additivity="false"> <level value="debug" /> <appender-ref ref="SOCKET" /> diff --git a/distribution/src/resources/runbit b/distribution/src/main/resources/runbit similarity index 100% rename from distribution/src/resources/runbit rename to distribution/src/main/resources/runbit diff --git a/distribution/src/resources/saffron.properties b/distribution/src/main/resources/saffron.properties similarity index 100% rename from distribution/src/resources/saffron.properties rename to distribution/src/main/resources/saffron.properties diff --git a/distribution/src/resources/sqlline b/distribution/src/main/resources/sqlline similarity index 100% rename from distribution/src/resources/sqlline rename to distribution/src/main/resources/sqlline diff --git a/distribution/src/resources/sqlline.bat b/distribution/src/main/resources/sqlline.bat similarity index 96% rename from distribution/src/resources/sqlline.bat rename to distribution/src/main/resources/sqlline.bat index daf5afe..c5ab816 100755 --- a/distribution/src/resources/sqlline.bat +++ b/distribution/src/main/resources/sqlline.bat @@ -156,6 +156,7 @@ if "test%HADOOP_HOME%" == "test" ( set HADOOP_CLASSPATH=%HADOOP_HOME%\conf;!HADOOP_CLASSPATH! set USE_HADOOP_CP=1 ) +set PATH=!HADOOP_HOME!\bin;!PATH! rem ---- rem Deal with HBase JARs, if HBASE_HOME was specified diff --git a/distribution/src/resources/storage-plugins-override-example.conf b/distribution/src/main/resources/storage-plugins-override-example.conf similarity index 100% rename from distribution/src/resources/storage-plugins-override-example.conf rename to distribution/src/main/resources/storage-plugins-override-example.conf diff --git a/distribution/src/resources/submit_plan b/distribution/src/main/resources/submit_plan similarity index 100% rename from distribution/src/resources/submit_plan rename to distribution/src/main/resources/submit_plan diff --git a/distribution/src/main/resources/winutils/hadoop.dll b/distribution/src/main/resources/winutils/hadoop.dll new file mode 100644 index 0000000..e7ffeac Binary files /dev/null and b/distribution/src/main/resources/winutils/hadoop.dll differ diff --git a/distribution/src/main/resources/winutils/winutils.exe b/distribution/src/main/resources/winutils/winutils.exe new file mode 100644 index 0000000..027ca13 Binary files /dev/null and b/distribution/src/main/resources/winutils/winutils.exe differ diff --git a/distribution/src/resources/yarn-client-log.xml b/distribution/src/main/resources/yarn-client-log.xml similarity index 99% rename from distribution/src/resources/yarn-client-log.xml rename to distribution/src/main/resources/yarn-client-log.xml index feca2c9..7e64be6 100644 --- a/distribution/src/resources/yarn-client-log.xml +++ b/distribution/src/main/resources/yarn-client-log.xml @@ -25,7 +25,7 @@ See http://logback.qos.ch/manual/index.html for more information. --> <configuration> - + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <encoder> <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n diff --git a/distribution/src/resources/yarn-drillbit.sh b/distribution/src/main/resources/yarn-drillbit.sh similarity index 100% rename from distribution/src/resources/yarn-drillbit.sh rename to distribution/src/main/resources/yarn-drillbit.sh diff --git a/docs/dev/HadoopWinutils.md b/docs/dev/HadoopWinutils.md new file mode 100644 index 0000000..7b5b2bc --- /dev/null +++ b/docs/dev/HadoopWinutils.md @@ -0,0 +1,11 @@ +## Hadoop Winutils + +Hadoop Winutils native libraries are required to run Drill on Windows. The last version present in maven repository is 2.7.1 and is not updated anymore. +That's why Winutils version matching Hadoop version used in Drill is located in distribution/src/main/resources. + +Current Winutils version: *3.2.1.* + +## References +- Official wiki: [Windows Problems](https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems). +- Winutils compiling process is described [here](https://github.com/steveloughran/winutils). +- Actual versions are being uploaded [here](https://github.com/cdarlint/winutils) \ No newline at end of file diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 176b388..4d4cd59 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -454,20 +454,6 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </exclusion> -<!----> - <!--<exclusion>--> - <!--<groupId>com.sun.jersey</groupId>--> - <!--<artifactId>jersey-core</artifactId>--> - <!--</exclusion>--> - <!--<exclusion>--> - <!--<groupId>com.sun.jersey</groupId>--> - <!--<artifactId>jersey-server</artifactId>--> - <!--</exclusion>--> - <!--<exclusion>--> - <!--<groupId>com.sun.jersey</groupId>--> - <!--<artifactId>jersey-json</artifactId>--> - <!--</exclusion>--> -<!----> </exclusions> </dependency> <dependency> @@ -487,7 +473,6 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </exclusion> - <!----> <exclusion> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> @@ -500,7 +485,6 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-json</artifactId> </exclusion> - <!----> <exclusion> <groupId>log4j</groupId> <artifactId>log4j</artifactId> @@ -537,6 +521,10 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/exec/java-exec/src/main/java/org/apache/commons/logging/impl/Log4JLogger.java b/exec/java-exec/src/main/java/org/apache/commons/logging/impl/Log4JLogger.java new file mode 100644 index 0000000..4b41ea6 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/commons/logging/impl/Log4JLogger.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.logging.impl; + +/** + * A mock class to avoid NoClassDefFoundError after excluding Apache commons-logging from Hadoop dependency. + * See <a href="https://issues.apache.org/jira/browse/HADOOP-10288";>HADOOP-10288</a> for the problem description. + */ +public class Log4JLogger { +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java index 21d66f0..617d409 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/LocalSyncableFileSystem.java @@ -65,7 +65,7 @@ public class LocalSyncableFileSystem extends FileSystem { @Override public FSDataOutputStream create(Path path, FsPermission fsPermission, boolean b, int i, short i2, long l, Progressable progressable) throws IOException { - return new FSDataOutputStream(new LocalSyncableOutputStream(path), new Statistics(path.toUri().getScheme())); + return new FSDataOutputStream(new LocalSyncableOutputStream(path), FileSystem.getStatistics(path.toUri().getScheme(), getClass())); } @Override @@ -143,8 +143,7 @@ public class LocalSyncableFileSystem extends FileSystem { // TODO: remove it after upgrade MapR profile onto hadoop.version 3.1 public void sync() throws IOException { - output.flush(); - fos.getFD().sync(); + hflush(); } @Override @@ -155,8 +154,7 @@ public class LocalSyncableFileSystem extends FileSystem { @Override public void hflush() throws IOException { - output.flush(); - fos.getFD().sync(); + hsync(); } @Override diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java index 97ffb21..d7b6170 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/TestOutputBatchSize.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.physical.unit; +import org.apache.commons.lang3.StringUtils; import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.calcite.rel.core.JoinRelType; @@ -327,7 +328,7 @@ public class TestOutputBatchSize extends PhysicalOpUnitTestBase { expr[i * 2] = "lower(" + baselineColumns[i] + ")"; expr[i * 2 + 1] = baselineColumns[i]; } - baselineValues[i] = (transfer ? testString : testString.toLowerCase()); + baselineValues[i] = (transfer ? testString : StringUtils.lowerCase(testString)); } jsonRow.append("}"); StringBuilder batchString = new StringBuilder("["); @@ -384,7 +385,7 @@ public class TestOutputBatchSize extends PhysicalOpUnitTestBase { expr[i * 2] = "lower(" + baselineColumns[i] + ")"; expr[i * 2 + 1] = baselineColumns[i]; - baselineValues[i] = testString.toLowerCase(); + baselineValues[i] = StringUtils.lowerCase(testString); } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java index 04e59f6..c736ce0 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java @@ -43,7 +43,7 @@ public class FileTest { FSDataOutputStream out = fs.create(path); byte[] s = "hello world".getBytes(); out.write(s); - out.hsync(); + out.hflush(); FSDataInputStream in = fs.open(path); byte[] bytes = new byte[s.length]; in.read(bytes); @@ -60,7 +60,7 @@ public class FileTest { bytes = new byte[256*1024]; Stopwatch watch = Stopwatch.createStarted(); out.write(bytes); - out.hsync(); + out.hflush(); long t = watch.elapsed(TimeUnit.MILLISECONDS); logger.info(String.format("Elapsed: %d. Rate %d.\n", t, (long) ((long) bytes.length * 1000L / t))); } diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index d523606..97433fa 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -531,7 +531,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>42600000</maxsize> + <maxsize>43000000</maxsize> <minsize>15000000</minsize> <files> <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file> diff --git a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/DrillbitClassLoader.java b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/DrillbitClassLoader.java index eaedf56..072e269 100644 --- a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/DrillbitClassLoader.java +++ b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/DrillbitClassLoader.java @@ -33,9 +33,9 @@ public class DrillbitClassLoader extends URLClassLoader { private static final URL[] URLS; static { - ArrayList<URL> urlList = new ArrayList<>(); + List<URL> urlList = new ArrayList<>(); final String classPath = System.getProperty("app.class.path"); - final String[] st = fracture(classPath); + final String[] st = classPath.split(File.pathSeparator); final int l = st.length; for (int i = 0; i < l; i++) { try { @@ -47,42 +47,8 @@ public class DrillbitClassLoader extends URLClassLoader { assert false : e.toString(); } } - urlList.toArray(new URL[urlList.size()]); - List<URL> urls = new ArrayList<>(urlList); - URLS = urls.toArray(new URL[urls.size()]); - } - - /** - * Helper method to avoid StringTokenizer using. - * - * Taken from Apache Harmony - */ - private static String[] fracture(String str) { - if (str.length() == 0) { - return new String[0]; - } - ArrayList<String> res = new ArrayList<>(); - int in = 0; - int curPos = 0; - int i = str.indexOf(File.pathSeparator); - int len = File.pathSeparator.length(); - while (i != -1) { - String s = str.substring(curPos, i); - res.add(s); - in++; - curPos = i + len; - i = str.indexOf(File.pathSeparator, curPos); - } - - len = str.length(); - if (curPos <= len) { - String s = str.substring(curPos, len); - in++; - res.add(s); - } - - return res.toArray(new String[in]); + URLS = urlList.toArray(new URL[0]); } @Override diff --git a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java index 19a4be8..3557a4a 100644 --- a/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java +++ b/exec/jdbc-all/src/test/java/org/apache/drill/jdbc/ITTestShadedJar.java @@ -105,7 +105,6 @@ public class ITTestShadedJar extends BaseTest { super.failed(e, description); done(); runMethod("failed", description); - logger.error("Check whether this test was running within 'integration-test' Maven phase"); } private void done() { diff --git a/pom.xml b/pom.xml index e012ec2..cbc3548 100644 --- a/pom.xml +++ b/pom.xml @@ -82,8 +82,8 @@ Apache Hive 2.3.2. If the version is changed, make sure the jars and their dependencies are updated. --> <hive.version>2.3.2</hive.version> - <hadoop.version>3.0.3</hadoop.version> - <hbase.version>2.1.4</hbase.version> + <hadoop.version>3.2.1</hadoop.version> + <hbase.version>2.2.2</hbase.version> <fmpp.version>1.0</fmpp.version> <freemarker.version>2.3.28</freemarker.version> <javassist.version>3.25.0-GA</javassist.version> @@ -511,7 +511,7 @@ <rules> <bannedDependencies> <excludes> - <!--<exclude>commons-logging</exclude>--> + <exclude>commons-logging</exclude> <exclude>javax.servlet:servlet-api</exclude> <exclude>org.mortbay.jetty:servlet-api</exclude> <exclude>org.mortbay.jetty:servlet-api-2.5</exclude> @@ -1064,22 +1064,22 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>${dep.slf4j.version}</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>jul-to-slf4j</artifactId> - <version>${dep.slf4j.version}</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>jcl-over-slf4j</artifactId> - <version>${dep.slf4j.version}</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>log4j-over-slf4j</artifactId> - <version>${dep.slf4j.version}</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>${calcite.groupId}</groupId> @@ -1913,14 +1913,14 @@ <artifactId>mockito-all</artifactId> <groupId>org.mockito</groupId> </exclusion> - <!--<exclusion>--> - <!--<artifactId>commons-logging-api</artifactId>--> - <!--<groupId>commons-logging</groupId>--> - <!--</exclusion>--> - <!--<exclusion>--> - <!--<artifactId>commons-logging</artifactId>--> - <!--<groupId>commons-logging</groupId>--> - <!--</exclusion>--> + <exclusion> + <artifactId>commons-logging-api</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> <exclusion> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> @@ -1963,7 +1963,56 @@ </exclusion> </exclusions> </dependency> - <!-- Hadoop Test Dependencies --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>net.minidev</groupId> + <artifactId>json-smart</artifactId> + </exclusion> + <exclusion> + <groupId>com.nimbusds</groupId> + <artifactId>nimbus-jose-jwt</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-archives</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-registry</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-distcp</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> @@ -2093,6 +2142,9 @@ </exclusion> </exclusions> </dependency> + + <!-- Hadoop Test Dependencies --> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> @@ -2130,7 +2182,7 @@ </exclusion> </exclusions> </dependency> - <!-- Hadoop Test Dependencies --> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId>
