DRILL-214: if HADOOP_HOME is set, add jars in hadoop lib directory to classpath. Minor packaging and script fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/10866cf0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/10866cf0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/10866cf0 Branch: refs/heads/master Commit: 10866cf06294842fce13c122724cee99c3f27a86 Parents: f5e2e61 Author: Steven Phillips <[email protected]> Authored: Tue Sep 3 20:11:35 2013 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Thu Sep 5 19:51:09 2013 -0700 ---------------------------------------------------------------------- distribution/src/assemble/bin.xml | 8 ++++++ distribution/src/resources/drill-config.sh | 19 +++++++++++++ distribution/src/resources/hadoop-excludes.txt | 13 +++++++++ distribution/src/resources/runbit | 5 ++-- distribution/src/resources/sqlline | 3 +- distribution/src/resources/storage-engines.json | 29 ++++++++++++++++++++ distribution/src/resources/submit_plan | 2 +- .../exec/store/parquet/ParquetGroupScan.java | 6 +--- .../store/parquet/ParquetSchemaProvider.java | 2 +- .../apache/drill/exec/work/FragmentRunner.java | 2 +- 10 files changed, 77 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index c00de72..8997a89 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -62,6 +62,10 @@ <outputDirectory>bin</outputDirectory> </file> <file> + <source>src/resources/hadoop-excludes.txt</source> + <outputDirectory>bin</outputDirectory> + </file> + <file> <source>src/resources/drillbit.sh</source> <outputDirectory>bin</outputDirectory> </file> @@ -89,5 +93,9 @@ <source>src/resources/drill-env.sh</source> <outputDirectory>conf</outputDirectory> </file> + <file> + <source>src/resources/storage-engines.json</source> + <outputDirectory>conf</outputDirectory> + </file> </files> </assembly> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/drill-config.sh ---------------------------------------------------------------------- diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh index 20102fc..7be5bad 100644 --- a/distribution/src/resources/drill-config.sh +++ b/distribution/src/resources/drill-config.sh @@ -66,8 +66,27 @@ done # Allow alternate drill conf dir location. export DRILL_CONF_DIR="${DRILL_CONF_DIR:-/etc/drill/conf}" +if [ ! -d $DRILL_CONF_DIR ] +then + export DRILL_CONF_DIR=$DRILL_HOME/conf +fi + . "${DRILL_CONF_DIR}/drill-env.sh" +if [ "${HADOOP_HOME}x" != "x" ] +then + HADOOP_CLASSPATH="" + for jar in `ls $HADOOP_HOME/lib/*jar` + do + echo $jar | grep -v -f $DRILL_HOME/bin/hadoop-excludes.txt >/dev/null + if [ "$?" -eq "0" ] + then + HADOOP_CLASSPATH=$jar:$HADOOP_CLASSPATH + fi + done + export HADOOP_CLASSPATH=$HADOOP_HOME/conf:$HADOOP_CLASSPATH +fi + # Newer versions of glibc use an arena memory allocator that causes virtual # memory usage to explode. Tune the variable down to prevent vmem explosion. export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/hadoop-excludes.txt ---------------------------------------------------------------------- diff --git a/distribution/src/resources/hadoop-excludes.txt b/distribution/src/resources/hadoop-excludes.txt new file mode 100644 index 0000000..d72efeb --- /dev/null +++ b/distribution/src/resources/hadoop-excludes.txt @@ -0,0 +1,13 @@ +asm +jackson +mockito +jets3t +log4j +logging +slf4j +jetty +jasper +jersey +eclipse +common +guava http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/runbit ---------------------------------------------------------------------- diff --git a/distribution/src/resources/runbit b/distribution/src/resources/runbit index 77f28a3..8fcb138 100755 --- a/distribution/src/resources/runbit +++ b/distribution/src/resources/runbit @@ -5,7 +5,7 @@ if [ -z $JAVA_HOME ] then JAVA=`which java` else - JAVA=`find $JAVA_HOME -name java` + JAVA=`find -L $JAVA_HOME -name java | head -n 1` fi if [ -e $JAVA ]; then @@ -21,10 +21,9 @@ if [ $? -ne 0 ]; then exit 1 fi - CP=$DRILL_HOME/jars/*:$CP CP=$DRILL_HOME/lib/*:$CP - CP=$CP:$DRILL_CONF_DIR +CP=$HADOOP_CLASSPATH:$CP exec $JAVA $DRILL_JAVA_OPTS -cp $CP org.apache.drill.exec.server.Drillbit http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/sqlline ---------------------------------------------------------------------- diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline index 5d13646..3915929 100755 --- a/distribution/src/resources/sqlline +++ b/distribution/src/resources/sqlline @@ -24,7 +24,7 @@ if [ -z $JAVA_HOME ] then JAVA=`which java` else - JAVA=`find $JAVA_HOME -name java` + JAVA=`find -L $JAVA_HOME -name java | head -n 1` fi if [ -e $JAVA ]; then @@ -49,6 +49,7 @@ fi CP=$DRILL_HOME/jars/*:$CP CP=$DRILL_HOME/lib/*:$CP CP=$DRILL_HOME/contrib/*:$CP +CP=$HADOOP_CLASSPATH:$CP CP=$DRILL_CONF_DIR:$CP http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/storage-engines.json ---------------------------------------------------------------------- diff --git a/distribution/src/resources/storage-engines.json b/distribution/src/resources/storage-engines.json new file mode 100644 index 0000000..d1d0413 --- /dev/null +++ b/distribution/src/resources/storage-engines.json @@ -0,0 +1,29 @@ +{ + "storage":{ + "parquet-local" : + { + "type":"parquet", + "dfsName" : "file:///" + }, + "parquet-cp" : + { + "type":"parquet", + "dfsName" : "classpath:///" + }, + "jsonl" : + { + "type":"json", + "dfsName" : "file:///" + }, + "json-cp" : + { + "type":"json", + "dfsName" : "classpath:///" + }, + "parquet" : + { + "type":"parquet", + "dfsName" : "file:///" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/distribution/src/resources/submit_plan ---------------------------------------------------------------------- diff --git a/distribution/src/resources/submit_plan b/distribution/src/resources/submit_plan index 6dc313c..f5d2bfd 100755 --- a/distribution/src/resources/submit_plan +++ b/distribution/src/resources/submit_plan @@ -24,7 +24,7 @@ if [ -z $JAVA_HOME ] then JAVA=`which java` else - JAVA=`find $JAVA_HOME -name java` + JAVA=`find -L $JAVA_HOME -name java | head -n 1` fi if [ -e $JAVA ]; then http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java index ac72116..df32ee5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java @@ -76,6 +76,7 @@ public class ParquetGroupScan extends AbstractGroupScan { return entries; } + @JsonProperty("storageengine") public ParquetStorageEngineConfig getEngineConfig() { return this.engineConfig; } @@ -360,11 +361,6 @@ public class ParquetGroupScan extends AbstractGroupScan { return new Size(10,10); } - @JsonProperty("storageengine") - public StorageEngineConfig getStorageEngineConfig(){ - return this.engineConfig; - } - @Override @JsonIgnore public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java index 0195bb8..74bebab 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java @@ -29,7 +29,7 @@ public class ParquetSchemaProvider implements SchemaProvider{ logger.debug("{}: {}",HADOOP_DEFAULT_NAME, configuration.getDfsName()); this.fs = FileSystem.get(conf); } catch (IOException ie) { - throw new RuntimeException("Error setting up filesystem"); + throw new RuntimeException("Error setting up filesystem", ie); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/10866cf0/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java index ec9f6db..8446be1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java @@ -92,7 +92,7 @@ public class FragmentRunner implements Runnable, CancelableQuery, StatusProvider } }catch(Exception ex){ - logger.debug("Caught exception while running fragment: {} ", ex); + logger.debug("Caught exception while running fragment", ex); internalFail(ex); }finally{ t.stop();
