DRILL-431: MapR profile - unit tests try to use localhost as the MapR cluster
This was caused by incorrect configuration key: "fs.name.default", but should be "fs.default.name" Replaced all occurrences of both with variable FileSystem.FS_DEFAULT_NAME_KEY also updated hadoo-core version number for mapr profile Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a43db154 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a43db154 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a43db154 Branch: refs/heads/master Commit: a43db154f6e9b3efc84a546dd01cbdc9c19f27a1 Parents: 1e25c0d Author: Steven Phillips <[email protected]> Authored: Mon Mar 24 15:20:09 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Wed Mar 26 23:39:39 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/physical/impl/trace/TraceRecordBatch.java | 2 +- .../org/apache/drill/exec/store/dfs/FileSystemPlugin.java | 3 ++- .../drill/exec/work/batch/SpoolingRawBatchBuffer.java | 3 +-- .../java/org/apache/drill/exec/cache/TestWriteToDisk.java | 2 +- .../test/java/org/apache/drill/exec/client/DumpCatTest.java | 2 +- .../drill/exec/physical/impl/trace/TestTraceOutputDump.java | 2 +- .../apache/drill/exec/store/hive/HiveTestDataGenerator.java | 3 ++- .../apache/drill/exec/store/parquet/TestFileGenerator.java | 2 +- .../java/org/apache/drill/exec/work/batch/FileTest.java | 2 +- pom.xml | 9 ++------- 10 files changed, 13 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java index c9a73f9..88667ea 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/trace/TraceRecordBatch.java @@ -77,7 +77,7 @@ public class TraceRecordBatch extends AbstractSingleRecordBatch<Trace> { /* Create the log file we will dump to and initialize the file descriptors */ try { Configuration conf = new Configuration(); - conf.set("fs.name.default", ExecConstants.TRACE_DUMP_FILESYSTEM); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, context.getConfig().getString(ExecConstants.TRACE_DUMP_FILESYSTEM)); FileSystem fs = FileSystem.get(conf); /* create the file */ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java index 3762c1a..d25d501 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration; import com.beust.jcommander.internal.Lists; import com.beust.jcommander.internal.Maps; +import org.apache.hadoop.fs.FileSystem; /** * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem, @@ -61,7 +62,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{ this.context = context; Configuration fsConf = new Configuration(); - fsConf.set("fs.default.name", config.connection); + fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection); fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName()); this.fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf); this.formatsByName = FormatCreator.getFormatPlugins(context, fs, config); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java index d7c407b..afac86f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java @@ -53,7 +53,6 @@ import com.google.common.collect.Queues; public class SpoolingRawBatchBuffer implements RawBatchBuffer { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SpoolingRawBatchBuffer.class); - private static String HADOOP_FILESYSTEM_DEFAULT_NAME = "fs.default.name"; private static String DRILL_LOCAL_IMPL_STRING = "fs.drill-local.impl"; private static final float STOP_SPOOLING_FRACTION = (float) 0.5; @@ -72,7 +71,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer { this.context = context; this.threshold = context.getConfig().getLong(ExecConstants.SPOOLING_BUFFER_MEMORY); Configuration conf = new Configuration(); - conf.set(HADOOP_FILESYSTEM_DEFAULT_NAME, context.getConfig().getString(ExecConstants.TEMP_FILESYSTEM)); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, context.getConfig().getString(ExecConstants.TEMP_FILESYSTEM)); conf.set(DRILL_LOCAL_IMPL_STRING, LocalSyncableFileSystem.class.getName()); this.fs = FileSystem.get(conf); this.path = new Path(getDir(), getFileName()); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java index fb3e821..52e8425 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/cache/TestWriteToDisk.java @@ -75,7 +75,7 @@ public class TestWriteToDisk { VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, context.getAllocator()); Configuration conf = new Configuration(); - conf.set("fs.name.default", "file:///"); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); FileSystem fs = FileSystem.get(conf); Path path = new Path("/tmp/drillSerializable"); if (fs.exists(path)) fs.delete(path, false); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java index c513cd0..9545845 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java @@ -103,7 +103,7 @@ public class DumpCatTest { System.out.println("File Name: " + filename); Configuration conf = new Configuration(); - conf.set("fs.name.default", c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM)); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM)); FileSystem fs = FileSystem.get(conf); Path path = new Path(filename); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java index f166fad..910d782 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/trace/TestTraceOutputDump.java @@ -114,7 +114,7 @@ public class TestTraceOutputDump { System.out.println("File Name: " + filename); Configuration conf = new Configuration(); - conf.set("fs.name.default", c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM)); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, c.getString(ExecConstants.TRACE_DUMP_FILESYSTEM)); FileSystem fs = FileSystem.get(conf); Path path = new Path(filename); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 87b14ed..8a71e2a 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.PrintWriter; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; @@ -59,7 +60,7 @@ public class HiveTestDataGenerator { HiveConf conf = new HiveConf(); conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", DB_DIR)); - conf.set("fs.default.name", "file:///"); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set("hive.metastore.warehouse.dir", WH_DIR); String tableName = "kv"; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java index d8892dc..2d2a2ec 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java @@ -108,7 +108,7 @@ public class TestFileGenerator { WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7); Configuration configuration = new Configuration(); - configuration.set("fs.default.name", "file:///"); + configuration.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}" FileSystem fs = FileSystem.get(configuration); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java index 4406e04..6248f57 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/batch/FileTest.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; public class FileTest { public static void main(String[] args) throws IOException { Configuration conf = new Configuration(); - conf.set("fs.default.name", "sync:///"); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "sync:///"); System.out.println(FileSystem.getDefaultUri(conf)); FileSystem fs = FileSystem.get(conf); // FileSystem fs = new LocalSyncableFileSystem(conf); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a43db154/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 76523bf..5998f99 100644 --- a/pom.xml +++ b/pom.xml @@ -608,7 +608,7 @@ <id>mapr</id> <properties> <alt-hadoop>mapr</alt-hadoop> - <rat.excludeSubprojects>false</rat.excludeSubprojects> + <rat.excludeSubprojects>true</rat.excludeSubprojects> </properties> <dependencyManagement> <dependencies> @@ -623,11 +623,6 @@ <version>2.7.1</version> </dependency> <dependency> - <groupId>com.mapr.hadoop</groupId> - <artifactId>maprfs</artifactId> - <version>1.0.3-mapr-3.0.0</version> - </dependency> - <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase</artifactId> <version>0.94.13-mapr-1401-m7-3.0.2</version> @@ -641,7 +636,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-core</artifactId> - <version>1.0.3-mapr-3.0.0</version> + <version>1.0.3-mapr-3.0.2</version> <exclusions> <exclusion> <artifactId>asm</artifactId>
