miniDFScluster for hdfs tests
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/f5f3543a Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/f5f3543a Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/f5f3543a Branch: refs/heads/steven/hdfs Commit: f5f3543a0c8b8d3839bfb04387b3a61500ad8baa Parents: e8c223c Author: efikalti <[email protected]> Authored: Mon Jul 13 18:48:39 2015 +0300 Committer: efikalti <[email protected]> Committed: Mon Jul 13 18:48:39 2015 +0300 ---------------------------------------------------------------------- .../java/org/apache/vxquery/xtest/Main.java | 4 +- .../java/org/apache/vxquery/xtest/XTest.java | 79 +++++++++++++++++--- .../org/apache/vxquery/xtest/XTestOptions.java | 5 +- .../TestSources/hdfs-test-data/AS000000003.xml | 18 +++++ .../hdfs-test-data/AS000000003_200303_0.xml | 18 +++++ .../TestSources/hdfs-test-data/US000000001.xml | 18 +++++ .../hdfs-test-data/US000000001_200101_0.xml | 18 +++++ .../TestSources/hdfs-test-data/US000000002.xml | 18 +++++ .../hdfs-test-data/US000000002_200202_0.xml | 18 +++++ .../TestSources/hdfs-test-data/US000000004.xml | 18 +++++ .../hdfs-test-data/US000000004_200404_0.xml | 18 +++++ .../test/resources/hadoop/conf/core-site.xml | 32 ++++++++ .../test/resources/hadoop/conf/hdfs-site.xml | 32 ++++++++ .../test/resources/hadoop/conf/mapred-site.xml | 39 ++++++++++ 14 files changed, 321 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/Main.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/Main.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/Main.java index dcb1b7e..83d56be 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/Main.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/Main.java @@ -30,7 +30,9 @@ public class Main { } XTest xts = new XTest(opts); + xts.setupHDFS(); xts.init(); xts.waitForCompletion(); + xts.shutdownDFS(); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java index cadae02..3b0b6a7 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java @@ -17,13 +17,21 @@ package org.apache.vxquery.xtest; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.vxquery.hdfs2.HDFSFunctions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.mapred.JobConf; import org.mortbay.jetty.Server; public class XTest { @@ -34,6 +42,7 @@ public class XTest { private TestRunnerFactory trf; private int count; private int finishCount; + private MiniDFSCluster dfsCluster; XTest(XTestOptions opts) { this.opts = opts; @@ -41,6 +50,7 @@ public class XTest { } void init() throws Exception { + finishCount = 0; if (opts.threads <= 0) { opts.threads = 1; @@ -62,13 +72,6 @@ public class XTest { if (opts.diffable != null) { reporters.add(new LineFileReporterImpl(new File(opts.diffable))); } - if (opts.hdfs) - { - //run tests for HDFS - //upload sources to hdfs - HDFSFunctions function = new HDFSFunctions(); - function.put("src/test/resources/TestSources/ghcnd", "vxquery-hdfs-test"); - } reporters.add(new ResultReporter() { @Override public void close() { @@ -85,9 +88,11 @@ public class XTest { } }); trf = new TestRunnerFactory(opts); + //setupHDFS(); trf.registerReporters(reporters); TestCaseFactory tcf = new TestCaseFactory(trf, eSvc, opts); count = tcf.process(); + } synchronized void waitForCompletion() throws InterruptedException { @@ -117,4 +122,60 @@ public class XTest { } } } -} \ No newline at end of file + + public void setupHDFS() throws IOException { + + FileSystem lfs = FileSystem.getLocal(new Configuration()); + JobConf conf = new JobConf(); + String PATH_TO_HADOOP_CONF = "vxquery-xtest/src/test/resources/hadoop/conf"; + Path hdfs_conf = new Path(PATH_TO_HADOOP_CONF); + if (!lfs.exists(hdfs_conf)) { + PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf"; + hdfs_conf = new Path(PATH_TO_HADOOP_CONF); + if (!lfs.exists(hdfs_conf)) { + PATH_TO_HADOOP_CONF = "../vxquery-xtest/src/test/resources/hadoop/conf"; + hdfs_conf = new Path(PATH_TO_HADOOP_CONF); + } + } + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); + conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); + int numDataNodes = 1; + int nameNodePort = 40000; + + // cleanup artifacts created on the local file system + lfs.delete(new Path("build"), true); + System.setProperty("hadoop.log.dir", "logs"); + MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); + build.nameNodePort(nameNodePort); + build.nameNodeHttpPort(nameNodePort + 34); + build.numDataNodes(numDataNodes); + build.checkExitOnShutdown(true); + build.startupOption(StartupOption.REGULAR); + build.format(true); + build.waitSafeMode(true); + dfsCluster = build.build(); + + FileSystem dfs = FileSystem.get(conf); + String DATA_PATH = "src/test/resources/TestSources/ghcnd"; + Path src = new Path(DATA_PATH); + if (!lfs.exists(src)) { + DATA_PATH = "vxquery-xtest/src/test/resources/TestSources/ghcnd"; + src = new Path(DATA_PATH); + if (!lfs.exists(src)) { + DATA_PATH = "../vxquery-xtest/src/test/resources/TestSources/ghcnd"; + src = new Path(DATA_PATH); + } + } + Path dest = new Path("vxquery-hdfs-test"); + dfs.copyFromLocalFile(src, dest); + if (dfs.exists(dest)) { + System.err.println("Test files copied to HDFS successfully"); + } + } + + public void shutdownDFS() { + System.err.println("Tests completed.Shutting down HDFS"); + dfsCluster.shutdown(); + } +} http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index 113fe9e..5ae4ba4 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -52,7 +52,4 @@ public class XTestOptions { @Option(name = "-htmlreport", required = false, usage = "HTML Report output file") String htmlReport; - - @Option(name = "-hdfs", required = false, usage = "run HDFS tests") - boolean hdfs; -} + } http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003.xml new file mode 100644 index 0000000..0f754c2 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<stationCollection pageSize="100" pageCount="1" totalCount="1"><station><id>GHCND:AS000000003</id><displayName>Station 3</displayName><latitude>-30.000</latitude><longitude>30.000</longitude><locationLabels><type>CNTRY</type><id>FIPS:AS</id><displayName>AUSTRALIA</displayName></locationLabels></station></stationCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003_200303_0.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003_200303_0.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003_200303_0.xml new file mode 100644 index 0000000..ec44bcd --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/AS000000003_200303_0.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<dataCollection pageCount="1" totalCount="2"><data><date>2003-03-03T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:AS000000003</station><value>13.75</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2003-03-03T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:AS000000003</station><value>33</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data></dataCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001.xml new file mode 100644 index 0000000..5479293 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<stationCollection pageSize="100" pageCount="1" totalCount="1"><station><id>GHCND:US000000001</id><displayName>Station 1</displayName><latitude>10.000</latitude><longitude>-10.000</longitude><elevation>1000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 1</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName>UNITED STATES</displayName></locationLabels></station></stationCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001_200101_0.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001_200101_0.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001_200101_0.xml new file mode 100644 index 0000000..30f2242 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000001_200101_0.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<dataCollection pageCount="1" totalCount="3"><data><date>2001-01-01T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000001</station><value>11.25</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2001-01-01T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000001</station><value>31</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2001-01-01T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000001</station><value>1000</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data></dataCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002.xml new file mode 100644 index 0000000..52b0797 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<stationCollection pageSize="100" pageCount="1" totalCount="1"><station><id>GHCND:US000000002</id><displayName>Station 2</displayName><latitude>20.000</latitude><longitude>-20.000</longitude><elevation>2000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 1</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 2</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName>UNITED STATES</displayName></locationLabels></station></stationCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002_200202_0.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002_200202_0.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002_200202_0.xml new file mode 100644 index 0000000..9f2695f --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000002_200202_0.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<dataCollection pageCount="1" totalCount="3"><data><date>2002-02-02T00:00:00.000</date><dataType>TMIN</dataType><station>GHCND:US000000002</station><value>12.5</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2002-02-02T00:00:00.000</date><dataType>TMAX</dataType><station>GHCND:US000000002</station><value>32</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2002-02-02T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000002</station><value>20</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data></dataCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004.xml new file mode 100644 index 0000000..15eef88 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<stationCollection pageSize="100" pageCount="1" totalCount="1"><station><id>GHCND:US000000004</id><displayName>Station 4</displayName><latitude>40.000</latitude><longitude>-40.000</longitude><elevation>4000.0</elevation><locationLabels><type>ST</type><id>FIPS:1</id><displayName>State 4</displayName></locationLabels><locationLabels><type>CNTY</type><id>FIPS:-9999</id><displayName>County 4</displayName></locationLabels><locationLabels><type>CNTRY</type><id>FIPS:US</id><displayName>UNITED STATES</displayName></locationLabels></station></stationCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004_200404_0.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004_200404_0.xml b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004_200404_0.xml new file mode 100644 index 0000000..d24fdf6 --- /dev/null +++ b/vxquery-xtest/src/test/resources/TestSources/hdfs-test-data/US000000004_200404_0.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<dataCollection pageCount="1" totalCount="2"><data><date>2004-04-04T00:00:00.000</date><dataType>PRCP</dataType><station>GHCND:US000000004</station><value>40</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data><data><date>2004-04-04T00:00:00.000</date><dataType>AWND</dataType><station>GHCND:US000000004</station><value>4</value><attributes><attribute></attribute><attribute></attribute><attribute>a</attribute><attribute></attribute></attributes></data></dataCollection> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml new file mode 100644 index 0000000..7684250 --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/core-site.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<!-- + ! Copyright 2009-2013 by The Regents of the University of California + ! Licensed under the Apache License, Version 2.0 (the "License"); + ! you may not use this file except in compliance with the License. + ! you may obtain a copy of the License from + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, software + ! distributed under the License is distributed on an "AS IS" BASIS, + ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ! See the License for the specific language governing permissions and + ! limitations under the License. + !--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +<property> + <name>fs.default.name</name> + <value>hdfs://localhost:40000</value> +</property> +<property> + <name>hadoop.tmp.dir</name> + <value>/tmp/hadoop</value> +</property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml new file mode 100644 index 0000000..4f3f777 --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/hdfs-site.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<!-- + ! Copyright 2009-2013 by The Regents of the University of California + ! Licensed under the Apache License, Version 2.0 (the "License"); + ! you may not use this file except in compliance with the License. + ! you may obtain a copy of the License from + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, software + ! distributed under the License is distributed on an "AS IS" BASIS, + ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ! See the License for the specific language governing permissions and + ! limitations under the License. + !--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + +<property> + <name>dfs.replication</name> + <value>1</value> +</property> + +<property> + <name>dfs.block.size</name> + <value>1048576</value> +</property> + +</configuration> http://git-wip-us.apache.org/repos/asf/vxquery/blob/f5f3543a/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml b/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml new file mode 100644 index 0000000..ab3b7c3 --- /dev/null +++ b/vxquery-xtest/src/test/resources/hadoop/conf/mapred-site.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<!-- + ! Copyright 2009-2013 by The Regents of the University of California + ! Licensed under the Apache License, Version 2.0 (the "License"); + ! you may not use this file except in compliance with the License. + ! you may obtain a copy of the License from + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, software + ! distributed under the License is distributed on an "AS IS" BASIS, + ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ! See the License for the specific language governing permissions and + ! limitations under the License. + !--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + + <property> + <name>mapred.job.tracker</name> + <value>localhost:29007</value> + </property> + <property> + <name>mapred.tasktracker.map.tasks.maximum</name> + <value>20</value> + </property> + <property> + <name>mapred.tasktracker.reduce.tasks.maximum</name> + <value>20</value> + </property> + <property> + <name>mapred.max.split.size</name> + <value>128</value> + </property> + +</configuration>
