http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js deleted file mode 100644 index d8b79df..0000000 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js +++ /dev/null @@ -1 +0,0 @@ -(function(B){var A={series:{threshold:null}};function C(D){function E(L,S,M){if(!S.threshold){return }var F=M.pointsize,I,O,N,G,K,H=B.extend({},S);H.datapoints={points:[],pointsize:F};H.label=null;H.color=S.threshold.color;H.threshold=null;H.originSeries=S;H.data=[];var P=S.threshold.below,Q=M.points,R=S.lines.show;threspoints=[];newpoints=[];for(I=0;I<Q.length;I+=F){O=Q[I];N=Q[I+1];K=G;if(N<P){G=threspoints}else{G=newpoints}if(R&&K!=G&&O!=null&&I>0&&Q[I-F]!=null){var J=(O-Q[I-F])/(N-Q[I-F+1])*(P-N)+O;K.push(J);K.push(P);for(m=2;m<F;++m){K.push(Q[I+m])}G.push(null);G.push(null);for(m=2;m<F;++m){G.push(Q[I+m])}G.push(J);G.push(P);for(m=2;m<F;++m){G.push(Q[I+m])}}G.push(O);G.push(N)}M.points=newpoints;H.datapoints.points=threspoints;if(H.datapoints.points.length>0){L.getData().push(H)}}D.hooks.processDatapoints.push(E)}B.plot.plugins.push({init:C,options:A,name:"threshold",version:"1.0"})})(jQuery); \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml index 8caa9ec..6cfaaa5 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml @@ -41,6 +41,20 @@ <root.dir>${basedir}/../..</root.dir> </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/test/resources/data/beer.txt</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + <dependencies> <dependency> <groupId>org.apache.hyracks</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml index 73086cd..b88d292 100644 --- a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml @@ -50,7 +50,7 @@ <executions> <execution> <id>copy-scripts</id> - <phase>package</phase> + <phase>generate-test-resources</phase> <goals> <goal>copy-resources</goal> </goals> @@ -61,7 +61,6 @@ <directory>src/main/resources</directory> </resource> </resources> - <directoryMode>0755</directoryMode> </configuration> </execution> </executions> @@ -73,7 +72,7 @@ <executions> <execution> <id>process-test-classes</id> - <phase>package</phase> + <phase>process-test-resources</phase> <configuration> <target> <chmod file="target/appassembler/bin/*" perm="755" /> @@ -85,6 +84,16 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/resources/conf/master</exclude> + <exclude>src/main/resources/conf/slaves</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml index d1f1c70..21e22ab 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml @@ -37,6 +37,15 @@ <skip>true</skip> </configuration> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>data/**</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> <properties> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml index 1327912..394fa11 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml @@ -99,6 +99,16 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>data/file1.txt</exclude> + <exclude>data/file2.txt</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> <profiles> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml index 9c25d13..b4abc4a 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml @@ -33,25 +33,6 @@ <build> <plugins> <plugin> - <artifactId>maven-clean-plugin</artifactId> - <version>2.5</version> - <configuration> - <filesets> - <fileset> - <directory>.</directory> - <includes> - <include>edu*</include> - <include>actual*</include> - <include>build*</include> - <include>expect*</include> - <include>ClusterController*</include> - <include>edu.uci.*</include> - </includes> - </fileset> - </filesets> - </configuration> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <version>2.2</version> @@ -81,6 +62,16 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/test/resources/data/customer.tbl</exclude> + <exclude>src/test/resources/expected/part-0</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> @@ -409,5 +400,11 @@ <artifactId>commons-io</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-test-support</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java index c5ca25e..2307a43 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java @@ -19,6 +19,8 @@ package org.apache.hyracks.hdfs.dataflow; +import static org.apache.hyracks.test.support.TestUtils.joinPath; + import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -67,15 +69,19 @@ import org.apache.hyracks.hdfs.utils.TestUtils; @SuppressWarnings({ "deprecation" }) public class DataflowTest extends TestCase { - private static final String ACTUAL_RESULT_DIR = "actual"; - private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected"; - private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf"; + protected static final String ACTUAL_RESULT_DIR = joinPath("target", "actual"); + private static final String TEST_RESOURCES = joinPath("src", "test", "resources"); + protected static final String EXPECTED_RESULT_PATH = joinPath(TEST_RESOURCES, "expected"); + private static final String PATH_TO_HADOOP_CONF = joinPath(TEST_RESOURCES, "hadoop", "conf"); + protected static final String BUILD_DIR = joinPath("target", "build"); + - private static final String DATA_PATH = "src/test/resources/data/customer.tbl"; - private static final String HDFS_INPUT_PATH = "/customer/"; - private static final String HDFS_OUTPUT_PATH = "/customer_result/"; + private static final String DATA_PATH = joinPath(TEST_RESOURCES, "data", "customer.tbl"); + protected static final String HDFS_INPUT_PATH = "/customer/"; + protected static final String HDFS_OUTPUT_PATH = "/customer_result/"; private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml"; + private static final String MINIDFS_BASEDIR = joinPath("target", "hdfs"); private MiniDFSCluster dfsCluster; private JobConf conf = new JobConf(); @@ -91,10 +97,16 @@ public class DataflowTest extends TestCase { } private void cleanupStores() throws IOException { - FileUtils.forceMkdir(new File("teststore")); - FileUtils.forceMkdir(new File("build")); - FileUtils.cleanDirectory(new File("teststore")); - FileUtils.cleanDirectory(new File("build")); + FileUtils.forceMkdir(new File(MINIDFS_BASEDIR)); + FileUtils.cleanDirectory(new File(MINIDFS_BASEDIR)); + } + + protected Configuration getConfiguration() { + return conf; + } + + protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws IOException { + return new MiniDFSCluster(conf, numberOfNC, true, null); } /** @@ -102,16 +114,17 @@ public class DataflowTest extends TestCase { * * @throws IOException */ - private void startHDFS() throws IOException { - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); - conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); + protected void startHDFS() throws IOException { + getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); + getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); + getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); FileSystem lfs = FileSystem.getLocal(new Configuration()); - lfs.delete(new Path("build"), true); - System.setProperty("hadoop.log.dir", "logs"); - dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null); - FileSystem dfs = FileSystem.get(conf); + lfs.delete(new Path(BUILD_DIR), true); + System.setProperty("hadoop.log.dir", joinPath("target", "logs")); + getConfiguration().set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR); + dfsCluster = getMiniDFSCluster(getConfiguration(), numberOfNC); + FileSystem dfs = FileSystem.get(getConfiguration()); Path src = new Path(DATA_PATH); Path dest = new Path(HDFS_INPUT_PATH); Path result = new Path(HDFS_OUTPUT_PATH); @@ -120,7 +133,7 @@ public class DataflowTest extends TestCase { dfs.copyFromLocalFile(src, dest); DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH))); - conf.writeXml(confOutput); + getConfiguration().writeXml(confOutput); confOutput.flush(); confOutput.close(); } @@ -178,14 +191,14 @@ public class DataflowTest extends TestCase { * @return true if correct * @throws Exception */ - private boolean checkResults() throws Exception { - FileSystem dfs = FileSystem.get(conf); + protected boolean checkResults() throws Exception { + FileSystem dfs = FileSystem.get(getConfiguration()); Path result = new Path(HDFS_OUTPUT_PATH); Path actual = new Path(ACTUAL_RESULT_DIR); dfs.copyToLocalFile(result, actual); - TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File( - ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0")); + TestUtils.compareWithResult(new File(joinPath(EXPECTED_RESULT_PATH, "part-0")), new File( + joinPath(ACTUAL_RESULT_DIR, "customer_result", "part-0"))); return true; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java index 09e3a35..d522b2b 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java @@ -29,7 +29,7 @@ import org.apache.hyracks.api.client.NodeControllerInfo; import org.apache.hyracks.api.client.NodeStatus; import org.apache.hyracks.api.comm.NetworkAddress; -public class TestUtils { +public class TestUtils extends org.apache.hyracks.test.support.TestUtils { public static void compareWithResult(File expectedFile, File actualFile) throws Exception { BufferedReader readerExpected = new BufferedReader(new FileReader(expectedFile)); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java index fb6f32e..3c9b1c0 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java @@ -19,18 +19,10 @@ package org.apache.hyracks.hdfs2.dataflow; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.util.List; import junit.framework.Assert; -import junit.framework.TestCase; - -import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.InputFormat; @@ -40,7 +32,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.ReflectionUtils; - import org.apache.hyracks.api.client.HyracksConnection; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.constraints.PartitionConstraintHelper; @@ -48,6 +39,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer; @@ -61,73 +53,32 @@ import org.apache.hyracks.hdfs.lib.RawBinaryHashFunctionFactory; import org.apache.hyracks.hdfs.lib.TextKeyValueParserFactory; import org.apache.hyracks.hdfs.lib.TextTupleWriterFactory; import org.apache.hyracks.hdfs.utils.HyracksUtils; -import org.apache.hyracks.hdfs.utils.TestUtils; import org.apache.hyracks.hdfs2.scheduler.Scheduler; /** * Test the org.apache.hyracks.hdfs2.dataflow package, * the operators for the Hadoop new API. */ -public class DataflowTest extends TestCase { - - private static final String ACTUAL_RESULT_DIR = "actual"; - private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected"; - private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf"; +public class DataflowTest extends org.apache.hyracks.hdfs.dataflow.DataflowTest { - private static final String DATA_PATH = "src/test/resources/data/customer.tbl"; - private static final String HDFS_INPUT_PATH = "/customer/"; - private static final String HDFS_OUTPUT_PATH = "/customer_result/"; - - private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml"; - private MiniDFSCluster dfsCluster; private MiniDFSClusterFactory dfsClusterFactory = new MiniDFSClusterFactory(); private Job conf; - private int numberOfNC = 2; @Override public void setUp() throws Exception { conf = new Job(); - cleanupStores(); - HyracksUtils.init(); - FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR)); - FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR)); - startHDFS(); + super.setUp(); } - private void cleanupStores() throws IOException { - FileUtils.forceMkdir(new File("teststore")); - FileUtils.forceMkdir(new File("build")); - FileUtils.cleanDirectory(new File("teststore")); - FileUtils.cleanDirectory(new File("build")); + @Override + protected Configuration getConfiguration() { + return conf.getConfiguration(); } - /** - * Start the HDFS cluster and setup the data files - * - * @throws IOException - */ - private void startHDFS() throws IOException { - conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); - conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); - conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); - - FileSystem lfs = FileSystem.getLocal(new Configuration()); - lfs.delete(new Path("build"), true); - System.setProperty("hadoop.log.dir", "logs"); - dfsCluster = dfsClusterFactory.getMiniDFSCluster(conf.getConfiguration(), numberOfNC); - FileSystem dfs = FileSystem.get(conf.getConfiguration()); - Path src = new Path(DATA_PATH); - Path dest = new Path(HDFS_INPUT_PATH); - Path result = new Path(HDFS_OUTPUT_PATH); - dfs.mkdirs(dest); - dfs.mkdirs(result); - dfs.copyFromLocalFile(src, dest); - - DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH))); - conf.getConfiguration().writeXml(confOutput); - confOutput.flush(); - confOutput.close(); + @Override + protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws HyracksDataException { + return dfsClusterFactory.getMiniDFSCluster(conf, numberOfNC); } /** @@ -142,7 +93,7 @@ public class DataflowTest extends TestCase { conf.setInputFormatClass(TextInputFormat.class); Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT); - InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), conf.getConfiguration()); + InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), getConfiguration()); List<InputSplit> splits = inputFormat.getSplits(conf); String[] readSchedule = scheduler.getLocationConstraints(splits); @@ -178,35 +129,4 @@ public class DataflowTest extends TestCase { Assert.assertEquals(true, checkResults()); } - - /** - * Check if the results are correct - * - * @return true if correct - * @throws Exception - */ - private boolean checkResults() throws Exception { - FileSystem dfs = FileSystem.get(conf.getConfiguration()); - Path result = new Path(HDFS_OUTPUT_PATH); - Path actual = new Path(ACTUAL_RESULT_DIR); - dfs.copyToLocalFile(result, actual); - - TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File( - ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0")); - return true; - } - - /** - * cleanup hdfs cluster - */ - private void cleanupHDFS() throws Exception { - dfsCluster.shutdown(); - } - - @Override - public void tearDown() throws Exception { - HyracksUtils.deinit(); - cleanupHDFS(); - } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml index 1628544..be43c5d 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml @@ -1,4 +1,5 @@ <?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- ! Licensed to the Apache Software Foundation (ASF) under one ! or more contributor license agreements. See the NOTICE file @@ -18,8 +19,6 @@ ! under the License. !--> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - <!-- Put site-specific property overrides in this file. --> <configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml index 8784730..9dca49a 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml @@ -1,4 +1,5 @@ <?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- ! Licensed to the Apache Software Foundation (ASF) under one ! or more contributor license agreements. See the NOTICE file @@ -18,8 +19,6 @@ ! under the License. !--> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - <!-- Put site-specific property overrides in this file. --> <configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml index 3e8bfe4..e765c65 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml @@ -1,4 +1,5 @@ <?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <!-- ! Licensed to the Apache Software Foundation (ASF) under one ! or more contributor license agreements. See the NOTICE file @@ -18,8 +19,6 @@ ! under the License. !--> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - <!-- Put site-specific property overrides in this file. --> <configuration> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java deleted file mode 100644 index e031a51..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.io.IOException; -import java.util.Random; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; - -/** - * Quick & dirty data generator for multi-thread testing. - */ -@SuppressWarnings("rawtypes") -public class DataGenThread extends Thread { - public final BlockingQueue<TupleBatch> tupleBatchQueue; - private final int maxNumBatches; - private final int maxOutstandingBatches; - private int numBatches = 0; - private final Random rnd; - - // maxOutstandingBatches pre-created tuple-batches for populating the queue. - private TupleBatch[] tupleBatches; - private int ringPos; - - public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes, - int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) { - this.maxNumBatches = maxNumBatches; - this.maxOutstandingBatches = maxOutstandingBatches; - rnd = new Random(rndSeed); - tupleBatches = new TupleBatch[maxOutstandingBatches]; - IFieldValueGenerator[] fieldGens = DataGenUtils.getFieldGensFromSerdes(fieldSerdes, rnd, sorted); - for (int i = 0; i < maxOutstandingBatches; i++) { - tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize); - } - tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches); - ringPos = 0; - } - - public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes, - IFieldValueGenerator[] fieldGens, int rndSeed, int maxOutstandingBatches) { - this.maxNumBatches = maxNumBatches; - this.maxOutstandingBatches = maxOutstandingBatches; - rnd = new Random(rndSeed); - tupleBatches = new TupleBatch[maxOutstandingBatches]; - for (int i = 0; i < maxOutstandingBatches; i++) { - tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, 0); - } - tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches); - ringPos = 0; - } - - @Override - public void run() { - while (numBatches < maxNumBatches) { - boolean added = false; - try { - if (tupleBatches[ringPos].inUse.compareAndSet(false, true)) { - tupleBatches[ringPos].generate(); - tupleBatchQueue.put(tupleBatches[ringPos]); - added = true; - } - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (added) { - numBatches++; - ringPos++; - if (ringPos >= maxOutstandingBatches) { - ringPos = 0; - } - } - } - } - - public TupleBatch getBatch() throws InterruptedException { - return tupleBatchQueue.take(); - } - - public void releaseBatch(TupleBatch batch) { - batch.inUse.set(false); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java deleted file mode 100644 index 130f7e2..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Random; - -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; -import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer; -import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer; -import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer; -import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer; - -@SuppressWarnings("rawtypes") -public class DataGenUtils { - public static IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde, Random rnd, boolean sorted) { - if (serde instanceof IntegerSerializerDeserializer) { - if (sorted) { - return new SortedIntegerFieldValueGenerator(); - } else { - return new IntegerFieldValueGenerator(rnd); - } - } else if (serde instanceof FloatSerializerDeserializer) { - if (sorted) { - return new SortedFloatFieldValueGenerator(); - } else { - return new FloatFieldValueGenerator(rnd); - } - } else if (serde instanceof DoubleSerializerDeserializer) { - if (sorted) { - return new SortedDoubleFieldValueGenerator(); - } else { - return new DoubleFieldValueGenerator(rnd); - } - } else if (serde instanceof UTF8StringSerializerDeserializer) { - return new StringFieldValueGenerator(20, rnd); - } - return null; - } - - public static IFieldValueGenerator[] getFieldGensFromSerdes(ISerializerDeserializer[] serdes, Random rnd, - boolean sorted) { - IFieldValueGenerator[] fieldValueGens = new IFieldValueGenerator[serdes.length]; - for (int i = 0; i < serdes.length; i++) { - fieldValueGens[i] = getFieldGenFromSerde(serdes[i], rnd, sorted); - } - return fieldValueGens; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java deleted file mode 100644 index 1e8077c..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.hyracks.util.MathUtil; - -public class DocumentStringFieldValueGenerator implements IFieldValueGenerator<String> { - private final String FIRST_NAMES_FILE = "dist.all.first.cleaned"; - private final String LAST_NAMES_FILE = "dist.all.last.cleaned"; - - private final int docMinWords; - private final int docMaxWords; - private final int maxDictionarySize; - private final Random rnd; - private int[] cumulIntRanges; - - private List<String> tokenDict = new ArrayList<String>(); - - public DocumentStringFieldValueGenerator(int docMinWords, int docMaxWords, int maxDictionarySize, Random rnd) - throws IOException { - this.docMinWords = docMinWords; - this.docMaxWords = docMaxWords; - this.maxDictionarySize = maxDictionarySize; - this.rnd = rnd; - initDictionary(); - double[] zipfProbDist = ProbabilityHelper.getZipfProbDist(tokenDict.size(), 1); - cumulIntRanges = ProbabilityHelper.getCumulIntRanges(zipfProbDist); - } - - private void initDictionary() throws IOException { - String line; - int count = 0; - - // Read first names from data file. - InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE); - BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn)); - try { - while (count < maxDictionarySize && (line = firstNamesReader.readLine()) != null) { - tokenDict.add(line.trim()); - count++; - } - } finally { - firstNamesReader.close(); - } - - // Read last names from data file. - InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE); - BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn)); - try { - while (count < maxDictionarySize && (line = lastNamesReader.readLine()) != null) { - tokenDict.add(line.trim()); - count++; - } - } finally { - lastNamesReader.close(); - } - } - - @Override - public String next() { - StringBuilder strBuilder = new StringBuilder(); - int numWords = MathUtil.stripSignBit(rnd.nextInt()) % (docMaxWords - docMinWords + 1) + docMinWords; - for (int i = 0; i < numWords; i++) { - int ix = ProbabilityHelper.choose(cumulIntRanges, rnd.nextInt()); - strBuilder.append(tokenDict.get(ix)); - if (i != numWords - 1) { - strBuilder.append(" "); - } - } - return strBuilder.toString(); - } - - public List<String> getTokenDictionary() { - return tokenDict; - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java deleted file mode 100644 index 8e36335..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Random; - -public class DoubleFieldValueGenerator implements IFieldValueGenerator<Double> { - protected final Random rnd; - - public DoubleFieldValueGenerator(Random rnd) { - this.rnd = rnd; - } - - @Override - public Double next() { - return rnd.nextDouble(); - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java deleted file mode 100644 index 851a0f4..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Random; - -public class FloatFieldValueGenerator implements IFieldValueGenerator<Float> { - protected final Random rnd; - - public FloatFieldValueGenerator(Random rnd) { - this.rnd = rnd; - } - - @Override - public Float next() { - return rnd.nextFloat(); - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java deleted file mode 100644 index dbd4bfc..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -public interface IFieldValueGenerator<T> { - public T next(); - public void reset(); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java deleted file mode 100644 index 6c5f649..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Random; - -public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> { - protected final Random rnd; - - public IntegerFieldValueGenerator(Random rnd) { - this.rnd = rnd; - } - - @Override - public Integer next() { - return rnd.nextInt(); - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java deleted file mode 100644 index 5af0b0f..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.hyracks.util.MathUtil; - -public class PersonNameFieldValueGenerator implements IFieldValueGenerator<String> { - private final String FIRST_NAMES_FILE = "dist.all.first.cleaned"; - private final String LAST_NAMES_FILE = "dist.all.last.cleaned"; - - private final Random rnd; - private final double middleInitialProb; - private final String letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - private List<String> firstNames = new ArrayList<String>(); - private List<String> lastNames = new ArrayList<String>(); - - public PersonNameFieldValueGenerator(Random rnd, double middleInitialProb) - throws IOException { - this.rnd = rnd; - this.middleInitialProb = middleInitialProb; - initNames(); - } - - private void initNames() throws IOException { - String line; - - // Read first names from data file. - InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE); - BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn)); - try { - while ((line = firstNamesReader.readLine()) != null) { - firstNames.add(line.trim()); - } - } finally { - firstNamesReader.close(); - } - - // Read last names from data file. - InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE); - BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn)); - try { - while ((line = lastNamesReader.readLine()) != null) { - lastNames.add(line.trim()); - } - } finally { - lastNamesReader.close(); - } - } - - @Override - public String next() { - StringBuilder strBuilder = new StringBuilder(); - - // First name. - int fix = MathUtil.stripSignBit(rnd.nextInt()) % firstNames.size(); - strBuilder.append(firstNames.get(fix)); - strBuilder.append(" "); - - // Optional middle initial. - double d = Math.abs(rnd.nextDouble()); - if (d <= middleInitialProb) { - int mix = MathUtil.stripSignBit(rnd.nextInt()) % letters.length(); - strBuilder.append(letters.charAt(mix)); - strBuilder.append(". "); - } - - // Last name. - int lix = MathUtil.stripSignBit(rnd.nextInt()) % lastNames.size(); - strBuilder.append(lastNames.get(lix)); - - return strBuilder.toString(); - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java deleted file mode 100644 index c7fbd3f..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Arrays; - -public class ProbabilityHelper { - public static double[] getUniformProbDist(int numChoices) { - double[] probDist = new double[numChoices]; - for (int i = 0; i < numChoices; i++) { - probDist[i] = 1.0 / (double) numChoices; - } - return probDist; - } - - public static double[] getZipfProbDist(int numChoices, int zipfSkew) { - double[] probDist = new double[numChoices]; - double divisor = 0; - for (int i = 1; i <= numChoices; i++) { - divisor += 1.0 / (double) Math.pow((double) i, (double) zipfSkew); - } - for (int i = 1; i <= numChoices; i++) { - probDist[i - 1] = (1.0 / (double) Math.pow((double) i, (double) zipfSkew)) / divisor; - } - return probDist; - } - - public static int[] getCumulIntRanges(double[] probDist) { - int[] opRanges = new int[probDist.length]; - if (opRanges.length > 1) { - opRanges[0] = (int) Math.floor(Integer.MAX_VALUE * probDist[0]); - for (int i = 1; i < opRanges.length - 1; i++) { - opRanges[i] = opRanges[i - 1] + (int) Math.floor(Integer.MAX_VALUE * probDist[i]); - } - opRanges[opRanges.length - 1] = Integer.MAX_VALUE; - } else { - opRanges[0] = Integer.MAX_VALUE; - } - return opRanges; - } - - public static int choose(int[] cumulIntRanges, int randomInt) { - int rndVal = Math.abs(randomInt); - int ix = Arrays.binarySearch(cumulIntRanges, rndVal); - if (ix < 0) { - ix = -ix - 1; - } - return ix; - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java deleted file mode 100644 index ddca6f3..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -public class SortedDoubleFieldValueGenerator implements IFieldValueGenerator<Double> { - private double val; - private final double startVal; - - public SortedDoubleFieldValueGenerator() { - startVal = 0.0d; - reset(); - } - - public SortedDoubleFieldValueGenerator(double startVal) { - this.startVal = startVal; - reset(); - } - - @Override - public Double next() { - return val++; - } - - @Override - public void reset() { - val = startVal; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java deleted file mode 100644 index 1e5dd19..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -public class SortedFloatFieldValueGenerator implements IFieldValueGenerator<Float> { - private float val = 0.0f; - private final float startVal; - - public SortedFloatFieldValueGenerator() { - startVal = 0.0f; - reset(); - } - - public SortedFloatFieldValueGenerator(float startVal) { - this.startVal = startVal; - reset(); - } - - @Override - public Float next() { - return val++; - } - - @Override - public void reset() { - val = startVal; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java deleted file mode 100644 index 3ab17ca..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> { - private int val = 0; - private final int startVal; - - public SortedIntegerFieldValueGenerator() { - startVal = 0; - reset(); - } - - public SortedIntegerFieldValueGenerator(int startVal) { - this.startVal = startVal; - reset(); - } - - @Override - public Integer next() { - return val++; - } - - @Override - public void reset() { - val = startVal; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java deleted file mode 100644 index 9418447..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.util.Random; - -import org.apache.hyracks.util.MathUtil; - -public class StringFieldValueGenerator implements IFieldValueGenerator<String> { - private int maxLen; - private final Random rnd; - - public StringFieldValueGenerator(int maxLen, Random rnd) { - this.maxLen = maxLen; - this.rnd = rnd; - } - - public void setMaxLength(int maxLen) { - this.maxLen = maxLen; - } - - @Override - public String next() { - String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble())); - StringBuilder strBuilder = new StringBuilder(); - for (int i = 0; i < s.length() && i < maxLen; i++) { - strBuilder.append(s.charAt(MathUtil.stripSignBit(rnd.nextInt()) % s.length())); - } - return strBuilder.toString(); - } - - @Override - public void reset() { - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java deleted file mode 100644 index c34c7bc..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; -import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; - -@SuppressWarnings("rawtypes") -public class TupleBatch { - private final int size; - private final TupleGenerator[] tupleGens; - public final AtomicBoolean inUse = new AtomicBoolean(false); - - public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) { - this.size = size; - tupleGens = new TupleGenerator[size]; - for (int i = 0; i < size; i++) { - tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize); - } - } - - public void generate() throws IOException { - for(TupleGenerator tupleGen : tupleGens) { - tupleGen.next(); - } - } - - public int size() { - return size; - } - - public ITupleReference get(int ix) { - return tupleGens[ix].get(); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java deleted file mode 100644 index 4f26065..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hyracks.storage.am.common.datagen; - -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; -import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; -import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; -import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; - -@SuppressWarnings({"rawtypes", "unchecked" }) -public class TupleGenerator { - protected final ISerializerDeserializer[] fieldSerdes; - protected final IFieldValueGenerator[] fieldGens; - protected final ArrayTupleBuilder tb; - protected final ArrayTupleReference tuple; - protected final byte[] payload; - protected final DataOutput tbDos; - - public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) { - this.fieldSerdes = fieldSerdes; - this.fieldGens = fieldGens; - tuple = new ArrayTupleReference(); - if (payloadSize > 0) { - tb = new ArrayTupleBuilder(fieldSerdes.length + 1); - payload = new byte[payloadSize]; - } else { - tb = new ArrayTupleBuilder(fieldSerdes.length); - payload = null; - } - tbDos = tb.getDataOutput(); - } - - public ITupleReference next() throws IOException { - tb.reset(); - for (int i = 0; i < fieldSerdes.length; i++) { - fieldSerdes[i].serialize(fieldGens[i].next(), tbDos); - tb.addFieldEndOffset(); - } - if (payload != null) { - tbDos.write(payload); - tb.addFieldEndOffset(); - } - tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray()); - return tuple; - } - - public ITupleReference get() { - return tuple; - } - - public void reset() { - for (IFieldValueGenerator fieldGen : fieldGens) { - fieldGen.reset(); - } - } - - public ISerializerDeserializer[] getFieldSerdes() { - return fieldSerdes; - } - - public IFieldValueGenerator[] getFieldGens() { - return fieldGens; - } -}
