http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/region.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/region.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/region.tbl deleted file mode 100644 index c5ebb63..0000000 --- a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/region.tbl +++ /dev/null @@ -1,5 +0,0 @@ -0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to | -1|AMERICA|hs use ironic, even requests. s| -2|ASIA|ges. thinly even pinto beans ca| -3|EUROPE|ly final courts cajole furiously final excuse| -4|MIDDLE EAST|uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl|
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/supplier.tbl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/supplier.tbl b/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/supplier.tbl deleted file mode 100644 index d9c0e9f..0000000 --- a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/supplier.tbl +++ /dev/null @@ -1,10 +0,0 @@ -1|Supplier#000000001| N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ|17|27-918-335-1736|5755.94|each slyly above the careful| -2|Supplier#000000002|89eJ5ksX3ImxJQBvxObC,|5|15-679-861-2259|4032.68| slyly bold instructions. idle dependen| -3|Supplier#000000003|q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3|1|11-383-516-1199|4192.40|blithely silent requests after the express dependencies are sl| -4|Supplier#000000004|Bk7ah4CK8SYQTepEmvMkkgMwg|15|25-843-787-7479|4641.08|riously even requests above the exp| -5|Supplier#000000005|Gcdm2rJRzl5qlTVzc|11|21-151-690-3663|-283.84|. slyly regular pinto bea| -6|Supplier#000000006|tQxuVm7s7CnK|14|24-696-997-4969|1365.79|final accounts. regular dolphins use against the furiously ironic decoys. | -7|Supplier#000000007|s,4TicNGB4uO6PaSqNBUq|23|33-990-965-2201|6820.35|s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit| -8|Supplier#000000008|9Sq4bBH2FQEmaFOocY45sRTxo6yuoG|17|27-498-742-3860|7627.85|al pinto beans. asymptotes haggl| -9|Supplier#000000009|1KhUgZegwM3ua7dsYmekYBsK|10|20-403-398-8662|5302.37|s. unusual, even requests along the furiously regular pac| -10|Supplier#000000010|Saygah3gYWMp72i PY|24|34-852-489-8585|3891.91|ing waters. regular requests ar| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/tpch.ddl ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/tpch.ddl b/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/tpch.ddl deleted file mode 100644 index 89efe56..0000000 --- a/hyracks-fullstack/algebricks/algebricks-tests/data/tpch0.001/tpch.ddl +++ /dev/null @@ -1,70 +0,0 @@ --- Sccsid: @(#)dss.ddl 2.1.8.1 -CREATE TABLE NATION ( N_NATIONKEY INTEGER NOT NULL, - N_NAME CHAR(25) NOT NULL, - N_REGIONKEY INTEGER NOT NULL, - N_COMMENT VARCHAR(152)); - -CREATE TABLE REGION ( R_REGIONKEY INTEGER NOT NULL, - R_NAME CHAR(25) NOT NULL, - R_COMMENT VARCHAR(152)); - -CREATE TABLE PART ( P_PARTKEY INTEGER NOT NULL, - P_NAME VARCHAR(55) NOT NULL, - P_MFGR CHAR(25) NOT NULL, - P_BRAND CHAR(10) NOT NULL, - P_TYPE VARCHAR(25) NOT NULL, - P_SIZE INTEGER NOT NULL, - P_CONTAINER CHAR(10) NOT NULL, - P_RETAILPRICE DECIMAL(15,2) NOT NULL, - P_COMMENT VARCHAR(23) NOT NULL ); - -CREATE TABLE SUPPLIER ( S_SUPPKEY INTEGER NOT NULL, - S_NAME CHAR(25) NOT NULL, - S_ADDRESS VARCHAR(40) NOT NULL, - S_NATIONKEY INTEGER NOT NULL, - S_PHONE CHAR(15) NOT NULL, - S_ACCTBAL DECIMAL(15,2) NOT NULL, - S_COMMENT VARCHAR(101) NOT NULL); - -CREATE TABLE PARTSUPP ( PS_PARTKEY INTEGER NOT NULL, - PS_SUPPKEY INTEGER NOT NULL, - PS_AVAILQTY INTEGER NOT NULL, - PS_SUPPLYCOST DECIMAL(15,2) NOT NULL, - PS_COMMENT VARCHAR(199) NOT NULL ); - -CREATE TABLE CUSTOMER ( C_CUSTKEY INTEGER NOT NULL, - C_NAME VARCHAR(25) NOT NULL, - C_ADDRESS VARCHAR(40) NOT NULL, - C_NATIONKEY INTEGER NOT NULL, - C_PHONE CHAR(15) NOT NULL, - C_ACCTBAL DECIMAL(15,2) NOT NULL, - C_MKTSEGMENT CHAR(10) NOT NULL, - C_COMMENT VARCHAR(117) NOT NULL); - -CREATE TABLE ORDERS ( O_ORDERKEY INTEGER NOT NULL, - O_CUSTKEY INTEGER NOT NULL, - O_ORDERSTATUS CHAR(1) NOT NULL, - O_TOTALPRICE DECIMAL(15,2) NOT NULL, - O_ORDERDATE DATE NOT NULL, - O_ORDERPRIORITY CHAR(15) NOT NULL, - O_CLERK CHAR(15) NOT NULL, - O_SHIPPRIORITY INTEGER NOT NULL, - O_COMMENT VARCHAR(79) NOT NULL); - -CREATE TABLE LINEITEM ( L_ORDERKEY INTEGER NOT NULL, - L_PARTKEY INTEGER NOT NULL, - L_SUPPKEY INTEGER NOT NULL, - L_LINENUMBER INTEGER NOT NULL, - L_QUANTITY DECIMAL(15,2) NOT NULL, - L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, - L_DISCOUNT DECIMAL(15,2) NOT NULL, - L_TAX DECIMAL(15,2) NOT NULL, - L_RETURNFLAG CHAR(1) NOT NULL, - L_LINESTATUS CHAR(1) NOT NULL, - L_SHIPDATE DATE NOT NULL, - L_COMMITDATE DATE NOT NULL, - L_RECEIPTDATE DATE NOT NULL, - L_SHIPINSTRUCT CHAR(25) NOT NULL, - L_SHIPMODE CHAR(10) NOT NULL, - L_COMMENT VARCHAR(44) NOT NULL); - http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml index 52eda0b..f49189d 100644 --- a/hyracks-fullstack/algebricks/algebricks-tests/pom.xml +++ b/hyracks-fullstack/algebricks/algebricks-tests/pom.xml @@ -163,5 +163,9 @@ <artifactId>hyracks-dataflow-std</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java index af6ddca..34ed142 100644 --- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java +++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java @@ -22,7 +22,9 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.io.FileUtils; import org.apache.hyracks.algebricks.data.IPrinterFactory; import org.apache.hyracks.algebricks.data.impl.BinaryBooleanInspectorImpl; import org.apache.hyracks.algebricks.data.impl.BinaryIntegerInspectorImpl; @@ -64,8 +66,11 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.ManagedFileSplit; import org.apache.hyracks.api.job.JobSpecification; +import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.data.std.primitive.UTF8StringPointable; @@ -100,6 +105,7 @@ public class PushRuntimeTest { private static final int FRAME_SIZE = 32768; private static final String[] DEFAULT_NODES = new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID }; + private static final AtomicInteger aInteger = new AtomicInteger(0); @BeforeClass public static void setUp() throws Exception { @@ -180,8 +186,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] intFileSplits = new FileSplit[1]; - intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/simple/int-part1.tbl") - .getAbsolutePath(), false); + intFileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, "data" + File.separator + + "simple" + File.separator + "int-part1.tbl"); IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(intFileSplits); RecordDescriptor intScannerDesc = new RecordDescriptor( new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE }); @@ -263,8 +269,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] fileSplits = new FileSplit[1]; - fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/customer.tbl") - .getAbsolutePath(), false); + fileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, "data" + File.separator + + "tpch0.001" + File.separator + "customer.tbl"); IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { @@ -345,8 +351,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] fileSplits = new FileSplit[1]; - fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, - new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false); + fileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, + "data" + File.separator + "tpch0.001" + File.separator + "customer-part1.tbl"); IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), @@ -397,8 +403,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] fileSplits = new FileSplit[1]; - fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, - new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false); + fileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, "data" + File.separator + + "tpch0.001" + File.separator + "customer.tbl"); IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), @@ -575,15 +581,10 @@ public class PushRuntimeTest { JobSpecification spec = new JobSpecification(FRAME_SIZE); - String inputFileName = "data/tpch0.001/customer.tbl"; - File inputFile = new File(inputFileName); - File[] outputFile = new File[outputArity]; - for (int i = 0; i < outputArity; i++) { - outputFile[i] = File.createTempFile("splitop", null); - } + String inputFileName = "data" + File.separator + "tpch0.001" + File.separator + "customer.tbl"; FileSplit[] inputSplits = new FileSplit[] { - new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFile.getAbsolutePath(), false) }; + new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFileName) }; DelimitedDataTupleParserFactory stringParser = new DelimitedDataTupleParserFactory( new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE }, '\u0000'); @@ -601,10 +602,12 @@ public class PushRuntimeTest { PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, replicateOp, new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID }); - IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length]; + IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputArity]; + File[] outputFile = new File[outputArity]; for (int i = 0; i < outputArity; i++) { - outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { - new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) }); + FileSplit fileSplit = createFile(AlgebricksHyracksIntegrationUtil.nc1); + outputFile[i] = fileSplit.getFile(AlgebricksHyracksIntegrationUtil.nc1.getIoManager()); + outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { fileSplit }); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i], new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID }); } @@ -620,29 +623,41 @@ public class PushRuntimeTest { AlgebricksHyracksIntegrationUtil.runJob(spec); for (int i = 0; i < outputArity; i++) { - compareFiles(inputFileName, outputFile[i].getAbsolutePath()); + compareFiles("data" + File.separator + "device0" + File.separator + inputFileName, outputFile[i] + .getAbsolutePath()); } } + public FileSplit createFile(NodeControllerService ncs) throws IOException { + String fileName = "f" + aInteger.getAndIncrement() + ".tmp"; + FileReference fileRef = ncs.getIoManager().getFileReference(0, fileName); + FileUtils.deleteQuietly(fileRef.getFile()); + fileRef.getFile().createNewFile(); + return new ManagedFileSplit(ncs.getId(), fileName); + } + @Test public void scanSplitWrite() throws Exception { final int outputArity = 2; JobSpecification spec = new JobSpecification(FRAME_SIZE); - String inputFileName[] = { "data/simple/int-string-part1.tbl", "data/simple/int-string-part1-split-0.tbl", - "data/simple/int-string-part1-split-1.tbl" }; + String inputFileName[] = { "data" + File.separator + "simple" + File.separator + "int-string-part1.tbl", "data" + + File.separator + "simple" + File.separator + "int-string-part1-split-0.tbl", + "data" + File.separator + "simple" + File.separator + "int-string-part1-split-1.tbl" }; File[] inputFiles = new File[inputFileName.length]; - for (int i=0; i<inputFileName.length; i++) { + for (int i = 0; i < inputFileName.length; i++) { inputFiles[i] = new File(inputFileName[i]); } File[] outputFile = new File[outputArity]; + FileSplit[] outputFileSplit = new FileSplit[outputArity]; for (int i = 0; i < outputArity; i++) { - outputFile[i] = File.createTempFile("splitop", null); + outputFileSplit[i] = createFile(AlgebricksHyracksIntegrationUtil.nc1); + outputFile[i] = outputFileSplit[i].getFile(AlgebricksHyracksIntegrationUtil.nc1.getIoManager()); } FileSplit[] inputSplits = new FileSplit[] { - new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFiles[0].getAbsolutePath(), false) }; + new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFileName[0]) }; IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(inputSplits); RecordDescriptor scannerDesc = new RecordDescriptor( @@ -665,8 +680,7 @@ public class PushRuntimeTest { IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length]; for (int i = 0; i < outputArity; i++) { - outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { - new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) }); + outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { outputFileSplit[i] }); PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i], new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID }); } @@ -682,7 +696,8 @@ public class PushRuntimeTest { AlgebricksHyracksIntegrationUtil.runJob(spec); for (int i = 0; i < outputArity; i++) { - compareFiles(inputFileName[i + 1], outputFile[i].getAbsolutePath()); + compareFiles("data" + File.separator + "device0" + File.separator + inputFileName[i + 1], outputFile[i] + .getAbsolutePath()); } } @@ -692,8 +707,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] fileSplits = new FileSplit[1]; - fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/nation.tbl") - .getAbsolutePath(), false); + fileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, "data" + File.separator + + "tpch0.001" + File.separator + "nation.tbl"); IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), @@ -801,8 +816,8 @@ public class PushRuntimeTest { // the scanner FileSplit[] fileSplits = new FileSplit[1]; - fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, - new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false); + fileSplits[0] = new ManagedFileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, + "data" + File.separator + "tpch0.001" + File.separator + "customer.tbl"); IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(), http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java index 17e1589..9880047 100644 --- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java +++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java @@ -18,8 +18,10 @@ */ package org.apache.hyracks.algebricks.tests.util; +import java.io.File; import java.util.EnumSet; +import org.apache.commons.io.FileUtils; import org.apache.hyracks.algebricks.core.config.AlgebricksConfig; import org.apache.hyracks.api.client.HyracksConnection; import org.apache.hyracks.api.client.IHyracksClientConnection; @@ -40,11 +42,13 @@ public class AlgebricksHyracksIntegrationUtil { public static final int TEST_HYRACKS_CC_CLIENT_NET_PORT = 4321; private static ClusterControllerService cc; - private static NodeControllerService nc1; - private static NodeControllerService nc2; + public static NodeControllerService nc1; + public static NodeControllerService nc2; private static IHyracksClientConnection hcc; public static void init() throws Exception { + FileUtils.deleteQuietly(new File("target" + File.separator + "data")); + FileUtils.copyDirectory(new File("data"), new File("target" + File.separator + "data")); CCConfig ccConfig = new CCConfig(); ccConfig.clientNetIpAddress = "127.0.0.1"; ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_NET_PORT; @@ -60,6 +64,9 @@ public class AlgebricksHyracksIntegrationUtil { ncConfig1.dataIPAddress = "127.0.0.1"; ncConfig1.resultIPAddress = "127.0.0.1"; ncConfig1.nodeId = NC1_ID; + ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" + + File.separator + "device0"; + FileUtils.forceMkdir(new File(ncConfig1.ioDevices)); nc1 = new NodeControllerService(ncConfig1); nc1.start(); @@ -70,6 +77,9 @@ public class AlgebricksHyracksIntegrationUtil { ncConfig2.dataIPAddress = "127.0.0.1"; ncConfig2.resultIPAddress = "127.0.0.1"; ncConfig2.nodeId = NC2_ID; + ncConfig1.ioDevices = System.getProperty("user.dir") + File.separator + "target" + File.separator + "data" + + File.separator + "device1"; + FileUtils.forceMkdir(new File(ncConfig1.ioDevices)); nc2 = new NodeControllerService(ncConfig2); nc2.start(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java index bd4567b..4754731 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java @@ -24,31 +24,24 @@ import java.io.Serializable; import org.apache.hyracks.api.exceptions.HyracksDataException; /** - * A node and a path (Can be relative: inside the IO device or absolute inside or outside IO devices) + * A node and a path (Can be managed: inside the IO device or absolute inside or outside IO devices) * Used to identify a file/dir across the cluster. */ -public class FileSplit implements Serializable { +public abstract class FileSplit implements Serializable { + private static final long serialVersionUID = 1L; private final String node; private final String path; - private final boolean managed; /** * Constructor * * @param node * @param path - * @param maanged */ - public FileSplit(String node, String path, boolean relative) { + protected FileSplit(String node, String path) { this.node = node; this.path = path; - this.managed = relative; - } - - @Override - public String toString() { - return "Node: " + node + " " + (managed ? "managed" : "absolute") + " path: " + path; } /** @@ -59,33 +52,13 @@ public class FileSplit implements Serializable { } /** - * @return true if relative - */ - public boolean isManaged() { - return managed; - } - - /** * Get the local file represented by this split * * @param ioManager * @return * @throws HyracksDataException */ - public File getFile(IIOManager ioManager) throws HyracksDataException { - return managed ? getFileRef(ioManager).getFile() : new File(path); - } - - /** - * Get the file reference for the split - * - * @param ioManager - * @return - * @throws HyracksDataException - */ - public FileReference getFileRef(IIOManager ioManager) throws HyracksDataException { - return ioManager.getFileRef(path, managed); - } + public abstract File getFile(IIOManager ioManager) throws HyracksDataException; /** * @return the node @@ -93,4 +66,8 @@ public class FileSplit implements Serializable { public String getNodeName() { return node; } + + // TODO(amoudi): This method should be removed from this class and moved to ManagedFileSplit since it is only + // applicable for that subclass + public abstract FileReference getFileReference(IIOManager ioManager) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java index 0b3f71a..50ef4f0 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java @@ -63,18 +63,36 @@ public interface IIOManager { /** * @param ioDeviceId - * @param relativePath + * @param path * @return A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath} */ - public FileReference getFileRef(int ioDeviceId, String relativePath); + FileReference getFileReference(int ioDeviceId, String path); /** - * A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath} + * determine which IO device holds the path and returns a FileReference based on that * * @param path - * @param relative - * @return + * @return A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code Path} + * @throws HyracksDataException + */ + FileReference resolve(String path) throws HyracksDataException; + + /** + * Gets a file reference from an absolute path + * @deprecated + * use getFileRef(int ioDeviceId, String path) instead + * @param path + * @return A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath} + * @throws HyracksDataException + */ + @Deprecated + FileReference resolveAbsolutePath(String path) throws HyracksDataException; + + /** + * Create a workspace file with the given prefix + * @param prefix + * @return A FileReference for the created workspace file * @throws HyracksDataException */ - public FileReference getFileRef(String path, boolean relative) throws HyracksDataException; + FileReference createWorkspaceFile(String prefix) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/ManagedFileSplit.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/ManagedFileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/ManagedFileSplit.java new file mode 100644 index 0000000..511e47f --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/ManagedFileSplit.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.io; + +import java.io.File; + +import org.apache.hyracks.api.exceptions.HyracksDataException; + +/** + * A node and a managed path inside an IO device. + */ +public class ManagedFileSplit extends FileSplit { + private static final long serialVersionUID = 1L; + + /** + * Constructor + * + * @param node + * @param path + */ + public ManagedFileSplit(String node, String path) { + super(node, path); + } + + @Override + public String toString() { + return "Node: " + getNodeName() + " managed path: " + getPath(); + } + + /** + * Get the local file represented by this split + * + * @param ioManager + * @return + * @throws HyracksDataException + */ + @Override + public File getFile(IIOManager ioManager) throws HyracksDataException { + return getFileReference(ioManager).getFile(); + } + + /** + * Get the file reference for the split + * + * @param ioManager + * @return + * @throws HyracksDataException + */ + @Override + public FileReference getFileReference(IIOManager ioManager) throws HyracksDataException { + return ioManager.resolve(getPath()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/MappedFileSplit.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/MappedFileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/MappedFileSplit.java new file mode 100644 index 0000000..0a0e026 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/MappedFileSplit.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.io; + +import java.io.File; + +import org.apache.hyracks.api.exceptions.HyracksDataException; + +/** + * A FileSplit that is mapped to a specific IO device + */ +public class MappedFileSplit extends ManagedFileSplit { + + private static final long serialVersionUID = 1L; + private final int ioDevice; + private transient FileReference cached; + + /** + * Construct a managed File split that is mapped to an IO device + * @param node + * @param path + * @param ioDevice + */ + public MappedFileSplit(String node, String path, int ioDevice) { + super(node, path); + this.ioDevice = ioDevice; + } + + public int getIoDevice() { + return ioDevice; + } + + @Override + public String toString() { + return "Node: " + getNodeName() + " IO Device: " + ioDevice + " managed path: " + getPath(); + } + + @Override + public File getFile(IIOManager ioManager) throws HyracksDataException { + return getFileReference(ioManager).getFile(); + } + + @Override + public FileReference getFileReference(IIOManager ioManager) throws HyracksDataException { + if (cached == null) { + cached = new FileReference(ioManager.getIODevices().get(ioDevice), getPath()); + } + return cached; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/UnmanagedFileSplit.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/UnmanagedFileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/UnmanagedFileSplit.java new file mode 100644 index 0000000..9a6f434 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/UnmanagedFileSplit.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.io; + +import java.io.File; + +import org.apache.hyracks.api.exceptions.HyracksDataException; + +/** + * A filesplit for files that are not managed by the IO device + */ +public class UnmanagedFileSplit extends FileSplit { + + private static final long serialVersionUID = 1L; + + public UnmanagedFileSplit(String node, String path) { + super(node, path); + } + + @Override + public String toString() { + return "Node: " + getNodeName() + " unmanaged path: " + getPath(); + } + + /** + * Get the local file represented by this split + * + * @param ioManager + * @return + * @throws HyracksDataException + */ + @Override + public File getFile(IIOManager ioManager) { + return new File(getPath()); + } + + /** + * Get the local file represented by this unmanaged file split + * + * @return + */ + public File getFile() { + return new File(getPath()); + } + + @Override + public FileReference getFileReference(IIOManager ioManager) throws HyracksDataException { + throw new HyracksDataException("FileReference is only for files inside an IO device"); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Joblet.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Joblet.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Joblet.java index 98ae7c4..25ab245 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Joblet.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Joblet.java @@ -56,7 +56,6 @@ import org.apache.hyracks.control.common.job.PartitionState; import org.apache.hyracks.control.common.job.profiling.counters.Counter; import org.apache.hyracks.control.common.job.profiling.om.JobletProfile; import org.apache.hyracks.control.common.job.profiling.om.TaskProfile; -import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.control.nc.io.WorkspaceFileFactory; import org.apache.hyracks.control.nc.resources.DefaultDeallocatableRegistry; import org.apache.hyracks.control.nc.resources.memory.FrameManager; @@ -115,7 +114,7 @@ public class Joblet implements IHyracksJobletContext, ICounterContext { taskMap = new HashMap<>(); counterMap = new HashMap<>(); deallocatableRegistry = new DefaultDeallocatableRegistry(); - fileFactory = new WorkspaceFileFactory(this, (IOManager) appCtx.getIoManager()); + fileFactory = new WorkspaceFileFactory(this, appCtx.getIoManager()); cleanupPending = false; IJobletEventListenerFactory jelf = acg.getJobletEventListenerFactory(); if (jelf != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 598110a..f00303f 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -28,11 +28,9 @@ import java.lang.management.RuntimeMXBean; import java.lang.management.ThreadMXBean; import java.lang.reflect.Field; import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.Hashtable; import java.util.List; import java.util.Map; -import java.util.StringTokenizer; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutorService; @@ -157,7 +155,7 @@ public class NodeControllerService implements IControllerService { new NodeControllerIPCI(this), new CCNCFunctions.SerializerDeserializer()); - ioManager = new IOManager(getDevices(ncConfig.ioDevices)); + ioManager = new IOManager(IODeviceHandle.getDevices(ncConfig.ioDevices)); if (id == null) { throw new Exception("id not set"); } @@ -195,16 +193,6 @@ public class NodeControllerService implements IControllerService { return lccm; } - private static List<IODeviceHandle> getDevices(String ioDevices) { - List<IODeviceHandle> devices = new ArrayList<>(); - StringTokenizer tok = new StringTokenizer(ioDevices, ","); - while (tok.hasMoreElements()) { - String devPath = tok.nextToken().trim(); - devices.add(new IODeviceHandle(new File(devPath), ".")); - } - return devices; - } - synchronized void setNodeRegistrationResult(NodeParameters parameters, Exception exception) { this.nodeParameters = parameters; this.registrationException = exception; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java index 7c626c1..71beab4 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java @@ -58,7 +58,6 @@ import org.apache.hyracks.control.common.job.profiling.counters.Counter; import org.apache.hyracks.control.common.job.profiling.om.PartitionProfile; import org.apache.hyracks.control.common.job.profiling.om.TaskProfile; import org.apache.hyracks.control.common.utils.ExceptionUtils; -import org.apache.hyracks.control.nc.io.IOManager; import org.apache.hyracks.control.nc.io.WorkspaceFileFactory; import org.apache.hyracks.control.nc.resources.DefaultDeallocatableRegistry; import org.apache.hyracks.control.nc.work.NotifyTaskCompleteWork; @@ -105,12 +104,12 @@ public class Task implements IHyracksTaskContext, ICounterContext, Runnable { this.taskAttemptId = taskId; this.displayName = displayName; this.executorService = executor; - fileFactory = new WorkspaceFileFactory(this, (IOManager) joblet.getIOManager()); + fileFactory = new WorkspaceFileFactory(this, joblet.getIOManager()); deallocatableRegistry = new DefaultDeallocatableRegistry(); - counterMap = new HashMap<String, Counter>(); + counterMap = new HashMap<>(); opEnv = joblet.getEnvironment(); - partitionSendProfile = new Hashtable<PartitionId, PartitionProfile>(); - pendingThreads = new LinkedHashSet<Thread>(); + partitionSendProfile = new Hashtable<>(); + pendingThreads = new LinkedHashSet<>(); exceptions = new ArrayList<>(); this.ncs = ncs; this.inputChannelsFromConnectors = inputChannelsFromConnectors; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java index bdb283a..80bb662 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.concurrent.Executor; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IFileDeviceComputer; import org.apache.hyracks.api.io.IFileHandle; @@ -55,12 +54,12 @@ public class IOManager implements IIOManager { private int workspaceIndex; private IFileDeviceComputer deviceComputer; - public IOManager(List<IODeviceHandle> devices, Executor executor) throws HyracksException { + public IOManager(List<IODeviceHandle> devices, Executor executor) throws HyracksDataException { this(devices); this.executor = executor; } - public IOManager(List<IODeviceHandle> devices) throws HyracksException { + public IOManager(List<IODeviceHandle> devices) throws HyracksDataException { this.ioDevices = Collections.unmodifiableList(devices); workspaces = new ArrayList<>(); for (IODeviceHandle d : ioDevices) { @@ -70,7 +69,7 @@ public class IOManager implements IIOManager { } } if (workspaces.isEmpty()) { - throw new HyracksException("No devices with work areas found"); + throw new HyracksDataException("No devices with workspace found"); } workspaceIndex = 0; deviceComputer = new DefaultDeviceComputer(this); @@ -165,7 +164,7 @@ public class IOManager implements IIOManager { * @param offset * @param data * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's - * current size + * current size * @throws HyracksDataException */ @Override @@ -213,6 +212,7 @@ public class IOManager implements IIOManager { } } + @Override public synchronized FileReference createWorkspaceFile(String prefix) throws HyracksDataException { IODeviceHandle dev = workspaces.get(workspaceIndex); workspaceIndex = (workspaceIndex + 1) % workspaces.size(); @@ -328,16 +328,18 @@ public class IOManager implements IIOManager { } @Override - public FileReference getFileRef(int ioDeviceId, String relativePath) { + public synchronized FileReference getFileReference(int ioDeviceId, String relativePath) { IODeviceHandle devHandle = ioDevices.get(ioDeviceId); return new FileReference(devHandle, relativePath); } @Override - public FileReference getFileRef(String path, boolean relative) throws HyracksDataException { - if (relative) { - return new FileReference(deviceComputer.compute(path), path); - } + public FileReference resolve(String path) throws HyracksDataException { + return new FileReference(deviceComputer.compute(path), path); + } + + @Override + public FileReference resolveAbsolutePath(String path) throws HyracksDataException { IODeviceHandle devHandle = getDevice(path); if (devHandle == null) { throw new HyracksDataException("The file with absolute path: " + path + " is outside all io devices"); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/WorkspaceFileFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/WorkspaceFileFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/WorkspaceFileFactory.java index 4f9eb29..0dedbbf 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/WorkspaceFileFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/WorkspaceFileFactory.java @@ -21,15 +21,16 @@ package org.apache.hyracks.control.nc.io; import org.apache.commons.io.FileUtils; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.io.IWorkspaceFileFactory; import org.apache.hyracks.api.resources.IDeallocatable; import org.apache.hyracks.api.resources.IDeallocatableRegistry; public final class WorkspaceFileFactory implements IWorkspaceFileFactory { private final IDeallocatableRegistry registry; - private final IOManager ioManager; + private final IIOManager ioManager; - public WorkspaceFileFactory(IDeallocatableRegistry registry, IOManager ioManager) { + public WorkspaceFileFactory(IDeallocatableRegistry registry, IIOManager ioManager) { this.registry = registry; this.ioManager = ioManager; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartition.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartition.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartition.java index 20b5bbe..bcfe517 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartition.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartition.java @@ -37,10 +37,10 @@ public class MaterializedPartition implements IPartition { private final Executor executor; - private final IOManager ioManager; + private final IIOManager ioManager; public MaterializedPartition(IHyracksTaskContext ctx, FileReference partitionFile, Executor executor, - IOManager ioManager) { + IIOManager ioManager) { this.ctx = ctx; this.partitionFile = partitionFile; this.executor = executor; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartitionWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartitionWriter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartitionWriter.java index 74aa350..37aed0a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartitionWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializedPartitionWriter.java @@ -32,7 +32,6 @@ import org.apache.hyracks.api.io.IFileHandle; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.partitions.PartitionId; import org.apache.hyracks.control.common.job.PartitionState; -import org.apache.hyracks.control.nc.io.IOManager; public class MaterializedPartitionWriter implements IFrameWriter { private static final Logger LOGGER = Logger.getLogger(MaterializedPartitionWriter.class.getName()); @@ -98,7 +97,7 @@ public class MaterializedPartitionWriter implements IFrameWriter { } if (!failed) { manager.registerPartition(pid, taId, - new MaterializedPartition(ctx, fRef, executor, (IOManager) ctx.getIOManager()), + new MaterializedPartition(ctx, fRef, executor, ctx.getIOManager()), PartitionState.COMMITTED, taId.getAttempt() == 0 ? false : true); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java index 4713aa6..137ef37 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java @@ -33,7 +33,6 @@ import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.partitions.IPartition; import org.apache.hyracks.api.partitions.PartitionId; import org.apache.hyracks.control.common.job.PartitionState; -import org.apache.hyracks.control.nc.io.IOManager; public class MaterializingPipelinedPartition implements IFrameWriter, IPartition { private static final Logger LOGGER = Logger.getLogger(MaterializingPipelinedPartition.class.getName()); @@ -42,7 +41,7 @@ public class MaterializingPipelinedPartition implements IFrameWriter, IPartition private final Executor executor; - private final IOManager ioManager; + private final IIOManager ioManager; private final PartitionManager manager; @@ -68,7 +67,7 @@ public class MaterializingPipelinedPartition implements IFrameWriter, IPartition TaskAttemptId taId, Executor executor) { this.ctx = ctx; this.executor = executor; - this.ioManager = (IOManager) ctx.getIOManager(); + this.ioManager = ctx.getIOManager(); this.manager = manager; this.pid = pid; this.taId = taId; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml index 39a30e1..e44feb7 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml @@ -84,7 +84,6 @@ <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-control-nc</artifactId> <version>${project.version}</version> - <scope>test</scope> </dependency> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java index 5028ccc..2207441 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java @@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; import org.apache.hyracks.dataflow.std.base.IOpenableDataWriterOperator; @@ -31,10 +32,12 @@ import org.apache.hyracks.dataflow.std.util.DeserializedOperatorNodePushable; public abstract class AbstractFileWriteOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { protected class FileWriteOperator implements IOpenableDataWriterOperator { + private final IIOManager ioManager; private int index; private IRecordWriter writer; - FileWriteOperator(int index) { + FileWriteOperator(IIOManager ioManager, int index) { + this.ioManager = ioManager; this.index = index; } @@ -46,11 +49,7 @@ public abstract class AbstractFileWriteOperatorDescriptor extends AbstractSingle @Override public void open() throws HyracksDataException { FileSplit split = splits[index]; - try { - writer = createRecordWriter(split, index); - } catch (Exception e) { - throw new HyracksDataException(e); - } + writer = createRecordWriter(ioManager, split, index); } @Override @@ -64,12 +63,7 @@ public abstract class AbstractFileWriteOperatorDescriptor extends AbstractSingle @Override public void writeData(Object[] data) throws HyracksDataException { - try { - writer.write(data); - - } catch (Exception e) { - throw new HyracksDataException(e); - } + writer.write(data); } @Override @@ -95,12 +89,13 @@ public abstract class AbstractFileWriteOperatorDescriptor extends AbstractSingle this.splits = splits; } - protected abstract IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception; + protected abstract IRecordWriter createRecordWriter(IIOManager ioManager, FileSplit fileSplit, int index) + throws HyracksDataException; @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { - return new DeserializedOperatorNodePushable(ctx, new FileWriteOperator(partition), - recordDescProvider.getInputRecordDescriptor(getActivityId(), 0)); + return new DeserializedOperatorNodePushable(ctx, new FileWriteOperator(ctx.getIOManager(), + partition), recordDescProvider.getInputRecordDescriptor(getActivityId(), 0)); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java index 94c5a68..8acb182 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java @@ -73,7 +73,7 @@ public class FileRemoveOperatorDescriptor extends AbstractSingleActivityOperator @Override public void initialize() throws HyracksDataException { // will only work for files inside the io devices - File f = ioManager.getFileRef(split.getPath(), split.isManaged()).getFile(); + File f = split.getFile(ioManager); if (quietly) { FileUtils.deleteQuietly(f); } else { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java index a85e76e..016ba46 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java @@ -22,6 +22,7 @@ import java.io.Serializable; import org.apache.hyracks.api.io.FileSplit; +@FunctionalInterface public interface IFileSplitProvider extends Serializable { - public FileSplit[] getFileSplits(); + FileSplit[] getFileSplits(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IRecordWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IRecordWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IRecordWriter.java index 845b012..4020d15 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IRecordWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IRecordWriter.java @@ -18,12 +18,12 @@ */ package org.apache.hyracks.dataflow.std.file; -import java.io.File; +import org.apache.hyracks.api.exceptions.HyracksDataException; public interface IRecordWriter { public void close(); - public void write(Object[] record) throws Exception; + public void write(Object[] record) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java index 1186283..5886eb4 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java @@ -19,25 +19,30 @@ package org.apache.hyracks.dataflow.std.file; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.OutputStream; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDescriptor { private static final long serialVersionUID = 1L; private static class LineWriterImpl extends RecordWriter { - LineWriterImpl(File file, int[] columns, char separator) throws Exception { + LineWriterImpl(File file, int[] columns, char separator) throws HyracksDataException { super(columns, separator, new Object[] { file }); } - private static final long serialVersionUID = 1L; - @Override - public OutputStream createOutputStream(Object[] args) throws Exception { - return new FileOutputStream((File) args[0]); + public OutputStream createOutputStream(Object[] args) throws HyracksDataException { + try { + return new FileOutputStream((File) args[0]); + } catch (FileNotFoundException e) { + throw new HyracksDataException(e); + } } } @@ -52,14 +57,16 @@ public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDe this(spec, splits, columns, RecordWriter.COMMA); } - public LineFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, int[] columns, char separator) { + public LineFileWriteOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, int[] columns, + char separator) { super(spec, splits); this.columns = columns; this.separator = separator; } @Override - protected IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception { - return new LineWriterImpl(fileSplit.getFile(null), columns, separator); + protected IRecordWriter createRecordWriter(IIOManager ioManager, FileSplit fileSplit, int index) + throws HyracksDataException { + return new LineWriterImpl(fileSplit.getFile(ioManager), columns, separator); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java index 97f12b5..86d0e3f 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java @@ -31,7 +31,8 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; public class RecordFileScanOperatorDescriptor extends AbstractDeserializedFileScanOperatorDescriptor { private static final long serialVersionUID = 1L; - public RecordFileScanOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, RecordDescriptor recordDescriptor) { + public RecordFileScanOperatorDescriptor(IOperatorDescriptorRegistry spec, FileSplit[] splits, + RecordDescriptor recordDescriptor) { super(spec, splits, recordDescriptor); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordWriter.java index 2f6b53e..b3c688a 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordWriter.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.std.util.StringSerializationUtils; public abstract class RecordWriter implements IRecordWriter { @@ -44,7 +45,7 @@ public abstract class RecordWriter implements IRecordWriter { this.separator = COMMA; } - public RecordWriter(int[] columns, char separator, Object[] args) throws Exception { + public RecordWriter(int[] columns, char separator, Object[] args) throws HyracksDataException { OutputStream outputStream = createOutputStream(args); if (outputStream != null) { bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream)); @@ -65,25 +66,29 @@ public abstract class RecordWriter implements IRecordWriter { } @Override - public void write(Object[] record) throws Exception { - if (columns == null) { - for (int i = 0; i < record.length; ++i) { - if (i != 0) { - bufferedWriter.write(separator); + public void write(Object[] record) throws HyracksDataException { + try { + if (columns == null) { + for (int i = 0; i < record.length; ++i) { + if (i != 0) { + bufferedWriter.write(separator); + } + bufferedWriter.write(StringSerializationUtils.toString(record[i])); } - bufferedWriter.write(StringSerializationUtils.toString(record[i])); - } - } else { - for (int i = 0; i < columns.length; ++i) { - if (i != 0) { - bufferedWriter.write(separator); + } else { + for (int i = 0; i < columns.length; ++i) { + if (i != 0) { + bufferedWriter.write(separator); + } + bufferedWriter.write(StringSerializationUtils.toString(record[columns[i]])); } - bufferedWriter.write(StringSerializationUtils.toString(record[columns[i]])); } + bufferedWriter.write("\n"); + } catch (IOException e) { + throw new HyracksDataException(e); } - bufferedWriter.write("\n"); } - public abstract OutputStream createOutputStream(Object[] args) throws Exception; + public abstract OutputStream createOutputStream(Object[] args) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java index c6e81a4..6082e7d 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java @@ -78,9 +78,6 @@ public class InsertPipelineExample { @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false) public int frameSize = 32768; - - @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false) - public boolean relative = true; } public static void main(String[] args) throws Exception { @@ -150,8 +147,7 @@ public class InsertPipelineExample { int[] primaryFieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input // tuple to field 0 of // B-Tree tuple, etc. - IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName, - options.relative); + IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName); IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true); @@ -178,7 +174,7 @@ public class InsertPipelineExample { // tuple int[] secondaryFieldPermutation = { 1, 2 }; IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, - options.secondaryBTreeName, true); + options.secondaryBTreeName); // create operator descriptor TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor( spec, recDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java index 05143a6..4b219e8 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java @@ -22,17 +22,17 @@ package org.apache.hyracks.examples.btree.client; import org.apache.hyracks.api.constraints.PartitionConstraintHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.api.io.ManagedFileSplit; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; public class JobHelper { - public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName, - boolean relative) { + public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName) { FileSplit[] fileSplits = new FileSplit[splitNCs.length]; for (int i = 0; i < splitNCs.length; ++i) { String fileName = btreeFileName + "." + splitNCs[i]; - fileSplits[i] = new FileSplit(splitNCs[i], fileName, relative); + fileSplits[i] = new ManagedFileSplit(splitNCs[i], fileName); } return new ConstantFileSplitProvider(fileSplits); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java index c455c0d..13d54db 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java @@ -76,9 +76,6 @@ public class PrimaryIndexBulkLoadExample { @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false) public int frameSize = 32768; - - @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false) - public boolean relative = true; } public static void main(String[] args) throws Exception { @@ -154,8 +151,7 @@ public class PrimaryIndexBulkLoadExample { int[] fieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input tuple // to field 0 of B-Tree tuple, // etc. - IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName, - options.relative); + IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName); IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true); TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, recDesc, storageManager, lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java index 1867a80..3ab3b5f 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java @@ -69,9 +69,6 @@ public class PrimaryIndexSearchExample { @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false) public int frameSize = 32768; - - @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false) - public boolean relative = true; } public static void main(String[] args) throws Exception { @@ -144,8 +141,7 @@ public class PrimaryIndexSearchExample { int[] highKeyFields = { 1 }; // low key is in field 1 of tuples going // into search op - IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName, - options.relative); + IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName); IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true); BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager, lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, lowKeyFields, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java index 2acc204..9fb59ac 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java @@ -72,9 +72,6 @@ public class SecondaryIndexBulkLoadExample { @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false) public int frameSize = 32768; - - @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false) - public boolean relative = true; } public static void main(String[] args) throws Exception { @@ -125,8 +122,7 @@ public class SecondaryIndexBulkLoadExample { comparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY); // use a disk-order scan to read primary index - IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName, - options.relative); + IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName); IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true); TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec, recDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, @@ -149,8 +145,7 @@ public class SecondaryIndexBulkLoadExample { // the B-Tree expects its keyfields to be at the front of its input // tuple int[] fieldPermutation = { 1, 0 }; - IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName, - options.relative); + IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName); TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, null, storageManager, lcManagerProvider, btreeSplitProvider, secondaryTypeTraits, comparatorFactories, null, fieldPermutation, 0.7f, false, 1000L, true, dataflowHelperFactory); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/72aa1b74/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java index 5e140ce..85ebef6 100644 --- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java +++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java @@ -72,9 +72,6 @@ public class SecondaryIndexSearchExample { @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false) public int frameSize = 32768; - - @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false) - public boolean relative = true; } public static void main(String[] args) throws Exception { @@ -170,7 +167,7 @@ public class SecondaryIndexSearchExample { // index search op IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, - options.secondaryBTreeName, options.relative); + options.secondaryBTreeName); IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true); BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits, @@ -188,8 +185,7 @@ public class SecondaryIndexSearchExample { // going into primary index search // op - IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName, - options.relative); + IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName); BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories, null, primaryLowKeyFields, primaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,
