Repository: incubator-hawq Updated Branches: refs/heads/master 80e8322c2 -> 2222f4b4a
HAWQ-1490. Added Demo PXF classes for standalone testing Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2222f4b4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2222f4b4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2222f4b4 Branch: refs/heads/master Commit: 2222f4b4afa2fa8f3f862fb22aabbacc77e990f5 Parents: 80e8322 Author: John Gaskin <[email protected]> Authored: Wed Jul 12 11:08:16 2017 -0700 Committer: shivzone <[email protected]> Committed: Wed Jul 12 11:08:16 2017 -0700 ---------------------------------------------------------------------- .../hawq/pxf/api/examples/DemoAccessor.java | 98 ++++++++++++++++++++ .../hawq/pxf/api/examples/DemoFragmenter.java | 59 ++++++++++++ .../hawq/pxf/api/examples/DemoResolver.java | 67 +++++++++++++ .../hawq/pxf/api/examples/DemoTextResolver.java | 63 +++++++++++++ .../apache/hawq/pxf/api/DemoAccessorTest.java | 79 ++++++++++++++++ .../apache/hawq/pxf/api/DemoResolverTest.java | 70 ++++++++++++++ 6 files changed, 436 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java new file mode 100644 index 0000000..bec3146 --- /dev/null +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoAccessor.java @@ -0,0 +1,98 @@ +package org.apache.hawq.pxf.api.examples; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.hawq.pxf.api.OneRow; +import org.apache.hawq.pxf.api.ReadAccessor; +import org.apache.hawq.pxf.api.utilities.InputData; +import org.apache.hawq.pxf.api.utilities.Plugin; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Internal interface that would defined the access to a file on HDFS, but in + * this case contains the data required. + * + * Demo implementation + */ +public class DemoAccessor extends Plugin implements ReadAccessor { + + private static final Log LOG = LogFactory.getLog(DemoAccessor.class); + private int rowNumber; + private int fragmentNumber; + private static int NUM_ROWS = 2; + + /** + * Constructs a DemoAccessor + * + * @param metaData the InputData + */ + public DemoAccessor(InputData metaData) { + super(metaData); + } + @Override + public boolean openForRead() throws Exception { + /* no-op, because this plugin doesn't read a file. */ + return true; + } + + /** + * Read the next record + * The record contains as many fields as defined by the DDL schema. + * + * @return one row which corresponds to one record + */ + @Override + public OneRow readNextObject() throws Exception { + /* return next row , <key=fragmentNo.rowNo, val=rowNo,text,fragmentNo>*/ + /* check for EOF */ + if (fragmentNumber > 0) + return null; /* signal EOF, close will be called */ + int fragment = inputData.getDataFragment(); + String fragmentMetadata = new String(inputData.getFragmentMetadata()); + int colCount = inputData.getColumns(); + + /* generate row with (colCount) columns */ + StringBuilder colValue = new StringBuilder(fragmentMetadata + " row" + (rowNumber+1)); + for(int colIndex=1; colIndex<colCount; colIndex++) { + colValue.append(",").append("value" + colIndex); + } + OneRow row = new OneRow(fragment + "." + rowNumber, colValue.toString()); + + /* advance */ + rowNumber += 1; + if (rowNumber == NUM_ROWS) { + rowNumber = 0; + fragmentNumber += 1; + } + + /* return data */ + return row; + } + + /** + * close the reader. no action here + * + */ + @Override + public void closeForRead() throws Exception { + /* Demo close doesn't do anything */ + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java new file mode 100644 index 0000000..685fcf2 --- /dev/null +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoFragmenter.java @@ -0,0 +1,59 @@ +package org.apache.hawq.pxf.api.examples; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.hawq.pxf.api.Fragmenter; +import org.apache.hawq.pxf.api.Fragment; +import org.apache.hawq.pxf.api.utilities.InputData; +import java.util.List; + +/** + * Class that defines the splitting of a data resource into fragments that can be processed in parallel + * getFragments() returns the fragments information of a given path (source name and location of each fragment). + * + * Demo implementation + */ + +public class DemoFragmenter extends Fragmenter{ + /** + * Constructs the DemoFragmenter + * + * @param metaData + */ + public DemoFragmenter(InputData metaData) { + super(metaData); + } + + /** + * Provide metadata for each data partition of the given datasource + * + * @return list of fragments + */ + @Override + public List<Fragment> getFragments() throws Exception { + String localhostname = java.net.InetAddress.getLocalHost().getHostName(); + String[] localHosts = new String[]{localhostname, localhostname}; + fragments.add(new Fragment(inputData.getDataSource() + ".1", localHosts, "fragment1".getBytes())); + fragments.add(new Fragment(inputData.getDataSource() + ".2", localHosts, "fragment2".getBytes())); + fragments.add(new Fragment(inputData.getDataSource() + ".3", localHosts, "fragment3".getBytes())); + return fragments; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java new file mode 100644 index 0000000..8780a19 --- /dev/null +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoResolver.java @@ -0,0 +1,67 @@ +package org.apache.hawq.pxf.api.examples; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.hawq.pxf.api.OneField; +import org.apache.hawq.pxf.api.OneRow; +import org.apache.hawq.pxf.api.ReadResolver; +import org.apache.hawq.pxf.api.utilities.InputData; +import org.apache.hawq.pxf.api.utilities.Plugin; +import java.util.LinkedList; +import java.util.List; +import static org.apache.hawq.pxf.api.io.DataType.INTEGER; +import static org.apache.hawq.pxf.api.io.DataType.VARCHAR; + +/** + * Class that defines the deserializtion of one record brought from the external input data. + * + * Demo implementation that returns record custom format + */ +public class DemoResolver extends Plugin implements ReadResolver { + /** + * Constructs the DemoResolver + * + * @param metaData + */ + public DemoResolver(InputData metaData) { + super(metaData); + } + + /** + * Read the next record + * The record contains as many fields as defined by the DDL schema. + * + * @param row one record + * @return list of fields or columns + */ + @Override + public List<OneField> getFields(OneRow row) throws Exception { + List<OneField> output = new LinkedList<OneField>(); + Object data = row.getData(); + + /* break up the row into fields */ + String[] fields = ((String) data).split(","); + for(int colIndex=0; colIndex<fields.length; colIndex++) { + output.add(new OneField(VARCHAR.getOID(), fields[colIndex])); + } + + return output; + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java new file mode 100644 index 0000000..772550e --- /dev/null +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/examples/DemoTextResolver.java @@ -0,0 +1,63 @@ +package org.apache.hawq.pxf.api.examples; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.hawq.pxf.api.OneField; +import org.apache.hawq.pxf.api.OneRow; +import org.apache.hawq.pxf.api.ReadResolver; +import org.apache.hawq.pxf.api.utilities.InputData; +import org.apache.hawq.pxf.api.utilities.Plugin; + +import java.util.LinkedList; +import java.util.List; + +import static org.apache.hawq.pxf.api.io.DataType.INTEGER; +import static org.apache.hawq.pxf.api.io.DataType.VARCHAR; + +/** + * Class that defines the deserializtion of one record brought from the external input data. + * + * Demo implementation of resolver that returns text format + */ +public class DemoTextResolver extends Plugin implements ReadResolver { + /** + * Constructs the DemoResolver + * + * @param metaData + */ + public DemoTextResolver(InputData metaData) { + super(metaData); + } + + /** + * Read the next record + * The record contains as many fields as defined by the DDL schema. + * + * @param row one record + * @return the first column contains the entire text data + */ + @Override + public List<OneField> getFields(OneRow row) throws Exception { + List<OneField> output = new LinkedList<OneField>(); + Object data = row.getData(); + output.add(new OneField(VARCHAR.getOID(), data)); + return output; + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java new file mode 100755 index 0000000..36cbf21 --- /dev/null +++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoAccessorTest.java @@ -0,0 +1,79 @@ +package org.apache.hawq.pxf.api; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + + + +import org.apache.hawq.pxf.api.examples.DemoAccessor; +import org.apache.hawq.pxf.api.utilities.InputData; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.when; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls + +public class DemoAccessorTest { + + @Mock InputData inputData; + DemoAccessor accessor; + + @Before + public void setup() throws Exception { + accessor = new DemoAccessor(inputData); + } + + @Test + public void testRowsWithSingleColumn() throws Exception { + + when(inputData.getDataFragment()).thenReturn(0); + when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes()); + when(inputData.getColumns()).thenReturn(1); + + int numRows = 2; + for (int i = 0; i < numRows; i++) { + OneRow row = accessor.readNextObject(); + assertEquals(row.toString(), "OneRow:0." + i + "->fragment1 row" + (i+1)); + } + assertNull(accessor.readNextObject()); + } + + @Test + public void testRowsWithMultipleColumns() throws Exception { + + when(inputData.getDataFragment()).thenReturn(0); + when(inputData.getFragmentMetadata()).thenReturn("fragment1".getBytes(), "fragment1".getBytes()); + when(inputData.getColumns()).thenReturn(3); + + int numRows = 2; + for (int i = 0; i < numRows; i++) { + OneRow row = accessor.readNextObject(); + assertEquals(row.toString(), "OneRow:0." + i + "->fragment1 row" + (i+1) + ",value1,value2"); + } + assertNull(accessor.readNextObject()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2222f4b4/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java new file mode 100755 index 0000000..dd22c87 --- /dev/null +++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/DemoResolverTest.java @@ -0,0 +1,70 @@ +package org.apache.hawq.pxf.api; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.apache.hawq.pxf.api.examples.DemoAccessor; +import org.apache.hawq.pxf.api.examples.DemoResolver; +import org.apache.hawq.pxf.api.examples.DemoTextResolver; +import org.apache.hawq.pxf.api.utilities.InputData; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.List; + +import static org.junit.Assert.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({DemoAccessor.class}) // Enables mocking 'new' calls + +public class DemoResolverTest { + + @Mock InputData inputData; + DemoResolver customResolver; + DemoTextResolver textResolver; + OneRow row; + + @Before + public void setup() throws Exception { + customResolver = new DemoResolver(inputData); + textResolver = new DemoTextResolver(inputData); + row = new OneRow("0.0","value1,value2"); + } + + @Test + public void testCustomData() throws Exception { + + List<OneField> output = customResolver.getFields(row); + assertEquals("value1", output.get(0).toString()); + assertEquals("value2", output.get(1).toString()); + } + + @Test + public void testTextData() throws Exception { + + List<OneField> output = textResolver.getFields(row); + assertEquals("value1,value2", output.get(0).toString()); + + } +}
