This is an automated email from the ASF dual-hosted git repository.
jenniferdai pushed a commit to branch orc
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/orc by this push:
new d9b0bce Addressing comments
d9b0bce is described below
commit d9b0bcec4ed373b92ddc6feb11835b5bb0287b31
Author: Jennifer Dai <[email protected]>
AuthorDate: Wed Mar 20 15:41:24 2019 -0700
Addressing comments
---
.../pinot/orc/data/readers/ORCRecordReader.java | 33 +++------
.../orc/data/readers/ORCRecordReaderTest.java | 81 ++++++++++++++++++++++
2 files changed, 90 insertions(+), 24 deletions(-)
diff --git
a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
index 5fce029..4e5a8b3 100644
---
a/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
+++
b/pinot-orc/src/main/java/org/apache/pinot/orc/data/readers/ORCRecordReader.java
@@ -19,7 +19,6 @@ package org.apache.pinot.orc.data.readers;
* under the License.
*/
-import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -67,9 +66,9 @@ public class ORCRecordReader implements RecordReader {
private static final Logger LOGGER =
LoggerFactory.getLogger(ORCRecordReader.class);
- @VisibleForTesting
- public ORCRecordReader(String inputPath) {
+ private void init(String inputPath, Schema schema) {
Configuration conf = new Configuration();
+ LOGGER.info("Creating segment for {}", inputPath);
try {
Path orcReaderPath = new Path("file://" + inputPath);
LOGGER.info("orc reader path is {}", orcReaderPath);
@@ -77,32 +76,13 @@ public class ORCRecordReader implements RecordReader {
_orcSchema = _reader.getSchema();
LOGGER.info("ORC schema is {}", _orcSchema.toJson());
- _recordReader = _reader.rows(_reader.options().schema(_orcSchema));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- _reusableVectorizedRowBatch = _orcSchema.createRowBatch(1);
- }
-
- @Override
- public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
- Configuration conf = new Configuration();
- LOGGER.info("Creating segment for {}",
segmentGeneratorConfig.getInputFilePath());
- try {
- Path orcReaderPath = new Path("file://" +
segmentGeneratorConfig.getInputFilePath());
- LOGGER.info("orc reader path is {}", orcReaderPath);
- _reader = OrcFile.createReader(orcReaderPath,
OrcFile.readerOptions(conf));
- _orcSchema = _reader.getSchema();
- LOGGER.info("ORC schema is {}", _orcSchema.toJson());
-
- _pinotSchema = segmentGeneratorConfig.getSchema();
+ _pinotSchema = schema;
if (_pinotSchema == null) {
LOGGER.warn("Pinot schema is not set in segment generator config");
}
_recordReader = _reader.rows(_reader.options().schema(_orcSchema));
} catch (Exception e) {
- LOGGER.error("Caught exception initializing record reader at path {}",
segmentGeneratorConfig.getInputFilePath());
+ LOGGER.error("Caught exception initializing record reader at path {}",
inputPath);
throw new RuntimeException(e);
}
@@ -111,6 +91,11 @@ public class ORCRecordReader implements RecordReader {
}
@Override
+ public void init(SegmentGeneratorConfig segmentGeneratorConfig) {
+ init(segmentGeneratorConfig.getInputFilePath(),
segmentGeneratorConfig.getSchema());
+ }
+
+ @Override
public boolean hasNext() {
try {
return _recordReader.getProgress() != 1;
diff --git
a/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
new file mode 100644
index 0000000..6f5965f
--- /dev/null
+++
b/pinot-orc/src/test/java/org/apache/pinot/orc/data/readers/ORCRecordReaderTest.java
@@ -0,0 +1,81 @@
+package org.apache.pinot.orc.data.readers;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Writer;
+import org.apache.orc.impl.writer.WriterImplV2;
+import org.apache.orc.mapred.OrcMapredRecordWriter;
+import org.apache.pinot.core.data.GenericRow;
+import org.apache.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class ORCRecordReaderTest {
+ private static final String ORC_DATA = "_test_sample_orc_data.orc";
+
+ @BeforeClass
+ public void setUp()
+ throws Exception {
+
+ }
+
+ @Test
+ public void testReadData()
+ throws IOException {
+// OrcOutputFormat orcOutputFormat = new OrcOutputFormat();
+// Writer recordWriter = orcOutputFormat.getRecordWriter(new
LocalFileSystem(), new JobConf(),
+// File.createTempFile("orc", "file").getAbsolutePath(), null);
+
+ String path = "";
+ Writer writer = new WriterImplV2(new LocalFileSystem(),
+ new Path(path), OrcFile.writerOptions(new Configuration()));
+ OrcMapredRecordWriter orcMapredRecordWriter = new
OrcMapredRecordWriter(writer);
+ orcMapredRecordWriter.write("1");
+// OrcMapredRecordWriter orcMapredRecordWriter = new
OrcMapredRecordWriter(recordWriter);
+// ORCRecordReader recordReader = new ORCRecordReader(ORC_DATA);
+
+ SegmentGeneratorConfig segmentGeneratorConfig = new
SegmentGeneratorConfig();
+ segmentGeneratorConfig.setInputFilePath(ORC_DATA);
+ recordReader.init(segmentGeneratorConfig);
+
+ List<GenericRow> genericRows = new ArrayList<>();
+ while (recordReader.hasNext()) {
+ genericRows.add(recordReader.next());
+ }
+ recordReader.close();
+ Assert.assertEquals(genericRows.size(), 2, "Generic row size must be 2");
+ int id = 1;
+ for (GenericRow outputRow : genericRows) {
+ Assert.assertEquals(outputRow.getValue("id"), id);
+ Assert.assertNull(outputRow.getValue("map_values"));
+ id++;
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
