[ 
https://issues.apache.org/jira/browse/DRILL-7177?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16955557#comment-16955557
 ] 

ASF GitHub Bot commented on DRILL-7177:
---------------------------------------

paul-rogers commented on pull request #1749: DRILL-7177: Format Plugin for 
Excel Files
URL: https://github.com/apache/drill/pull/1749#discussion_r334292293
 
 

 ##########
 File path: 
contrib/format-excel/src/main/java/org/apache/drill/exec/store/excel/ExcelBatchReader.java
 ##########
 @@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.store.excel;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.physical.impl.scan.file.FileScanFramework;
+import org.apache.drill.exec.physical.impl.scan.framework.ManagedReader;
+import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
+import org.apache.drill.exec.physical.resultSet.RowSetLoader;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.MetadataUtils;
+import org.apache.drill.exec.record.metadata.SchemaBuilder;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.vector.accessor.ScalarWriter;
+import org.apache.drill.exec.vector.accessor.TupleWriter;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellValue;
+import org.apache.poi.ss.usermodel.DateUtil;
+import org.apache.poi.ss.usermodel.FormulaEvaluator;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import 
org.apache.drill.exec.physical.impl.scan.file.FileScanFramework.FileSchemaNegotiator;
+import org.joda.time.Instant;
+
+import java.util.Iterator;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class ExcelBatchReader implements ManagedReader<FileSchemaNegotiator> {
+  private ExcelReaderConfig readerConfig;
+
+  private static final org.slf4j.Logger logger = 
org.slf4j.LoggerFactory.getLogger(ExcelBatchReader.class);
+
+  private XSSFWorkbook workbook;
+
+  private FSDataInputStream fsStream;
+
+  private static final String SAFE_WILDCARD = "_$";
+
+  private static final String SAFE_SEPARATOR = "_";
+
+  private static final String PARSER_WILDCARD = ".*";
+
+  private static final String MISSING_FIELD_NAME_HEADER = "field_";
+
+  private static final String SAFE_NEWLINE_REPLACEMENT = " ";
+
+  private XSSFSheet sheet;
+
+  private FormulaEvaluator evaluator;
+
+  private ArrayList<String> excelFieldNames;
+
+  private Iterator<Row> rowIterator;
+
+  private int totalColumnCount;
+
+  private int lineCount;
+
+  private FileSplit split;
+
+  private ResultSetLoader loader;
+
+  private int recordCount;
+
+  public static class ExcelReaderConfig {
+    protected final ExcelFormatPlugin plugin;
+
+    protected int headerRow;
+
+    protected int lastRow;
+
+    protected int firstColumn;
+
+    protected int lastColumn;
+
+    protected boolean readAllFieldsAsVarChar;
+
+    protected boolean evaluateFormulae;
+
+    protected TupleMetadata schema;
+
+    protected String sheetName;
+
+    public ExcelReaderConfig(ExcelFormatPlugin plugin, int headerRow, int 
lastRow, int firstColumn, int lastColumn, boolean readAllFieldsAsVarChar, 
boolean evaluateFormulae,
+                             //TupleMetadata schema,
+                             String sheetName) {
+      this.plugin = plugin;
+      this.headerRow = headerRow;
+      this.lastRow = lastRow;
+      this.firstColumn = firstColumn;
+      this.lastColumn = lastColumn;
+      this.readAllFieldsAsVarChar = readAllFieldsAsVarChar;
+      this.evaluateFormulae = evaluateFormulae;
+      this.sheetName = sheetName;
+
+    }
+  }
+
+  public ExcelBatchReader(ExcelReaderConfig readerConfig) {
+    this.readerConfig = readerConfig;
+  }
+
+  @Override
+  public boolean open(FileSchemaNegotiator negotiator) {
+    verifyConfigOptions();
+    split = negotiator.split();
+    openFile(negotiator);
+    TupleMetadata schema = defineSchema();
+    this.loader = negotiator.build();
+    return true;
+  }
+
+  private void openFile(FileScanFramework.FileSchemaNegotiator negotiator) {
+    try {
+      String filePath = split.getPath().toString();
+      this.fsStream = negotiator.fileSystem().open(new Path(filePath));
+      this.workbook = new XSSFWorkbook(this.fsStream.getWrappedStream());
+    } catch (Exception e) {
+      throw UserException.dataReadError(e).message("Failed to open open input 
file: %s", split.getPath().toString()).message(e.getMessage()).build(logger);
+    }
+
+    // Evaluate formulae
+    if (readerConfig.evaluateFormulae) {
+      this.evaluator = workbook.getCreationHelper().createFormulaEvaluator();
+    }
+    
this.workbook.setMissingCellPolicy(Row.MissingCellPolicy.CREATE_NULL_AS_BLANK);
+    this.sheet = getSheet();
+  }
+
+  /**
+   * This helper function gets the column headers.
+   * If the user specified no columns headers it populates an array with field 
names of field_n
+   */
+  private TupleMetadata defineSchema() {
+    // TODO Use case if there are no column headers
+    SchemaBuilder builder = new SchemaBuilder();
+    return getColumnHeaders(builder);
+  }
+
+  protected TupleMetadata getColumnHeaders(SchemaBuilder builder) {
+    //Get the field names
+    int columnCount = 0;
+    if (readerConfig.headerRow >= 0) {
+      columnCount = 
sheet.getRow(readerConfig.headerRow).getPhysicalNumberOfCells();
+    } else {
+      columnCount = sheet.getRow(0).getPhysicalNumberOfCells();
+    }
+    this.excelFieldNames = new ArrayList<>(columnCount);
+    this.rowIterator = sheet.iterator();
+
+    //If there are no headers, create columns names of field_n
+    if (readerConfig.headerRow == -1) {
+      String missingFieldName;
+      for (int i = 0; i < columnCount; i++) {
+        missingFieldName = MISSING_FIELD_NAME_HEADER + (i + 1);
+        ExcelFormatPlugin.makeColumn(builder, missingFieldName, 
TypeProtos.MinorType.VARCHAR);
+        excelFieldNames.add(i, missingFieldName);
+      }
+      return builder.buildSchema();
+    } else if (rowIterator.hasNext()) {
+      //Find the header row
+      while (this.lineCount < readerConfig.headerRow) {
+        Row row = rowIterator.next();
+        this.lineCount++;
+      }
+      //Get the header row and column count
+      Row row = rowIterator.next();
+      this.totalColumnCount = row.getLastCellNum();
+
+      //Read the header row
+      Iterator<Cell> cellIterator = row.cellIterator();
+      int colPosition = 0;
+      String tempColumnName = "";
+
+      while (cellIterator.hasNext()) {
+        Cell cell = cellIterator.next();
+        // TODO Potential NPE if cell evaluation is turned off
+        CellValue cellValue = evaluator.evaluate(cell);
+        switch (cellValue.getCellTypeEnum()) {
+          case STRING:
+            tempColumnName = cell.getStringCellValue()
+              //.replaceAll("_", "__")  // Not sure if we need this
+              .replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", 
SAFE_SEPARATOR);
+            ExcelFormatPlugin.makeColumn(builder, tempColumnName, 
TypeProtos.MinorType.VARCHAR);
+            excelFieldNames.add(colPosition, tempColumnName);
+            break;
+          case NUMERIC:
+            tempColumnName = String.valueOf(cell.getNumericCellValue());
+            ExcelFormatPlugin.makeColumn(builder, tempColumnName, 
TypeProtos.MinorType.FLOAT8);
+            excelFieldNames.add(colPosition, tempColumnName);
 
 Review comment:
   The above two lines appear the same, save type, in both cases. Pull out as a 
function?
 
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


> Format Plugin for Excel Files
> -----------------------------
>
>                 Key: DRILL-7177
>                 URL: https://issues.apache.org/jira/browse/DRILL-7177
>             Project: Apache Drill
>          Issue Type: Improvement
>    Affects Versions: 1.17.0
>            Reporter: Charles Givre
>            Assignee: Charles Givre
>            Priority: Major
>              Labels: doc-impacting
>             Fix For: 1.17.0
>
>
> This pull request adds the functionality which enables Drill to query 
> Microsoft Excel files. 



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

Reply via email to