exceptionfactory commented on code in PR #9379: URL: https://github.com/apache/nifi/pull/9379#discussion_r1800137398
########## nifi-extension-bundles/nifi-poi-bundle/nifi-poi-services/src/main/java/org/apache/nifi/excel/StandardCellFieldTypeReader.java: ########## @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.excel; + +import org.apache.nifi.schema.inference.FieldTypeInference; +import org.apache.nifi.schema.inference.TimeValueInference; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.DateUtil; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Standard implementation of Cell Field Type Reader uses Cell Type and Cell Value information with inference based on Timestamp formats + */ +class StandardCellFieldTypeReader implements CellFieldTypeReader { + private final TimeValueInference timeValueInference; + + StandardCellFieldTypeReader(final TimeValueInference timeValueInference) { + this.timeValueInference = Objects.requireNonNull(timeValueInference, "Time Value Inference required"); + } + + /** + * Infer Cell Field Type and append possible Data Types to mapped Field Type Inference information + * + * @param cell Spreadsheet Cell required + * @param fieldName Cell field name for tracking in Field Types + * @param fieldTypes Map of Field Name to Field Type Inference information + */ + @Override + public void inferCellFieldType(final Cell cell, final String fieldName, final Map<String, FieldTypeInference> fieldTypes) { + Objects.requireNonNull(cell, "Cell required"); + Objects.requireNonNull(fieldName, "Field Name required"); + Objects.requireNonNull(fieldTypes, "Field Types required"); + + final FieldTypeInference fieldTypeInference = fieldTypes.computeIfAbsent(fieldName, key -> new FieldTypeInference()); + final DataType dataType = getCellDataType(cell); + fieldTypeInference.addPossibleDataType(dataType); + } + + /** + * Get Record Data Type from Spreadsheet Cell Type and additional resolution of NUMERIC and STRING types + * + * @param cell Spreadsheet Cell required + * @return Record Data Type + */ + @Override + public DataType getCellDataType(final Cell cell) { + Objects.requireNonNull(cell, "Cell required"); + + final CellType cellType = cell.getCellType(); + + final DataType dataType; + + if (CellType.NUMERIC == cellType) { + // Date Formatting check limited to NUMERIC Cell Types + if (DateUtil.isCellDateFormatted(cell)) { + // Default to TIMESTAMP for date formatted values using standard Date Time Pattern + dataType = RecordFieldType.TIMESTAMP.getDataType(); Review Comment: Thanks for highlighting these details @dan-s1. This is an important issue of the way that Excel stores dates, and the way the POI library handles dates. For all date and time values, the POI library always returns a `java.sql.Timestamp` or ` java.time.LocalDateTime`. The library interprets date values with zero for hours, minutes, and seconds. For time values, the POI library uses a default starting date, that is the reason for these inferred values. When using a defined schema, the time and date values will be handled as expected. When using an inferred schema, the Schema Inference logic is unable to differentiate between Timestamp, Date, and Time, so the inferred Schema handles everything as a Timestamp. It might be possible to evaluate some Timestamp values and infer that they contain only a Date or only Time, but such inference would be prone to error. For this scenario, the inference process is necessarily general, and is unable to narrow down values to either or Date or Time. It might be possible to try something based on the date format from Excel, if specified, but this is also prone to error based on the variety of values, which are different than Java date formatting. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
