Github user fhueske commented on a diff in the pull request:
https://github.com/apache/flink/pull/4660#discussion_r139295981
--- Diff:
flink-libraries/flink-table/src/main/java/org/apache/flink/table/runtime/batch/io/RFCCsvInputFormat.java
---
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.batch.io;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.io.ParseException;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.types.parser.FieldParser;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
+import com.fasterxml.jackson.databind.MappingIterator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvParser;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+
+import org.apache.commons.io.input.BoundedInputStream;
+
+import java.io.IOException;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * InputFormat that reads csv files and compliant with RFC 4180 standards.
+ *
+ * @param <OUT>
+ */
+@Internal
+public abstract class RFCCsvInputFormat<OUT> extends FileInputFormat<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String DEFAULT_LINE_DELIMITER = "\n";
+
+ public static final String DEFAULT_FIELD_DELIMITER = ",";
+
+ private String recordDelimiter = "\n";
+ private char fieldDelimiter = ',';
+
+ private boolean skipFirstLineAsHeader;
+ private boolean skipFirstLine = false; // only for first split
+
+ private boolean quotedStringParsing = false;
+ private char quoteCharacter;
+
+ private boolean lenient;
+
+ private String commentPrefix = null;
+ private boolean allowComments = false;
+
+ private static final Class<?>[] EMPTY_TYPES = new Class<?>[0];
+
+ private static final boolean[] EMPTY_INCLUDED = new boolean[0];
+
+ private Class<?>[] fieldTypes = EMPTY_TYPES;
+
+ private boolean[] fieldIncluded = EMPTY_INCLUDED;
+
+ MappingIterator<Object[]> recordIterator = null;
+
+ private boolean endOfSplit = false;
+
+ protected RFCCsvInputFormat(Path filePath) {
+ super(filePath);
+ }
+
+ @Override
+ public void open(FileInputSplit split) throws IOException {
+
+ super.open(split);
+
+ CsvMapper mapper = new CsvMapper();
+ mapper.disable(CsvParser.Feature.WRAP_AS_ARRAY);
+
+ long firstDelimiterPosition = findFirstDelimiterPosition();
+ long lastDelimiterPosition = findLastDelimiterPosition();
+ long startPos = this.splitStart + firstDelimiterPosition;
+ long endPos = this.splitLength + lastDelimiterPosition -
firstDelimiterPosition;
+ this.stream.seek(startPos);
+ BoundedInputStream boundedInputStream = new
BoundedInputStream(this.stream, endPos);
+
+ if (skipFirstLineAsHeader && startPos == 0) {
+ skipFirstLine = true;
+ }
+
+ CsvParser csvParser =
mapper.getFactory().createParser(boundedInputStream);
+ CsvSchema csvSchema = configureParserSettings();
+ csvParser.setSchema(csvSchema);
+
+ recordIterator =
mapper.readerFor(Object[].class).readValues(csvParser);
+ }
+
+ private CsvSchema configureParserSettings() {
+
+ CsvSchema csvSchema = CsvSchema.builder()
+ .setLineSeparator(this.recordDelimiter)
+ .setColumnSeparator(this.fieldDelimiter)
+ .setSkipFirstDataRow(skipFirstLine)
+ .setQuoteChar(this.quoteCharacter)
+ .setAllowComments(allowComments)
+ .build();
+ return csvSchema;
+ }
+
+ public OUT nextRecord(OUT reuse) throws IOException {
+
+ if (recordIterator == null) {
+ return null;
+ }
+
+ if (recordIterator.hasNext()) {
+
+ Object[] record = recordIterator.next();
--- End diff --
does this call never fail or should we add a `lenient` check here as well?
---
