This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 87a15d2860a [IOTDB-6189] Fix bugs for csv-tools (The output is
incorrect when the import fails. The csv file with bom header can not be
imported. The last line can not be imported) (#11308)
87a15d2860a is described below
commit 87a15d2860a824f0a363461af6cb53e38ce69496
Author: Xuan Ronaldo <[email protected]>
AuthorDate: Tue Oct 17 11:30:15 2023 +0800
[IOTDB-6189] Fix bugs for csv-tools (The output is incorrect when the
import fails. The csv file with bom header can not be imported. The last line
can not be imported) (#11308)
---
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 84 ++++++++++++----------
1 file changed, 48 insertions(+), 36 deletions(-)
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
index f6ea5138645..7919e3f7302 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
@@ -417,8 +417,8 @@ public class ImportCsv extends AbstractCsvTool {
IoTPrinter.println("Empty file!");
return;
}
- if (!timeColumn.equalsIgnoreCase(headerNames.get(0))) {
- IoTPrinter.println("No headers!");
+ if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0))))
{
+ IoTPrinter.println("The first field of header must be `Time`!");
return;
}
String failedFilePath = null;
@@ -458,11 +458,7 @@ public class ImportCsv extends AbstractCsvTool {
Set<String> devices = deviceAndMeasurementNames.keySet();
if (headerTypeMap.isEmpty()) {
- try {
- queryType(devices, headerTypeMap, "Time");
- } catch (IoTDBConnectionException e) {
- IoTPrinter.printException(e);
- }
+ queryType(devices, headerTypeMap, "Time");
}
List<String> deviceIds = new ArrayList<>();
@@ -620,27 +616,21 @@ public class ImportCsv extends AbstractCsvTool {
boolean hasResult = false;
// query the data type in iotdb
if (!typeQueriedDevice.contains(deviceName.get())) {
- try {
- if (headerTypeMap.isEmpty()) {
- Set<String> devices = new HashSet<>();
- devices.add(deviceName.get());
- hasResult = queryType(devices, headerTypeMap,
deviceColumn);
- }
- typeQueriedDevice.add(deviceName.get());
- } catch (IoTDBConnectionException e) {
- IoTPrinter.printException(e);
+ if (headerTypeMap.isEmpty()) {
+ Set<String> devices = new HashSet<>();
+ devices.add(deviceName.get());
+ queryType(devices, headerTypeMap, deviceColumn);
}
+ typeQueriedDevice.add(deviceName.get());
}
- if (!hasResult) {
- type = typeInfer(value);
- if (type != null) {
- headerTypeMap.put(headerNameWithoutType, type);
- } else {
- IoTPrinter.printf(
- "Line '%s', column '%s': '%s' unknown type%n",
- recordObj.getRecordNumber(), headerNameWithoutType,
value);
- isFail.set(true);
- }
+ type = typeInfer(value);
+ if (type != null) {
+ headerTypeMap.put(headerNameWithoutType, type);
+ } else {
+ IoTPrinter.printf(
+ "Line '%s', column '%s': '%s' unknown type%n",
+ recordObj.getRecordNumber(), headerNameWithoutType,
value);
+ isFail.set(true);
}
}
type = headerTypeMap.get(headerNameWithoutType);
@@ -670,7 +660,7 @@ public class ImportCsv extends AbstractCsvTool {
measurementsList.add(measurements);
}
});
- if (times.isEmpty()) {
+ if (!times.isEmpty()) {
writeAndEmptyDataSet(deviceName.get(), times, typesList, valuesList,
measurementsList, 3);
pointSize.set(0);
}
@@ -722,6 +712,12 @@ public class ImportCsv extends AbstractCsvTool {
}
} catch (StatementExecutionException e) {
IoTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
+ try {
+ session.close();
+ } catch (IoTDBConnectionException ex) {
+ // do nothing
+ }
+ System.exit(1);
} finally {
times.clear();
typesList.clear();
@@ -755,6 +751,12 @@ public class ImportCsv extends AbstractCsvTool {
}
} catch (StatementExecutionException e) {
IoTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
+ try {
+ session.close();
+ } catch (IoTDBConnectionException ex) {
+ // do nothing
+ }
+ System.exit(1);
} finally {
deviceIds.clear();
times.clear();
@@ -802,7 +804,7 @@ public class ImportCsv extends AbstractCsvTool {
String regex = "(?<=\\()\\S+(?=\\))";
Pattern pattern = Pattern.compile(regex);
for (String headerName : headerNames) {
- if ("Time".equalsIgnoreCase(headerName)) {
+ if ("Time".equalsIgnoreCase(filterBomHeader(headerName))) {
timeColumn = headerName;
continue;
} else if ("Device".equalsIgnoreCase(headerName)) {
@@ -840,10 +842,8 @@ public class ImportCsv extends AbstractCsvTool {
* @throws IoTDBConnectionException
* @throws StatementExecutionException
*/
- private static boolean queryType(
- Set<String> deviceNames, HashMap<String, TSDataType> headerTypeMap,
String alignedType)
- throws IoTDBConnectionException {
- boolean hasResult = false;
+ private static void queryType(
+ Set<String> deviceNames, HashMap<String, TSDataType> headerTypeMap,
String alignedType) {
for (String deviceName : deviceNames) {
String sql = "show timeseries " + deviceName + ".*";
SessionDataSet sessionDataSet = null;
@@ -852,7 +852,6 @@ public class ImportCsv extends AbstractCsvTool {
int tsIndex =
sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES);
int dtIndex =
sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE);
while (sessionDataSet.hasNext()) {
- hasResult = true;
RowRecord rowRecord = sessionDataSet.next();
List<Field> fields = rowRecord.getFields();
String timeseries = fields.get(tsIndex).getStringValue();
@@ -865,13 +864,17 @@ public class ImportCsv extends AbstractCsvTool {
headerTypeMap.put(measurement, getType(dataType));
}
}
- } catch (StatementExecutionException | IllegalPathException e) {
+ } catch (StatementExecutionException | IllegalPathException |
IoTDBConnectionException e) {
IoTPrinter.println(
"Meet error when query the type of timeseries because " +
e.getMessage());
- return false;
+ try {
+ session.close();
+ } catch (IoTDBConnectionException ex) {
+ // do nothing
+ }
+ System.exit(1);
}
}
- return hasResult;
}
/**
@@ -995,4 +998,13 @@ public class ImportCsv extends AbstractCsvTool {
}
return timestamp;
}
+
+ private static String filterBomHeader(String s) {
+ byte[] bom = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF};
+ byte[] bytes = Arrays.copyOf(s.getBytes(), 3);
+ if (Arrays.equals(bom, bytes)) {
+ return s.substring(1);
+ }
+ return s;
+ }
}
