SteveYurongSu commented on a change in pull request #1914:
URL: https://github.com/apache/iotdb/pull/1914#discussion_r526835324
##########
File path: README_ZH.md
##########
@@ -75,9 +75,7 @@ IoTDB的主要特点如下:
- [从 csv 文件导入数据的示例](#从-csv-文件导入数据的示例)
- [运行 import shell](#运行-import-shell)
- [错误的数据文件](#错误的数据文件)
- - [使用 export-csv.sh](#使用-export-csvsh)
- - [运行 export shell](#运行-export-shell)
- - [执行查询](#执行查询)
+ - [导入导出CSV工具](#导入导出CSV工具)
Review comment:
Remove "使用 import-csv.sh" please.
##########
File path: README_ZH.md
##########
@@ -333,61 +331,9 @@ server 可以使用 "ctrl-C" 或者执行下面的脚本:
编译完成后, IoTDB cli 将生成在 "cli/target/iotdb-cli-{project.version}".
-## 使用 import-csv.sh
-
-### 创建元数据
-
-```
-SET STORAGE GROUP TO root.fit.d1;
-SET STORAGE GROUP TO root.fit.d2;
-SET STORAGE GROUP TO root.fit.p;
-CREATE TIMESERIES root.fit.d1.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d1.s2 WITH DATATYPE=TEXT,ENCODING=PLAIN;
-CREATE TIMESERIES root.fit.d2.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d2.s3 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.p.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-```
-
-### 从 csv 文件导入数据的示例
-
-```
-Time,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3,root.fit.p.s1
-1,100,'hello',200,300,400
-2,500,'world',600,700,800
-3,900,'IoTDB',1000,1100,1200
-```
-
-### 运行 import shell
-```
-# Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-
-# Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-```
-
-### 错误的数据文件
-
-`csvInsertError.error`
-
-## 使用 export-csv.sh
-
-### 运行 export shell
-
-```
-# Unix/OS X
-> tools/export-csv.sh -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-
-# Windows
-> tools\export-csv.bat -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-```
-
-### 执行查询
-
-```
-select * from root.fit.d1
-```
+# 导入导出CSV工具
+查看 [导入导出CSV工具](docs/zh/UserGuide/System Tools/CSV Tool.md)
Review comment:
This is a broken link...
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
##########
@@ -263,112 +228,123 @@ private static void dumpResult(String sql, int index)
return;
}
} catch (IOException e) {
- System.out.println("Cannot create dump file " + path + "because: " +
e.getMessage());
+ System.out.println("Cannot create dump file " + path + " " + "because: "
+ e.getMessage());
return;
}
System.out.println("Start to export data from sql statement: " + sql);
- try (Statement statement = connection.createStatement();
- ResultSet rs = statement.executeQuery(sql);
- BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
- ResultSetMetaData metadata = rs.getMetaData();
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql);
long startTime = System.currentTimeMillis();
-
- int count = metadata.getColumnCount();
// write data in csv file
- writeMetadata(bw, count, metadata);
+ writeMetadata(bw, sessionDataSet.getColumnNames());
- int line = writeResultSet(rs, bw, count);
+ int line = writeResultSet(sessionDataSet, bw);
System.out
- .println(String.format("Statement [%s] has dumped to file %s
successfully! It costs "
- + "%dms to export %d lines.", sql, path,
System.currentTimeMillis() - startTime,
- line));
- } catch (IOException e) {
+ .printf("Statement [%s] has dumped to file %s successfully! It costs
"
+ + "%dms to export %d lines.%n", sql, path,
System.currentTimeMillis() - startTime,
+ line);
+ } catch (IOException | StatementExecutionException |
IoTDBConnectionException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
}
- private static void writeMetadata(BufferedWriter bw, int count,
ResultSetMetaData metadata)
- throws SQLException, IOException {
- for (int i = 1; i <= count; i++) {
- if (i < count) {
- bw.write(metadata.getColumnLabel(i) + ",");
- } else {
- bw.write(metadata.getColumnLabel(i) + "\n");
- }
- typeList.add(metadata.getColumnType(i));
+ private static void writeMetadata(BufferedWriter bw, List<String>
columnNames)
+ throws IOException {
+ if (!columnNames.get(0).equals("Time")) {
+ bw.write("Time" + ",");
+ }
+ for (int i = 0; i < columnNames.size() - 1; i++) {
+ bw.write(columnNames.get(i) + ",");
}
+ bw.write(columnNames.get(columnNames.size() - 1) + "\n");
}
- private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
- throws SQLException, IOException {
+ private static int writeResultSet(SessionDataSet rs, BufferedWriter bw)
+ throws IOException, StatementExecutionException,
IoTDBConnectionException {
int line = 0;
long timestamp = System.currentTimeMillis();
- while (rs.next()) {
- if (rs.getString(1) == null ||
- "null".equalsIgnoreCase(rs.getString(1))) {
- bw.write(",");
- } else {
- writeTime(rs, bw);
- writeValue(rs, count, bw);
- }
+ while (rs.hasNext()) {
+ RowRecord rowRecord = rs.next();
+ List<Field> fields = rowRecord.getFields();
+ writeTime(rowRecord.getTimestamp(), bw);
+ writeValue(fields, bw);
line++;
if (line % EXPORT_PER_LINE_COUNT == 0) {
long tmp = System.currentTimeMillis();
- System.out.println(
- String.format("%d lines have been exported, it takes %dms", line,
(tmp - timestamp)));
+ System.out.printf("%d lines have been exported, it takes %dms%n",
line, (tmp - timestamp));
timestamp = tmp;
}
}
return line;
}
- private static void writeTime(ResultSet rs, BufferedWriter bw) throws
SQLException, IOException {
+ private static void writeTime(Long time, BufferedWriter bw) throws
IOException {
ZonedDateTime dateTime;
+ String timestampPrecision = "ms";
switch (timeFormat) {
case "default":
- long timestamp = rs.getLong(1);
String str = AbstractCli
-
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME,
timestamp, zoneId,
- TIMESTAMP_PRECISION);
+
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME, time,
zoneId,
+ timestampPrecision);
bw.write(str + ",");
break;
case "timestamp":
case "long":
case "nubmer":
- bw.write(rs.getLong(1) + ",");
+ bw.write(time + ",");
break;
default:
- dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
+ dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(time),
zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) +
",");
break;
}
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private static void writeValue(ResultSet rs, int count, BufferedWriter bw)
- throws SQLException, IOException {
- for (int j = 2; j <= count; j++) {
- if (j < count) {
- if ("null".equals(rs.getString(j))) {
- bw.write(",");
- } else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ ",");
+ private static void writeValue(List<Field> fields, BufferedWriter bw) throws
IOException {
+ for (int j = 0; j < fields.size() - 1; j++) {
+ String value = fields.get(j).getStringValue();
+ if ("null".equalsIgnoreCase(value)) {
+ bw.write(",");
+ } else {
+ if (fields.get(j).getDataType() == TSDataType.TEXT) {
+ int location = value.indexOf("\"");
+ if (location > -1) {
+ if (value.charAt(location - 1) != '\\') {
Review comment:
And why we need this judgement?
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
##########
@@ -263,112 +228,123 @@ private static void dumpResult(String sql, int index)
return;
}
} catch (IOException e) {
- System.out.println("Cannot create dump file " + path + "because: " +
e.getMessage());
+ System.out.println("Cannot create dump file " + path + " " + "because: "
+ e.getMessage());
return;
}
System.out.println("Start to export data from sql statement: " + sql);
- try (Statement statement = connection.createStatement();
- ResultSet rs = statement.executeQuery(sql);
- BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
- ResultSetMetaData metadata = rs.getMetaData();
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql);
long startTime = System.currentTimeMillis();
-
- int count = metadata.getColumnCount();
// write data in csv file
- writeMetadata(bw, count, metadata);
+ writeMetadata(bw, sessionDataSet.getColumnNames());
- int line = writeResultSet(rs, bw, count);
+ int line = writeResultSet(sessionDataSet, bw);
System.out
- .println(String.format("Statement [%s] has dumped to file %s
successfully! It costs "
- + "%dms to export %d lines.", sql, path,
System.currentTimeMillis() - startTime,
- line));
- } catch (IOException e) {
+ .printf("Statement [%s] has dumped to file %s successfully! It costs
"
+ + "%dms to export %d lines.%n", sql, path,
System.currentTimeMillis() - startTime,
+ line);
+ } catch (IOException | StatementExecutionException |
IoTDBConnectionException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
}
- private static void writeMetadata(BufferedWriter bw, int count,
ResultSetMetaData metadata)
- throws SQLException, IOException {
- for (int i = 1; i <= count; i++) {
- if (i < count) {
- bw.write(metadata.getColumnLabel(i) + ",");
- } else {
- bw.write(metadata.getColumnLabel(i) + "\n");
- }
- typeList.add(metadata.getColumnType(i));
+ private static void writeMetadata(BufferedWriter bw, List<String>
columnNames)
+ throws IOException {
+ if (!columnNames.get(0).equals("Time")) {
+ bw.write("Time" + ",");
+ }
+ for (int i = 0; i < columnNames.size() - 1; i++) {
+ bw.write(columnNames.get(i) + ",");
}
+ bw.write(columnNames.get(columnNames.size() - 1) + "\n");
}
- private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
- throws SQLException, IOException {
+ private static int writeResultSet(SessionDataSet rs, BufferedWriter bw)
+ throws IOException, StatementExecutionException,
IoTDBConnectionException {
int line = 0;
long timestamp = System.currentTimeMillis();
- while (rs.next()) {
- if (rs.getString(1) == null ||
- "null".equalsIgnoreCase(rs.getString(1))) {
- bw.write(",");
- } else {
- writeTime(rs, bw);
- writeValue(rs, count, bw);
- }
+ while (rs.hasNext()) {
+ RowRecord rowRecord = rs.next();
+ List<Field> fields = rowRecord.getFields();
+ writeTime(rowRecord.getTimestamp(), bw);
+ writeValue(fields, bw);
line++;
if (line % EXPORT_PER_LINE_COUNT == 0) {
long tmp = System.currentTimeMillis();
- System.out.println(
- String.format("%d lines have been exported, it takes %dms", line,
(tmp - timestamp)));
+ System.out.printf("%d lines have been exported, it takes %dms%n",
line, (tmp - timestamp));
timestamp = tmp;
}
}
return line;
}
- private static void writeTime(ResultSet rs, BufferedWriter bw) throws
SQLException, IOException {
+ private static void writeTime(Long time, BufferedWriter bw) throws
IOException {
ZonedDateTime dateTime;
+ String timestampPrecision = "ms";
switch (timeFormat) {
case "default":
- long timestamp = rs.getLong(1);
String str = AbstractCli
-
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME,
timestamp, zoneId,
- TIMESTAMP_PRECISION);
+
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME, time,
zoneId,
+ timestampPrecision);
bw.write(str + ",");
break;
case "timestamp":
case "long":
case "nubmer":
- bw.write(rs.getLong(1) + ",");
+ bw.write(time + ",");
break;
default:
- dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
+ dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(time),
zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) +
",");
break;
}
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private static void writeValue(ResultSet rs, int count, BufferedWriter bw)
- throws SQLException, IOException {
- for (int j = 2; j <= count; j++) {
- if (j < count) {
- if ("null".equals(rs.getString(j))) {
- bw.write(",");
- } else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ ",");
+ private static void writeValue(List<Field> fields, BufferedWriter bw) throws
IOException {
+ for (int j = 0; j < fields.size() - 1; j++) {
+ String value = fields.get(j).getStringValue();
+ if ("null".equalsIgnoreCase(value)) {
+ bw.write(",");
+ } else {
+ if (fields.get(j).getDataType() == TSDataType.TEXT) {
+ int location = value.indexOf("\"");
+ if (location > -1) {
+ if (value.charAt(location - 1) != '\\') {
+ bw.write("\"" + value.replace("\"", "\\\"") + "\",");
+ } else {
+ bw.write("\"" + value + "\",");
+ }
+ } else if (value.contains(",")) {
+ bw.write("\"" + value + "\",");
} else {
- bw.write(rs.getString(j) + ",");
+ bw.write(value + ",");
}
- }
- } else {
- if ("null".equals(rs.getString(j))) {
- bw.write("\n");
} else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ "\n");
+ bw.write(value + ",");
+ }
+ }
+ }
+ String lastValue = fields.get(fields.size() - 1).getStringValue();
+ if ("null".equalsIgnoreCase(lastValue)) {
+ bw.write("\n");
+ } else {
+ if (fields.get(fields.size() - 1).getDataType() == TSDataType.TEXT) {
+ int location = lastValue.indexOf("\"");
+ if (location > -1) {
+ if (lastValue.charAt(location - 1) != '\\') {
Review comment:
And why we need this judgement?
##########
File path: README.md
##########
@@ -333,59 +326,9 @@ Under the root path of iotdb:
After being built, the IoTDB cli is located at the folder
"cli/target/iotdb-cli-{project.version}".
-## Usage of import-csv.sh
-
-### Create metadata
-```
-SET STORAGE GROUP TO root.fit.d1;
-SET STORAGE GROUP TO root.fit.d2;
-SET STORAGE GROUP TO root.fit.p;
-CREATE TIMESERIES root.fit.d1.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d1.s2 WITH DATATYPE=TEXT,ENCODING=PLAIN;
-CREATE TIMESERIES root.fit.d2.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d2.s3 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.p.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-```
-
-### An example of import csv file
-
-```
-Time,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3,root.fit.p.s1
-1,100,'hello',200,300,400
-2,500,'world',600,700,800
-3,900,'IoTDB',1000,1100,1200
-```
-
-### Run import shell
-```
-# Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-
-# Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-```
-
-### Error data file
-
-`csvInsertError.error`
-
-## Usage of export-csv.sh
-
-### Run export shell
-```
-# Unix/OS X
-> tools/export-csv.sh -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-
-# Windows
-> tools\export-csv.bat -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-```
-
-### Input query
-
-```
-select * from root.fit.d1
-```
+# Usage of CSV Import and Export Tool
+see [Usage of CSV Import and Export Tool](docs/UserGuide/System Tools/CSV
Tool.md)
Review comment:
This is a broken link...
##########
File path: README_ZH.md
##########
@@ -333,61 +331,9 @@ server 可以使用 "ctrl-C" 或者执行下面的脚本:
编译完成后, IoTDB cli 将生成在 "cli/target/iotdb-cli-{project.version}".
-## 使用 import-csv.sh
-
-### 创建元数据
-
-```
-SET STORAGE GROUP TO root.fit.d1;
-SET STORAGE GROUP TO root.fit.d2;
-SET STORAGE GROUP TO root.fit.p;
-CREATE TIMESERIES root.fit.d1.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d1.s2 WITH DATATYPE=TEXT,ENCODING=PLAIN;
-CREATE TIMESERIES root.fit.d2.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d2.s3 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.p.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-```
-
-### 从 csv 文件导入数据的示例
-
-```
-Time,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3,root.fit.p.s1
-1,100,'hello',200,300,400
-2,500,'world',600,700,800
-3,900,'IoTDB',1000,1100,1200
-```
-
-### 运行 import shell
-```
-# Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-
-# Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-```
-
-### 错误的数据文件
-
-`csvInsertError.error`
-
-## 使用 export-csv.sh
-
-### 运行 export shell
-
-```
-# Unix/OS X
-> tools/export-csv.sh -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-
-# Windows
-> tools\export-csv.bat -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-```
-
-### 执行查询
-
-```
-select * from root.fit.d1
-```
+# 导入导出CSV工具
+查看 [导入导出CSV工具](docs/zh/UserGuide/System Tools/CSV Tool.md)
Review comment:
This is a broken link.
##########
File path: README.md
##########
@@ -333,59 +326,9 @@ Under the root path of iotdb:
After being built, the IoTDB cli is located at the folder
"cli/target/iotdb-cli-{project.version}".
-## Usage of import-csv.sh
-
-### Create metadata
-```
-SET STORAGE GROUP TO root.fit.d1;
-SET STORAGE GROUP TO root.fit.d2;
-SET STORAGE GROUP TO root.fit.p;
-CREATE TIMESERIES root.fit.d1.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d1.s2 WITH DATATYPE=TEXT,ENCODING=PLAIN;
-CREATE TIMESERIES root.fit.d2.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.d2.s3 WITH DATATYPE=INT32,ENCODING=RLE;
-CREATE TIMESERIES root.fit.p.s1 WITH DATATYPE=INT32,ENCODING=RLE;
-```
-
-### An example of import csv file
-
-```
-Time,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3,root.fit.p.s1
-1,100,'hello',200,300,400
-2,500,'world',600,700,800
-3,900,'IoTDB',1000,1100,1200
-```
-
-### Run import shell
-```
-# Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-
-# Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv>
-```
-
-### Error data file
-
-`csvInsertError.error`
-
-## Usage of export-csv.sh
-
-### Run export shell
-```
-# Unix/OS X
-> tools/export-csv.sh -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-
-# Windows
-> tools\export-csv.bat -h <ip> -p <port> -u <username> -pw <password> -td
<directory> [-tf <time-format>]
-```
-
-### Input query
-
-```
-select * from root.fit.d1
-```
+# Usage of CSV Import and Export Tool
+see [Usage of CSV Import and Export Tool](docs/UserGuide/System Tools/CSV
Tool.md)
Review comment:
This is a broken link.
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
##########
@@ -263,112 +228,123 @@ private static void dumpResult(String sql, int index)
return;
}
} catch (IOException e) {
- System.out.println("Cannot create dump file " + path + "because: " +
e.getMessage());
+ System.out.println("Cannot create dump file " + path + " " + "because: "
+ e.getMessage());
return;
}
System.out.println("Start to export data from sql statement: " + sql);
- try (Statement statement = connection.createStatement();
- ResultSet rs = statement.executeQuery(sql);
- BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
- ResultSetMetaData metadata = rs.getMetaData();
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql);
long startTime = System.currentTimeMillis();
-
- int count = metadata.getColumnCount();
// write data in csv file
- writeMetadata(bw, count, metadata);
+ writeMetadata(bw, sessionDataSet.getColumnNames());
- int line = writeResultSet(rs, bw, count);
+ int line = writeResultSet(sessionDataSet, bw);
System.out
- .println(String.format("Statement [%s] has dumped to file %s
successfully! It costs "
- + "%dms to export %d lines.", sql, path,
System.currentTimeMillis() - startTime,
- line));
- } catch (IOException e) {
+ .printf("Statement [%s] has dumped to file %s successfully! It costs
"
+ + "%dms to export %d lines.%n", sql, path,
System.currentTimeMillis() - startTime,
+ line);
+ } catch (IOException | StatementExecutionException |
IoTDBConnectionException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
}
- private static void writeMetadata(BufferedWriter bw, int count,
ResultSetMetaData metadata)
- throws SQLException, IOException {
- for (int i = 1; i <= count; i++) {
- if (i < count) {
- bw.write(metadata.getColumnLabel(i) + ",");
- } else {
- bw.write(metadata.getColumnLabel(i) + "\n");
- }
- typeList.add(metadata.getColumnType(i));
+ private static void writeMetadata(BufferedWriter bw, List<String>
columnNames)
+ throws IOException {
+ if (!columnNames.get(0).equals("Time")) {
+ bw.write("Time" + ",");
+ }
+ for (int i = 0; i < columnNames.size() - 1; i++) {
+ bw.write(columnNames.get(i) + ",");
}
+ bw.write(columnNames.get(columnNames.size() - 1) + "\n");
}
- private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
- throws SQLException, IOException {
+ private static int writeResultSet(SessionDataSet rs, BufferedWriter bw)
+ throws IOException, StatementExecutionException,
IoTDBConnectionException {
int line = 0;
long timestamp = System.currentTimeMillis();
- while (rs.next()) {
- if (rs.getString(1) == null ||
- "null".equalsIgnoreCase(rs.getString(1))) {
- bw.write(",");
- } else {
- writeTime(rs, bw);
- writeValue(rs, count, bw);
- }
+ while (rs.hasNext()) {
+ RowRecord rowRecord = rs.next();
+ List<Field> fields = rowRecord.getFields();
+ writeTime(rowRecord.getTimestamp(), bw);
+ writeValue(fields, bw);
line++;
if (line % EXPORT_PER_LINE_COUNT == 0) {
long tmp = System.currentTimeMillis();
- System.out.println(
- String.format("%d lines have been exported, it takes %dms", line,
(tmp - timestamp)));
+ System.out.printf("%d lines have been exported, it takes %dms%n",
line, (tmp - timestamp));
timestamp = tmp;
}
}
return line;
}
- private static void writeTime(ResultSet rs, BufferedWriter bw) throws
SQLException, IOException {
+ private static void writeTime(Long time, BufferedWriter bw) throws
IOException {
ZonedDateTime dateTime;
+ String timestampPrecision = "ms";
switch (timeFormat) {
case "default":
- long timestamp = rs.getLong(1);
String str = AbstractCli
-
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME,
timestamp, zoneId,
- TIMESTAMP_PRECISION);
+
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME, time,
zoneId,
+ timestampPrecision);
bw.write(str + ",");
break;
case "timestamp":
case "long":
case "nubmer":
- bw.write(rs.getLong(1) + ",");
+ bw.write(time + ",");
break;
default:
- dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
+ dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(time),
zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) +
",");
break;
}
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private static void writeValue(ResultSet rs, int count, BufferedWriter bw)
- throws SQLException, IOException {
- for (int j = 2; j <= count; j++) {
- if (j < count) {
- if ("null".equals(rs.getString(j))) {
- bw.write(",");
- } else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ ",");
+ private static void writeValue(List<Field> fields, BufferedWriter bw) throws
IOException {
+ for (int j = 0; j < fields.size() - 1; j++) {
+ String value = fields.get(j).getStringValue();
+ if ("null".equalsIgnoreCase(value)) {
+ bw.write(",");
+ } else {
+ if (fields.get(j).getDataType() == TSDataType.TEXT) {
+ int location = value.indexOf("\"");
+ if (location > -1) {
+ if (value.charAt(location - 1) != '\\') {
Review comment:
Note that `location` may equals to 0, which will cause Exception...
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ExportCsv.java
##########
@@ -263,112 +228,123 @@ private static void dumpResult(String sql, int index)
return;
}
} catch (IOException e) {
- System.out.println("Cannot create dump file " + path + "because: " +
e.getMessage());
+ System.out.println("Cannot create dump file " + path + " " + "because: "
+ e.getMessage());
return;
}
System.out.println("Start to export data from sql statement: " + sql);
- try (Statement statement = connection.createStatement();
- ResultSet rs = statement.executeQuery(sql);
- BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
- ResultSetMetaData metadata = rs.getMetaData();
+ try (BufferedWriter bw = new BufferedWriter(new FileWriter(tf))) {
+ SessionDataSet sessionDataSet = session.executeQueryStatement(sql);
long startTime = System.currentTimeMillis();
-
- int count = metadata.getColumnCount();
// write data in csv file
- writeMetadata(bw, count, metadata);
+ writeMetadata(bw, sessionDataSet.getColumnNames());
- int line = writeResultSet(rs, bw, count);
+ int line = writeResultSet(sessionDataSet, bw);
System.out
- .println(String.format("Statement [%s] has dumped to file %s
successfully! It costs "
- + "%dms to export %d lines.", sql, path,
System.currentTimeMillis() - startTime,
- line));
- } catch (IOException e) {
+ .printf("Statement [%s] has dumped to file %s successfully! It costs
"
+ + "%dms to export %d lines.%n", sql, path,
System.currentTimeMillis() - startTime,
+ line);
+ } catch (IOException | StatementExecutionException |
IoTDBConnectionException e) {
System.out.println("Cannot dump result because: " + e.getMessage());
}
}
- private static void writeMetadata(BufferedWriter bw, int count,
ResultSetMetaData metadata)
- throws SQLException, IOException {
- for (int i = 1; i <= count; i++) {
- if (i < count) {
- bw.write(metadata.getColumnLabel(i) + ",");
- } else {
- bw.write(metadata.getColumnLabel(i) + "\n");
- }
- typeList.add(metadata.getColumnType(i));
+ private static void writeMetadata(BufferedWriter bw, List<String>
columnNames)
+ throws IOException {
+ if (!columnNames.get(0).equals("Time")) {
+ bw.write("Time" + ",");
+ }
+ for (int i = 0; i < columnNames.size() - 1; i++) {
+ bw.write(columnNames.get(i) + ",");
}
+ bw.write(columnNames.get(columnNames.size() - 1) + "\n");
}
- private static int writeResultSet(ResultSet rs, BufferedWriter bw, int count)
- throws SQLException, IOException {
+ private static int writeResultSet(SessionDataSet rs, BufferedWriter bw)
+ throws IOException, StatementExecutionException,
IoTDBConnectionException {
int line = 0;
long timestamp = System.currentTimeMillis();
- while (rs.next()) {
- if (rs.getString(1) == null ||
- "null".equalsIgnoreCase(rs.getString(1))) {
- bw.write(",");
- } else {
- writeTime(rs, bw);
- writeValue(rs, count, bw);
- }
+ while (rs.hasNext()) {
+ RowRecord rowRecord = rs.next();
+ List<Field> fields = rowRecord.getFields();
+ writeTime(rowRecord.getTimestamp(), bw);
+ writeValue(fields, bw);
line++;
if (line % EXPORT_PER_LINE_COUNT == 0) {
long tmp = System.currentTimeMillis();
- System.out.println(
- String.format("%d lines have been exported, it takes %dms", line,
(tmp - timestamp)));
+ System.out.printf("%d lines have been exported, it takes %dms%n",
line, (tmp - timestamp));
timestamp = tmp;
}
}
return line;
}
- private static void writeTime(ResultSet rs, BufferedWriter bw) throws
SQLException, IOException {
+ private static void writeTime(Long time, BufferedWriter bw) throws
IOException {
ZonedDateTime dateTime;
+ String timestampPrecision = "ms";
switch (timeFormat) {
case "default":
- long timestamp = rs.getLong(1);
String str = AbstractCli
-
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME,
timestamp, zoneId,
- TIMESTAMP_PRECISION);
+
.parseLongToDateWithPrecision(DateTimeFormatter.ISO_OFFSET_DATE_TIME, time,
zoneId,
+ timestampPrecision);
bw.write(str + ",");
break;
case "timestamp":
case "long":
case "nubmer":
- bw.write(rs.getLong(1) + ",");
+ bw.write(time + ",");
break;
default:
- dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)),
+ dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(time),
zoneId);
bw.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) +
",");
break;
}
}
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- private static void writeValue(ResultSet rs, int count, BufferedWriter bw)
- throws SQLException, IOException {
- for (int j = 2; j <= count; j++) {
- if (j < count) {
- if ("null".equals(rs.getString(j))) {
- bw.write(",");
- } else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ ",");
+ private static void writeValue(List<Field> fields, BufferedWriter bw) throws
IOException {
+ for (int j = 0; j < fields.size() - 1; j++) {
+ String value = fields.get(j).getStringValue();
+ if ("null".equalsIgnoreCase(value)) {
+ bw.write(",");
+ } else {
+ if (fields.get(j).getDataType() == TSDataType.TEXT) {
+ int location = value.indexOf("\"");
+ if (location > -1) {
+ if (value.charAt(location - 1) != '\\') {
+ bw.write("\"" + value.replace("\"", "\\\"") + "\",");
+ } else {
+ bw.write("\"" + value + "\",");
+ }
+ } else if (value.contains(",")) {
+ bw.write("\"" + value + "\",");
} else {
- bw.write(rs.getString(j) + ",");
+ bw.write(value + ",");
}
- }
- } else {
- if ("null".equals(rs.getString(j))) {
- bw.write("\n");
} else {
- if(typeList.get(j-1) == Types.VARCHAR) {
- bw.write("\'" + rs.getString(j) + "\'"+ "\n");
+ bw.write(value + ",");
+ }
+ }
+ }
+ String lastValue = fields.get(fields.size() - 1).getStringValue();
+ if ("null".equalsIgnoreCase(lastValue)) {
+ bw.write("\n");
+ } else {
+ if (fields.get(fields.size() - 1).getDataType() == TSDataType.TEXT) {
+ int location = lastValue.indexOf("\"");
+ if (location > -1) {
+ if (lastValue.charAt(location - 1) != '\\') {
Review comment:
Note that `location` may equals to 0, which will cause Exception...
##########
File path: cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
##########
@@ -119,296 +89,90 @@ private static Options createOptions() {
/**
* Data from csv To tsfile.
*/
- private static void loadDataFromCSV(File file, int index) {
- statement = null;
+ private static void loadDataFromCSV(File file) {
int fileLine;
try {
fileLine = getFileLineCount(file);
} catch (IOException e) {
System.out.println("Failed to import file: " + file.getName());
return;
}
- File errorFile = new File(errorInsertInfo + index);
- if (!errorFile.exists()) {
- try {
- errorFile.createNewFile();
- } catch (IOException e) {
- System.out.println("Cannot create a errorFile because: " +
e.getMessage());
- return;
- }
- }
System.out.println("Start to import data from: " + file.getName());
- errorFlag = true;
- try(BufferedReader br = new BufferedReader(new FileReader(file));
- BufferedWriter bw = new BufferedWriter(new FileWriter(errorFile));
+ try (BufferedReader br = new BufferedReader(new FileReader(file));
ProgressBar pb = new ProgressBar("Import from: " + file.getName(),
fileLine)) {
pb.setExtraMessage("Importing...");
String header = br.readLine();
-
- bw.write("From " + file.getAbsolutePath());
- bw.newLine();
- bw.newLine();
- bw.write(header);
- bw.newLine();
- bw.newLine();
-
- // storage csv table head info
- Map<String, ArrayList<Integer>> deviceToColumn = new HashMap<>();
- // storage csv table head info
- List<String> colInfo = new ArrayList<>();
- // storage csv device sensor info, corresponding csv table head
- List<String> headInfo = new ArrayList<>();
-
- String[] strHeadInfo = header.split(",");
- if (strHeadInfo.length <= 1) {
- System.out.println("The CSV file "+ file.getName() +" illegal, please
check first line");
+ String[] cols = splitCsvLine(header);
+ if (cols.length <= 1) {
+ System.out.println("The CSV file " + file.getName() + " illegal,
please check first line");
return;
}
- long startTime = System.currentTimeMillis();
- Map<String, String> timeseriesDataType = new HashMap<>();
+ List<String> devices = new ArrayList<>();
+ List<Long> times = new ArrayList<>();
+ List<List<String>> measurementsList = new ArrayList<>();
+ List<List<String>> valuesList = new ArrayList<>();
+ Map<String, List<Integer>> devicesToPositions = new HashMap<>();
+ Map<String, List<String>> devicesToMeasurements = new HashMap<>();
- boolean success = queryDatabaseMeta(strHeadInfo, file, bw,
timeseriesDataType, headInfo,
- deviceToColumn, colInfo);
- if (!success) {
- errorFlag = false;
- return;
+ for (int i = 1; i < cols.length; i++) {
+ splitColToDeviceAndMeasurement(cols[i], devicesToPositions,
devicesToMeasurements, i);
}
- statement = connection.createStatement();
-
+ int lineNumber = 0;
+ String line;
+ while ((line = br.readLine()) != null) {
+ cols = splitCsvLine(line);
+ lineNumber++;
+ for (Entry<String, List<Integer>> deviceToPositions :
devicesToPositions
+ .entrySet()) {
+ String device = deviceToPositions.getKey();
+ devices.add(device);
+
+ times.add(Long.parseLong(cols[0]));
+
+ List<String> values = new ArrayList<>();
+ for (int position : deviceToPositions.getValue()) {
+ if (cols[position].equals("") && cols[position].equals("null")) {
Review comment:
This equation is always false and ...
Does `""` represent `null`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
