This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 81f5ca61c2 [IOTDB-4156] import csv has error time after
timestamp_precision set ns (#7032)
81f5ca61c2 is described below
commit 81f5ca61c29006bad558209e38597ff43c4aa62a
Author: cmlmakahts <[email protected]>
AuthorDate: Thu Aug 18 18:24:41 2022 +0800
[IOTDB-4156] import csv has error time after timestamp_precision set ns
(#7032)
---
cli/pom.xml | 37 +++++++++-
cli/src/assembly/cli.xml | 3 +
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 79 +++++++++-------------
docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 9 ++-
.../zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md | 9 ++-
.../apache/iotdb/db/qp/utils/DatetimeUtils.java | 24 ++++---
.../db/qp/utils/DatetimeQueryDataSetUtilsTest.java | 6 +-
7 files changed, 103 insertions(+), 64 deletions(-)
diff --git a/cli/pom.xml b/cli/pom.xml
index c655eb0e4b..38544e2779 100644
--- a/cli/pom.xml
+++ b/cli/pom.xml
@@ -83,7 +83,12 @@
<groupId>org.apache.iotdb</groupId>
<artifactId>iotdb-server</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -124,6 +129,36 @@
<skipITs>${cli.it.skip}</skipITs>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.3.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
<createDependencyReducedPom>false</createDependencyReducedPom>
+ <artifactSet>
+ <includes>
+
<include>org.apache.iotdb:iotdb-cli</include>
+
<include>org.apache.iotdb:iotdb-server</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+
<artifact>org.apache.iotdb:iotdb-server</artifact>
+ <includes>
+
<include>org/apache/iotdb/db/qp/utils/*</include>
+ </includes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
diff --git a/cli/src/assembly/cli.xml b/cli/src/assembly/cli.xml
index ca537d760c..c4e823f967 100644
--- a/cli/src/assembly/cli.xml
+++ b/cli/src/assembly/cli.xml
@@ -28,6 +28,9 @@
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
+ <excludes>
+ <exclude>org.apache.iotdb:iotdb-server</exclude>
+ </excludes>
<outputDirectory>lib</outputDirectory>
</dependencySet>
</dependencySets>
diff --git a/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
b/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
index dd3ff6a0f1..89dfa4f176 100644
--- a/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
+++ b/cli/src/main/java/org/apache/iotdb/tool/ImportCsv.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tool;
+import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
@@ -42,7 +43,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -82,6 +82,9 @@ public class ImportCsv extends AbstractCsvTool {
private static final String CSV_SUFFIXS = "csv";
private static final String TXT_SUFFIXS = "txt";
+ private static final String TIMESTAMP_PRECISION_ARGS = "tp";
+ private static final String TIMESTAMP_PRECISION_NAME = "timestamp precision
(ms/us/ns)";
+
private static final String TSFILEDB_CLI_PREFIX = "ImportCsv";
private static String targetPath;
@@ -93,6 +96,8 @@ public class ImportCsv extends AbstractCsvTool {
private static int batchPointSize = 100_000;
+ private static String timestampPrecision = "ms";
+
/**
* create the commandline options.
*
@@ -153,6 +158,14 @@ public class ImportCsv extends AbstractCsvTool {
.build();
options.addOption(opBatchPointSize);
+ Option opTimestampPrecision =
+ Option.builder(TIMESTAMP_PRECISION_ARGS)
+ .argName(TIMESTAMP_PRECISION_ARGS)
+ .hasArg()
+ .desc("Timestamp precision (ms/us/ns)")
+ .build();
+ options.addOption(opTimestampPrecision);
+
return options;
}
@@ -178,6 +191,10 @@ public class ImportCsv extends AbstractCsvTool {
if (commandLine.getOptionValue(ALIGNED_ARGS) != null) {
aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS));
}
+
+ if (commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS) != null) {
+ timestampPrecision =
commandLine.getOptionValue(TIMESTAMP_PRECISION_ARGS);
+ }
}
public static void main(String[] args) throws IoTDBConnectionException {
@@ -345,7 +362,6 @@ public class ImportCsv extends AbstractCsvTool {
List<List<TSDataType>> typesList = new ArrayList<>();
List<List<Object>> valuesList = new ArrayList<>();
- AtomicReference<SimpleDateFormat> timeFormatter = new
AtomicReference<>(null);
AtomicReference<Boolean> hasStarted = new AtomicReference<>(false);
AtomicInteger pointSize = new AtomicInteger(0);
@@ -355,7 +371,6 @@ public class ImportCsv extends AbstractCsvTool {
record -> {
if (!hasStarted.get()) {
hasStarted.set(true);
- timeFormatter.set(formatterInit(record.get(0)));
} else if (pointSize.get() >= batchPointSize) {
writeAndEmptyDataSet(deviceIds, times, typesList, valuesList,
measurementsList, 3);
pointSize.set(0);
@@ -403,17 +418,7 @@ public class ImportCsv extends AbstractCsvTool {
}
}
if (!measurements.isEmpty()) {
- if (timeFormatter.get() == null) {
- times.add(Long.valueOf(record.get(timeColumn)));
- } else {
- try {
-
times.add(timeFormatter.get().parse(record.get(timeColumn)).getTime());
- } catch (ParseException e) {
- System.out.println(
- "Meet error when insert csv because the format of time
is not supported");
- System.exit(0);
- }
- }
+ times.add(parseTimestamp(record.get(timeColumn)));
deviceIds.add(deviceId);
typesList.add(types);
valuesList.add(values);
@@ -472,7 +477,7 @@ public class ImportCsv extends AbstractCsvTool {
// only run in first record
if (deviceName.get() == null) {
deviceName.set(record.get(1));
- timeFormatter.set(formatterInit(record.get(0)));
+ // timeFormatter.set(formatterInit(record.get(0)));
} else if (!Objects.equals(deviceName.get(), record.get(1))) {
// if device changed
writeAndEmptyDataSet(
@@ -547,13 +552,7 @@ public class ImportCsv extends AbstractCsvTool {
if (timeFormatter.get() == null) {
times.add(Long.valueOf(record.get(timeColumn)));
} else {
- try {
-
times.add(timeFormatter.get().parse(record.get(timeColumn)).getTime());
- } catch (ParseException e) {
- System.out.println(
- "Meet error when insert csv because the format of time is
not supported");
- System.exit(0);
- }
+ times.add(parseTimestamp(record.get(timeColumn)));
}
typesList.add(types);
valuesList.add(values);
@@ -739,32 +738,6 @@ public class ImportCsv extends AbstractCsvTool {
}
}
- /**
- * return a suit time formatter
- *
- * @param time
- * @return
- */
- private static SimpleDateFormat formatterInit(String time) {
- try {
- Long.parseLong(time);
- return null;
- } catch (Exception ignored) {
- // do nothing
- }
-
- for (String timeFormat : STRING_TIME_FORMAT) {
- SimpleDateFormat format = new SimpleDateFormat(timeFormat);
- try {
- format.parse(time);
- return format;
- } catch (java.text.ParseException ignored) {
- // do nothing
- }
- }
- return null;
- }
-
/**
* return the TSDataType
*
@@ -854,4 +827,14 @@ public class ImportCsv extends AbstractCsvTool {
return null;
}
}
+
+ private static long parseTimestamp(String str) {
+ long timestamp;
+ try {
+ timestamp = Long.parseLong(str);
+ } catch (NumberFormatException e) {
+ timestamp = DatetimeUtils.convertDatetimeStrToLong(str, zoneId,
timestampPrecision);
+ }
+ return timestamp;
+ }
}
diff --git a/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md
b/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md
index a9fc08b931..1f4f9c6cc1 100644
--- a/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md
+++ b/docs/UserGuide/Write-And-Delete-Data/CSV-Tool.md
@@ -174,9 +174,9 @@ Time,Device,str(TEXT),int(INT32)
```shell
# Unix/OS X
-> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+> tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
# Windows
-> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+> tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
```
Description:
@@ -197,6 +197,9 @@ Description:
- specifying the point's number of a batch. If the program throw the
exception `org.apache.thrift.transport.TTransportException: Frame size larger
than protect max size`, you can lower this parameter as appropriate.
- example: `-batch 100000`, `100000` is the default value.
+* `-tp <time-precision>`:
+ - specifying a time precision. Options includes `ms`(millisecond),
`ns`(nanosecond), and `us`(microsecond), `ms` is default.
+
### Example
```sh
@@ -208,6 +211,8 @@ Description:
> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
> example-filename.csv
# or
> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
> example-filename.csv -fd .\failed
+# or
+> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
example-filename.csv -fd .\failed -tp ns
```
### Note
diff --git a/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md
b/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md
index 6abd676aa6..10028269f2 100644
--- a/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md
+++ b/docs/zh/UserGuide/Write-And-Delete-Data/CSV-Tool.md
@@ -175,9 +175,9 @@ Time,Device,str(TEXT),int(INT32)
```shell
# Unix/OS X
->tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+>tools/import-csv.sh -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
# Windows
->tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>]
+>tools\import-csv.bat -h <ip> -p <port> -u <username> -pw <password> -f
<xxx.csv> [-fd <./failedDirectory>] [-aligned <true>] [-tp <ms/ns/us>]
```
参数:
@@ -198,6 +198,9 @@ Time,Device,str(TEXT),int(INT32)
- 用于指定每一批插入的数据的点数。如果程序报了`org.apache.thrift.transport.TTransportException:
Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。
- 例如: `-batch 100000`，`100000`是默认值。
+* `-tp`:
+ - 用于指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒），默认值为`ms`。
+
### 运行示例
```sh
@@ -209,6 +212,8 @@ Time,Device,str(TEXT),int(INT32)
>tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
>example-filename.csv
# or
>tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
>example-filename.csv -fd .\failed
+# or
+> tools\import-csv.bat -h 127.0.0.1 -p 6667 -u root -pw root -f
example-filename.csv -fd .\failed -tp ns
```
### 注意
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
b/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
index e77d67344b..e7ed8e6478 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
@@ -450,7 +450,16 @@ public class DatetimeUtils {
.toFormatter();
public static long convertDatetimeStrToLong(String str, ZoneId zoneId) {
- return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0);
+ return convertDatetimeStrToLong(
+ str,
+ toZoneOffset(zoneId),
+ 0,
+ IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision());
+ }
+
+ public static long convertDatetimeStrToLong(
+ String str, ZoneId zoneId, String timestampPrecision) {
+ return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0,
timestampPrecision);
}
public static long getInstantWithPrecision(String str, String
timestampPrecision) {
@@ -478,10 +487,8 @@ public class DatetimeUtils {
}
/** convert date time string to millisecond, microsecond or nanosecond. */
- public static long convertDatetimeStrToLong(String str, ZoneOffset offset,
int depth) {
-
- String timestampPrecision =
IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
-
+ public static long convertDatetimeStrToLong(
+ String str, ZoneOffset offset, int depth, String timestampPrecision) {
if (depth >= 2) {
throw new DateTimeException(
String.format(
@@ -490,12 +497,13 @@ public class DatetimeUtils {
str, offset));
}
if (str.contains("Z")) {
- return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) +
"+00:00", offset, depth);
+ return convertDatetimeStrToLong(
+ str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth,
timestampPrecision);
} else if (str.length() == 10) {
- return convertDatetimeStrToLong(str + "T00:00:00", offset, depth);
+ return convertDatetimeStrToLong(str + "T00:00:00", offset, depth,
timestampPrecision);
} else if (str.length() - str.lastIndexOf('+') != 6
&& str.length() - str.lastIndexOf('-') != 6) {
- return convertDatetimeStrToLong(str + offset, offset, depth + 1);
+ return convertDatetimeStrToLong(str + offset, offset, depth + 1,
timestampPrecision);
} else if (str.contains("[") || str.contains("]")) {
throw new DateTimeException(
String.format(
diff --git
a/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java
b/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java
index 469bad0421..fa764a172a 100644
---
a/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/qp/utils/DatetimeQueryDataSetUtilsTest.java
@@ -189,7 +189,7 @@ public class DatetimeQueryDataSetUtilsTest {
"2019.01.02T15:13:27" + zoneOffset,
};
for (String str : timeFormatWithoutMs) {
- Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0));
+ Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0, "ms"));
}
for (String str : timeFormatWithoutMs) {
@@ -215,7 +215,7 @@ public class DatetimeQueryDataSetUtilsTest {
"2019.01.02T15:13:27.689" + zoneOffset,
};
for (String str : timeFormatWithoutMs) {
- assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0));
+ assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0, "ms"));
}
for (String str : timeFormatWithoutMs) {
@@ -230,7 +230,7 @@ public class DatetimeQueryDataSetUtilsTest {
"2019-01-02", "2019/01/02", "2019.01.02",
};
for (String str : timeFormatWithoutMs) {
- assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0));
+ assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str,
zoneOffset, 0, "ms"));
}
for (String str : timeFormatWithoutMs) {
