This is an automated email from the ASF dual-hosted git repository.
miao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 544a3bccbb9 update Export/ImportDataTestIT (#12399)
544a3bccbb9 is described below
commit 544a3bccbb9bbcb1e0997b5ce34c58db89e086de
Author: Summer <[email protected]>
AuthorDate: Thu Apr 25 16:40:54 2024 +0800
update Export/ImportDataTestIT (#12399)
* it
* update start/stop-all.sh reminder message
* - export only timestamp
- importData change jdbc to session
- aligned true->aligned values
- fix -h tips
* rollback linesPerFile
* ImportData failFile
* start/stop-all.sh ICID tips
* check tips
* aligned
---------
Co-authored-by: 2b3c511 <[email protected]>
---
.../apache/iotdb/tools/it/ExportDataTestIT.java | 6 +-
.../apache/iotdb/tools/it/ImportDataTestIT.java | 4 +-
.../org/apache/iotdb/tool/AbstractDataTool.java | 1 +
.../java/org/apache/iotdb/tool/ExportData.java | 21 ++++---
.../java/org/apache/iotdb/tool/ImportData.java | 73 +++++++++++-----------
.../src/assembly/resources/sbin/start-all.sh | 2 +-
.../src/assembly/resources/sbin/stop-all.sh | 2 +-
7 files changed, 59 insertions(+), 50 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java
index cb95adb363d..ec53326f427 100644
---
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportDataTestIT.java
@@ -143,6 +143,8 @@ public class ExportDataTestIT extends AbstractScript {
"root",
"-td",
"target",
+ "-type",
+ "sql",
"-q",
"select * from root.test.t2 where time > 1 and time <
1000000000000",
"&",
@@ -216,8 +218,10 @@ public class ExportDataTestIT extends AbstractScript {
"root",
"-td",
"target",
+ "-type",
+ "sql",
"-q",
- "select * from root.**");
+ "select * from root.test.t2 where time > 1 and time <
1000000000000");
builder2.environment().put("CLASSPATH", libPath);
testOutput(builder2, output2, 0);
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java
index 350aa9918e6..625d2b9481c 100644
---
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportDataTestIT.java
@@ -74,7 +74,7 @@ public class ImportDataTestIT extends AbstractScript {
@Override
protected void testOnWindows() throws IOException {
final String[] output = {
- "The file name must end with \"csv\" or \"txt\"!",
+ "The file name must end with \"csv\" or \"txt\" or \"sql\"!",
};
ProcessBuilder builder =
new ProcessBuilder(
@@ -101,7 +101,7 @@ public class ImportDataTestIT extends AbstractScript {
@Override
protected void testOnUnix() throws IOException {
final String[] output = {
- "The file name must end with \"csv\" or \"txt\"!",
+ "The file name must end with \"csv\" or \"txt\" or \"sql\"!",
};
ProcessBuilder builder =
new ProcessBuilder(
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java
index 57ccada4cb6..6ae1a1791dc 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractDataTool.java
@@ -117,6 +117,7 @@ public abstract class AbstractDataTool {
protected static String timeZoneID;
protected static String timeFormat;
protected static String exportType;
+ protected static String aligned;
protected static Session session;
private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java
index 69c342cec73..bdc518a6124 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportData.java
@@ -89,11 +89,11 @@ public class ExportData extends AbstractDataTool {
private static final String EXPORT_SQL_TYPE_NAME = "sql";
private static final String ALIGNED_ARGS = "aligned";
- private static final String ALIGNED_NAME = "create the aligned insert sql";
+ private static final String ALIGNED_NAME = "export aligned insert sql";
private static final String LINES_PER_FILE_ARGS = "linesPerFile";
private static final String LINES_PER_FILE_ARGS_NAME = "Lines Per File";
- private static final String TSFILEDB_CLI_PREFIX = "ExportCsv";
+ private static final String TSFILEDB_CLI_PREFIX = "ExportData";
private static final String DUMP_FILE_NAME_DEFAULT = "dump";
private static String targetFile = DUMP_FILE_NAME_DEFAULT;
@@ -110,6 +110,8 @@ public class ExportData extends AbstractDataTool {
private static long timeout = -1;
+ private static Boolean aligned = false;
+
private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
@SuppressWarnings({
@@ -231,6 +233,9 @@ public class ExportData extends AbstractDataTool {
if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) {
linesPerFile =
Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS));
}
+ if (commandLine.getOptionValue(ALIGNED_ARGS) != null) {
+ aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS));
+ }
}
/**
@@ -273,7 +278,7 @@ public class ExportData extends AbstractDataTool {
.desc(
"Output time Format in csv file. "
+ "You can choose 1) timestamp, number, long 2) ISO8601,
default 3) "
- + "user-defined pattern like yyyy-MM-dd\\ HH:mm:ss,
default ISO8601 (optional)")
+ + "user-defined pattern like yyyy-MM-dd\\ HH:mm:ss,
default ISO8601.\n OutPut timestamp in sql file, No matter what time format is
set(optional)")
.build();
options.addOption(opTimeFormat);
@@ -316,7 +321,7 @@ public class ExportData extends AbstractDataTool {
Option.builder(ALIGNED_ARGS)
.argName(ALIGNED_NAME)
.hasArg()
- .desc("Whether to use the interface of aligned (optional)")
+ .desc("Whether export to sql of aligned (only sql optional)")
.build();
options.addOption(opAligned);
@@ -525,10 +530,10 @@ public class ExportData extends AbstractDataTool {
timeseries.remove(0);
}
String sqlMiddle = null;
- if (Boolean.TRUE.equals(ALIGNED_ARGS)) {
- sqlMiddle = " ALIGNED VALUES (" +
timeTrans(rowRecord.getTimestamp()) + ",";
+ if (Boolean.TRUE.equals(aligned)) {
+ sqlMiddle = " ALIGNED VALUES (" + rowRecord.getTimestamp() + ",";
} else {
- sqlMiddle = " VALUES (" + timeTrans(rowRecord.getTimestamp()) +
",";
+ sqlMiddle = " VALUES (" + rowRecord.getTimestamp() + ",";
}
List<String> values = new ArrayList<>();
if (headers.contains("Device")) {
@@ -560,7 +565,7 @@ public class ExportData extends AbstractDataTool {
writer.write(
"INSERT INTO "
+ deviceName
- + " (TIMESTAMP,"
+ + "(TIMESTAMP,"
+ String.join(",", headersTemp)
+ ")"
+ sqlMiddle
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java
index 8d1b3536395..7695fd433ff 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportData.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.exception.ArgsErrorException;
import org.apache.iotdb.isession.SessionDataSet;
-import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
@@ -41,6 +40,7 @@ import org.apache.commons.cli.Options;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.thrift.annotation.Nullable;
import org.apache.tsfile.common.constant.TsFileConstant;
@@ -52,12 +52,9 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -73,7 +70,6 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.iotdb.jdbc.Config.IOTDB_ERROR_PREFIX;
import static org.apache.tsfile.enums.TSDataType.BOOLEAN;
import static org.apache.tsfile.enums.TSDataType.DOUBLE;
import static org.apache.tsfile.enums.TSDataType.FLOAT;
@@ -109,7 +105,7 @@ public class ImportData extends AbstractDataTool {
private static final String LINES_PER_FAILED_FILE_ARGS =
"linesPerFailedFile";
private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "Lines Per
FailedFile";
- private static final String TSFILEDB_CLI_PREFIX = "ImportCsv";
+ private static final String TSFILEDB_CLI_PREFIX = "ImportData";
private static String targetPath;
private static String failedFileDirectory = null;
@@ -192,7 +188,7 @@ public class ImportData extends AbstractDataTool {
Option.builder(ALIGNED_ARGS)
.argName(ALIGNED_NAME)
.hasArg()
- .desc("Whether to use the interface of aligned (optional)")
+ .desc("Whether to use the interface of aligned(only csv optional)")
.build();
options.addOption(opAligned);
@@ -462,42 +458,45 @@ public class ImportData extends AbstractDataTool {
}
private static void importFromSqlFile(File file) {
+ ArrayList<List<Object>> failedRecords = new ArrayList<>();
+ String failedFilePath = null;
+ if (failedFileDirectory == null) {
+ failedFilePath = file.getAbsolutePath() + ".failed";
+ } else {
+ failedFilePath = failedFileDirectory + file.getName() + ".failed";
+ }
try (BufferedReader br = new BufferedReader(new
FileReader(file.getAbsolutePath()))) {
- String line;
- while ((line = br.readLine()) != null) {
- executeSql(line);
+ String sql;
+ while ((sql = br.readLine()) != null) {
+ try {
+ session.executeNonQueryStatement(sql);
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ failedRecords.add(Arrays.asList(sql));
+ }
}
ioTPrinter.println(file.getName() + " Import completely!");
} catch (IOException e) {
ioTPrinter.println("SQL file read exception because: " + e.getMessage());
}
- }
-
- private static void executeSql(String sql) {
- try (Statement statement = getConnection().createStatement()) {
- statement.setFetchSize(fetchSize);
- statement.execute(sql.trim());
- } catch (SQLException e) {
- ioTPrinter.println(IOTDB_ERROR_PREFIX + " Can't execute sql because " +
e.getMessage());
- System.exit(CODE_ERROR);
- }
- }
-
- private static Connection getConnection() {
- // JDBC driver name and database URL
- String driver = org.apache.iotdb.jdbc.IoTDBDriver.class.getName();
- String url = Config.IOTDB_URL_PREFIX + host + ":" + port + "/";
-
- Connection connection = null;
- try {
- Class.forName(driver);
- connection = DriverManager.getConnection(url, username, password);
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- } catch (SQLException e) {
- e.printStackTrace();
+ if (!failedRecords.isEmpty()) {
+ FileWriter writer = null;
+ try {
+ writer = new FileWriter(failedFilePath);
+ for (List<Object> failedRecord : failedRecords) {
+ writer.write(failedRecord.get(0).toString() + "\n");
+ }
+ } catch (IOException e) {
+ ioTPrinter.println("Cannot dump fail result because: " +
e.getMessage());
+ } finally {
+ if (ObjectUtils.isNotEmpty(writer)) {
+ try {
+ writer.flush();
+ writer.close();
+ } catch (IOException e) {;
+ }
+ }
+ }
}
- return connection;
}
/**
diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/start-all.sh
b/iotdb-core/node-commons/src/assembly/resources/sbin/start-all.sh
index 255985170ad..d9204cdcd38 100644
--- a/iotdb-core/node-commons/src/assembly/resources/sbin/start-all.sh
+++ b/iotdb-core/node-commons/src/assembly/resources/sbin/start-all.sh
@@ -37,7 +37,7 @@ fi
function validateParam() {
if [[ -z $1 || -z $2 || -z $3 || -z $4 || -z $5 || -z $6 ]]; then
- echo "The iotdb-cluster.properties file is incomplete, the current 1C1D
will be started ... "
+ echo "The iotdb-cluster.properties file only contains default settings. It
will start 1C1D."
exec ${IOTDB_HOME}/sbin/start-standalone.sh
exit
fi
diff --git a/iotdb-core/node-commons/src/assembly/resources/sbin/stop-all.sh
b/iotdb-core/node-commons/src/assembly/resources/sbin/stop-all.sh
index c162b78b2e1..b28fba9bd75 100644
--- a/iotdb-core/node-commons/src/assembly/resources/sbin/stop-all.sh
+++ b/iotdb-core/node-commons/src/assembly/resources/sbin/stop-all.sh
@@ -37,7 +37,7 @@ fi
function validateParam() {
if [[ -z $1 || -z $2 || -z $3 || -z $4 || -z $5 || -z $6 ]]; then
- echo "The iotdb-cluster.properties file is incomplete, the current 1C1D
will be stopped ... "
+ echo "The iotdb-cluster.properties file only contains default settings. It
will stop 1C1D."
exec ${IOTDB_HOME}/sbin/stop-standalone.sh
fi
}
