This is an automated email from the ASF dual-hosted git repository.
miao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8a6e31ffe5a add import-schema.sh/bat and export-schema.sh/bat in
cli/tools (#12623)
8a6e31ffe5a is described below
commit 8a6e31ffe5ab85b42a5d1b6e1be8823fef0ac025
Author: Summer <[email protected]>
AuthorDate: Sat Jun 1 00:14:26 2024 +0800
add import-schema.sh/bat and export-schema.sh/bat in cli/tools (#12623)
* add import-schema.sh/bat and export-schema.sh/bat in cli/tools
* update it
* update it
---------
Co-authored-by: 2b3c511 <[email protected]>
---
.../apache/iotdb/tools/it/ExportSchemaTestIT.java | 149 +++++
.../apache/iotdb/tools/it/ImportSchemaTestIT.java | 126 ++++
.../src/assembly/resources/tools/export-schema.bat | 62 ++
.../src/assembly/resources/tools/export-schema.sh | 57 ++
.../src/assembly/resources/tools/import-schema.bat | 63 ++
.../src/assembly/resources/tools/import-schema.sh | 57 ++
.../org/apache/iotdb/tool/AbstractSchemaTool.java | 242 ++++++++
.../java/org/apache/iotdb/tool/ExportSchema.java | 339 +++++++++++
.../java/org/apache/iotdb/tool/ImportSchema.java | 653 +++++++++++++++++++++
9 files changed, 1748 insertions(+)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java
new file mode 100644
index 00000000000..f8a4cdae171
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ExportSchemaTestIT.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tools.it;
+
+import org.apache.iotdb.cli.it.AbstractScript;
+import org.apache.iotdb.isession.ISession;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class ExportSchemaTestIT extends AbstractScript {
+ private static String ip;
+
+ private static String port;
+
+ private static String toolsPath;
+
+ private static String libPath;
+
+ private static String homePath;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ ip = EnvFactory.getEnv().getIP();
+ port = EnvFactory.getEnv().getPort();
+ toolsPath = EnvFactory.getEnv().getToolsPath();
+ libPath = EnvFactory.getEnv().getLibPath();
+ homePath =
+ libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" +
File.separator + "*"));
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void test() throws IOException {
+ String os = System.getProperty("os.name").toLowerCase();
+ if (os.startsWith("windows")) {
+ testOnWindows();
+ } else {
+ testOnUnix();
+ }
+ }
+
+ @Override
+ protected void testOnWindows() throws IOException {
+ prepareSchema();
+ final String[] output = {"Export completely!"};
+ ProcessBuilder builder =
+ new ProcessBuilder(
+ "cmd.exe",
+ "/c",
+ toolsPath + File.separator + "export-schema.bat",
+ "-h",
+ ip,
+ "-p",
+ port,
+ "-u",
+ "root",
+ "-pw",
+ "root",
+ "-t",
+ "target",
+ "-path",
+ "root.**",
+ "&",
+ "exit",
+ "%^errorlevel%");
+ builder.environment().put("IOTDB_HOME", homePath);
+ testOutput(builder, output, 0);
+ }
+
+ @Override
+ protected void testOnUnix() throws IOException {
+ prepareSchema();
+ final String[] output = {"Export completely!"};
+ ProcessBuilder builder =
+ new ProcessBuilder(
+ "bash",
+ toolsPath + File.separator + "export-schema.sh",
+ "-h",
+ ip,
+ "-p",
+ port,
+ "-u",
+ "root",
+ "-pw",
+ "root",
+ "-t",
+ "target",
+ "-path",
+ "root.**");
+ builder.environment().put("IOTDB_HOME", homePath);
+ testOutput(builder, output, 0);
+ }
+
+ public void prepareSchema() {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.open();
+ session.createTimeseries(
+ "root.schema.t2.c1",
+ TSDataType.DOUBLE,
+ TSEncoding.GORILLA,
+ CompressionType.LZ4,
+ null,
+ null,
+ null,
+ null);
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java
new file mode 100644
index 00000000000..ea7e1ba07f1
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/tools/it/ImportSchemaTestIT.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tools.it;
+
+import org.apache.iotdb.cli.it.AbstractScript;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class ImportSchemaTestIT extends AbstractScript {
+
+ private static String ip;
+
+ private static String port;
+
+ private static String toolsPath;
+
+ private static String libPath;
+
+ private static String homePath;
+
+ @BeforeClass
+ public static void setUp() {
+ EnvFactory.getEnv().initClusterEnvironment();
+ ip = EnvFactory.getEnv().getIP();
+ port = EnvFactory.getEnv().getPort();
+ toolsPath = EnvFactory.getEnv().getToolsPath();
+ libPath = EnvFactory.getEnv().getLibPath();
+ homePath =
+ libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" +
File.separator + "*"));
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void test() throws IOException {
+ String os = System.getProperty("os.name").toLowerCase();
+ if (os.startsWith("windows")) {
+ testOnWindows();
+ } else {
+ testOnUnix();
+ }
+ }
+
+ @Override
+ protected void testOnWindows() throws IOException {
+ final String[] output = {
+ "The file name must end with \"csv\"!",
+ };
+ ProcessBuilder builder =
+ new ProcessBuilder(
+ "cmd.exe",
+ "/c",
+ toolsPath + File.separator + "import-schema.bat",
+ "-h",
+ ip,
+ "-p",
+ port,
+ "-u",
+ "root",
+ "-pw",
+ "root",
+ "-s",
+ "./",
+ "&",
+ "exit",
+ "%^errorlevel%");
+ builder.environment().put("IOTDB_HOME", homePath);
+ testOutput(builder, output, 0);
+ }
+
+ @Override
+ protected void testOnUnix() throws IOException {
+ final String[] output = {
+ "The file name must end with \"csv\"!",
+ };
+ ProcessBuilder builder =
+ new ProcessBuilder(
+ "bash",
+ toolsPath + File.separator + "import-schema.sh",
+ "-h",
+ ip,
+ "-p",
+ port,
+ "-u",
+ "root",
+ "-pw",
+ "root",
+ "-s",
+ "./");
+ builder.environment().put("IOTDB_HOME", homePath);
+ testOutput(builder, output, 0);
+ }
+}
diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat
b/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat
new file mode 100644
index 00000000000..8ed2b81f5c7
--- /dev/null
+++ b/iotdb-client/cli/src/assembly/resources/tools/export-schema.bat
@@ -0,0 +1,62 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+
+title IoTDB Export
+
+echo ````````````````````````````````````````````````
+echo Starting IoTDB Client Export Schema Script
+echo ````````````````````````````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ExportSchema
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM
-----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -DIOTDB_HOME="%IOTDB_HOME%"
+
+@REM ***** CLASSPATH library setting *****
+if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set
CLASSPATH="%IOTDB_HOME%\..\lib\*"
+
+REM
-----------------------------------------------------------------------------
+
+"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH%
%MAIN_CLASS% %*
+set ret_code=%ERRORLEVEL%
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+pause
+
+@REM
-----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL
+
+EXIT /B %ret_code%
diff --git a/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh
b/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh
new file mode 100644
index 00000000000..b65a7d3950a
--- /dev/null
+++ b/iotdb-client/cli/src/assembly/resources/tools/export-schema.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ------------------------------------------
+echo Starting IoTDB Client Export Schema Script
+echo ------------------------------------------
+
+if [ -z "${IOTDB_INCLUDE}" ]; then
+ #do nothing
+ :
+elif [ -r "$IOTDB_INCLUDE" ]; then
+ . "$IOTDB_INCLUDE"
+fi
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment
variables. > /dev/stderr
+ exit 1;
+fi
+
+CLASSPATH=${IOTDB_HOME}/lib/*
+
+MAIN_CLASS=org.apache.iotdb.tool.ExportSchema
+
+"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
\ No newline at end of file
diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat
b/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat
new file mode 100644
index 00000000000..46a8d5abf6e
--- /dev/null
+++ b/iotdb-client/cli/src/assembly/resources/tools/import-schema.bat
@@ -0,0 +1,63 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements. See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership. The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied. See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+@echo off
+
+title IoTDB Import
+
+echo ````````````````````````````````````````````````
+echo Starting IoTDB Client Import Schema Script
+echo ````````````````````````````````````````````````
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.tool.ImportSchema
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM
-----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -DIOTDB_HOME="%IOTDB_HOME%"
+
+@REM ***** CLASSPATH library setting *****
+if EXIST "%IOTDB_HOME%\lib" (set CLASSPATH="%IOTDB_HOME%\lib\*") else set
CLASSPATH="%IOTDB_HOME%\..\lib\*"
+
+REM
-----------------------------------------------------------------------------
+
+"%JAVA_HOME%\bin\java" -DIOTDB_HOME="%IOTDB_HOME%" %JAVA_OPTS% -cp %CLASSPATH%
%MAIN_CLASS% %*
+set ret_code=%ERRORLEVEL%
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+set ret_code=1
+pause
+
+
+@REM
-----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL
+
+EXIT /B %ret_code%
diff --git a/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh
b/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh
new file mode 100644
index 00000000000..ca91293e235
--- /dev/null
+++ b/iotdb-client/cli/src/assembly/resources/tools/import-schema.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+echo ------------------------------------------
+echo Starting IoTDB Client Import Schema Script
+echo ------------------------------------------
+
+if [ -z "${IOTDB_INCLUDE}" ]; then
+ #do nothing
+ :
+elif [ -r "$IOTDB_INCLUDE" ]; then
+ . "$IOTDB_INCLUDE"
+fi
+
+if [ -z "${IOTDB_HOME}" ]; then
+ export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+if [ -n "$JAVA_HOME" ]; then
+ for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+ if [ -x "$java" ]; then
+ JAVA="$java"
+ break
+ fi
+ done
+else
+ JAVA=java
+fi
+
+if [ -z $JAVA ] ; then
+ echo Unable to find java executable. Check JAVA_HOME and PATH environment
variables. > /dev/stderr
+ exit 1;
+fi
+
+CLASSPATH=${IOTDB_HOME}/lib/*
+
+MAIN_CLASS=org.apache.iotdb.tool.ImportSchema
+
+"$JAVA" -DIOTDB_HOME=${IOTDB_HOME} -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+exit $?
\ No newline at end of file
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java
new file mode 100644
index 00000000000..9e91d404d12
--- /dev/null
+++
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/AbstractSchemaTool.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool;
+
+import org.apache.iotdb.cli.utils.IoTPrinter;
+import org.apache.iotdb.exception.ArgsErrorException;
+import org.apache.iotdb.session.Session;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.QuoteMode;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+public abstract class AbstractSchemaTool {
+
+ protected static final String HOST_ARGS = "h";
+ protected static final String HOST_NAME = "host";
+ protected static final String HOST_DEFAULT_VALUE = "127.0.0.1";
+
+ protected static final String HELP_ARGS = "help";
+
+ protected static final String PORT_ARGS = "p";
+ protected static final String PORT_NAME = "port";
+ protected static final String PORT_DEFAULT_VALUE = "6667";
+
+ protected static final String PW_ARGS = "pw";
+ protected static final String PW_NAME = "password";
+ protected static final String PW_DEFAULT_VALUE = "root";
+
+ protected static final String USERNAME_ARGS = "u";
+ protected static final String USERNAME_NAME = "username";
+ protected static final String USERNAME_DEFAULT_VALUE = "root";
+
+ protected static final String TIMEOUT_ARGS = "timeout";
+ protected static final String TIMEOUT_ARGS_NAME = "queryTimeout";
+
+ protected static final int MAX_HELP_CONSOLE_WIDTH = 92;
+
+ protected static final int CODE_OK = 0;
+ protected static final int CODE_ERROR = 1;
+
+ protected static String host;
+ protected static String port;
+ protected static String username;
+ protected static String password;
+
+ protected static String aligned;
+ protected static Session session;
+
+ protected static final List<String> HEAD_COLUMNS =
+ Arrays.asList("Timeseries", "Alias", "DataType", "Encoding",
"Compression");
+ private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
+ private static final Logger LOGGER =
LoggerFactory.getLogger(AbstractSchemaTool.class);
+
+ protected AbstractSchemaTool() {}
+
+ protected static String checkRequiredArg(
+ String arg, String name, CommandLine commandLine, String defaultValue)
+ throws ArgsErrorException {
+ String str = commandLine.getOptionValue(arg);
+ if (str == null) {
+ if (StringUtils.isNotBlank(defaultValue)) {
+ return defaultValue;
+ }
+ String msg = String.format("Required values for option '%s' not
provided", name);
+ LOGGER.info(msg);
+ LOGGER.info("Use -help for more information");
+ throw new ArgsErrorException(msg);
+ }
+ return str;
+ }
+
+ protected static void parseBasicParams(CommandLine commandLine) throws
ArgsErrorException {
+ host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine,
HOST_DEFAULT_VALUE);
+ port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine,
PORT_DEFAULT_VALUE);
+ username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine,
USERNAME_DEFAULT_VALUE);
+ password = checkRequiredArg(PW_ARGS, PW_NAME, commandLine,
PW_DEFAULT_VALUE);
+ }
+
+ protected static Options createNewOptions() {
+ Options options = new Options();
+
+ Option opHost =
+ Option.builder(HOST_ARGS)
+ .longOpt(HOST_NAME)
+ .optionalArg(true)
+ .argName(HOST_NAME)
+ .hasArg()
+ .desc("Host Name (optional)")
+ .build();
+ options.addOption(opHost);
+
+ Option opPort =
+ Option.builder(PORT_ARGS)
+ .longOpt(PORT_NAME)
+ .optionalArg(true)
+ .argName(PORT_NAME)
+ .hasArg()
+ .desc("Port (optional)")
+ .build();
+ options.addOption(opPort);
+
+ Option opUsername =
+ Option.builder(USERNAME_ARGS)
+ .longOpt(USERNAME_NAME)
+ .optionalArg(true)
+ .argName(USERNAME_NAME)
+ .hasArg()
+ .desc("Username (optional)")
+ .build();
+ options.addOption(opUsername);
+
+ Option opPassword =
+ Option.builder(PW_ARGS)
+ .longOpt(PW_NAME)
+ .optionalArg(true)
+ .argName(PW_NAME)
+ .hasArg()
+ .desc("Password (optional)")
+ .build();
+ options.addOption(opPassword);
+ return options;
+ }
+
+ /**
+ * write data to CSV file.
+ *
+ * @param records the records of CSV file
+ * @param filePath the directory to save the file
+ */
+ public static Boolean writeCsvFile(List<List<Object>> records, String
filePath) {
+ try {
+ final CSVPrinterWrapper csvPrinterWrapper = new
CSVPrinterWrapper(filePath);
+ for (List<Object> CsvRecord : records) {
+ csvPrinterWrapper.printRecordln(CsvRecord);
+ }
+ csvPrinterWrapper.flush();
+ csvPrinterWrapper.close();
+ return true;
+ } catch (IOException e) {
+ ioTPrinter.printException(e);
+ return false;
+ }
+ }
+
+ static class CSVPrinterWrapper {
+ private final String filePath;
+ private final CSVFormat csvFormat;
+ private CSVPrinter csvPrinter;
+
+ public CSVPrinterWrapper(String filePath) {
+ this.filePath = filePath;
+ this.csvFormat =
+ CSVFormat.Builder.create(CSVFormat.DEFAULT)
+ .setHeader()
+ .setSkipHeaderRecord(true)
+ .setEscape('\\')
+ .setQuoteMode(QuoteMode.NONE)
+ .build();
+ }
+
+ public void printRecord(final Iterable<?> values) throws IOException {
+ if (csvPrinter == null) {
+ csvPrinter = csvFormat.print(new PrintWriter(filePath));
+ }
+ csvPrinter.printRecord(values);
+ }
+
+ public void printRecordln(final Iterable<?> values) throws IOException {
+ if (csvPrinter == null) {
+ csvPrinter = csvFormat.print(new PrintWriter(filePath));
+ }
+ Iterator var2 = values.iterator();
+
+ while (var2.hasNext()) {
+ Object value = var2.next();
+ csvPrinter.print(value);
+ }
+ csvPrinter.println();
+ }
+
+ public void print(Object value) {
+ if (csvPrinter == null) {
+ try {
+ csvPrinter = csvFormat.print(new PrintWriter(filePath));
+ } catch (IOException e) {
+ ioTPrinter.printException(e);
+ return;
+ }
+ }
+ try {
+ csvPrinter.print(value);
+ } catch (IOException e) {
+ ioTPrinter.printException(e);
+ }
+ }
+
+ public void println() throws IOException {
+ csvPrinter.println();
+ }
+
+ public void close() throws IOException {
+ if (csvPrinter != null) {
+ csvPrinter.close();
+ }
+ }
+
+ public void flush() throws IOException {
+ if (csvPrinter != null) {
+ csvPrinter.flush();
+ }
+ }
+ }
+}
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java
new file mode 100644
index 00000000000..6693336e1f9
--- /dev/null
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ExportSchema.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool;
+
+import org.apache.iotdb.cli.type.ExitType;
+import org.apache.iotdb.cli.utils.CliContext;
+import org.apache.iotdb.cli.utils.IoTPrinter;
+import org.apache.iotdb.cli.utils.JlineUtils;
+import org.apache.iotdb.exception.ArgsErrorException;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tsfile.read.common.Field;
+import org.apache.tsfile.read.common.RowRecord;
+import org.jline.reader.LineReader;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE;
+
+/** Export Schema CSV file. */
+public class ExportSchema extends AbstractSchemaTool {
+
+ private static final String TARGET_DIR_ARGS = "t";
+ private static final String TARGET_DIR_ARGS_NAME = "target";
+ private static final String TARGET_DIR_NAME = "targetDir";
+
+ private static final String TARGET_PATH_ARGS = "path";
+ private static final String TARGET_PATH_ARGS_NAME = "path_pattern";
+ private static final String TARGET_PATH_NAME = "exportPathPattern";
+ private static String queryPath;
+
+ private static final String TARGET_FILE_ARGS = "pf";
+ private static final String TARGET_FILE_ARGS_NAME = "path_pattern_file";
+ private static final String TARGET_FILE_NAME = "exportPathPatternFile";
+
+ private static final String LINES_PER_FILE_ARGS = "lpf";
+ private static final String LINES_PER_FILE_ARGS_NAME = "lines_per_file";
+ private static final String LINES_PER_FILE_NAME = "linesPerFile";
+ private static int linesPerFile = 10000;
+
+ private static final String EXPORT_SCHEMA_CLI_PREFIX = "ExportSchema";
+
+ private static final String DUMP_FILE_NAME_DEFAULT = "dump";
+ private static String targetFile = DUMP_FILE_NAME_DEFAULT;
+
+ private static String targetDirectory;
+
+ private static long timeout = 60000;
+
+ private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
+
+ private static final String BASE_VIEW_TYPE = "BASE";
+ private static final String HEADER_VIEW_TYPE = "ViewType";
+ private static final String HEADER_TIMESERIES = "Timeseries";
+
+ @SuppressWarnings({
+ "squid:S3776",
+ "squid:S2093"
+ }) // Suppress high Cognitive Complexity warning, ignore try-with-resources
+ /* main function of export csv tool. */
+ public static void main(String[] args) {
+ Options options = createOptions();
+ HelpFormatter hf = new HelpFormatter();
+ CommandLine commandLine = null;
+ CommandLineParser parser = new DefaultParser();
+ hf.setOptionComparator(null); // avoid reordering
+ hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
+
+ if (args == null || args.length == 0) {
+ ioTPrinter.println("Too few params input, please check the following
hint.");
+ hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (ParseException e) {
+ ioTPrinter.println(e.getMessage());
+ hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ if (commandLine.hasOption(HELP_ARGS)) {
+ hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ int exitCode = CODE_OK;
+ try {
+ parseBasicParams(commandLine);
+ parseSpecialParams(commandLine);
+ session = new Session(host, Integer.parseInt(port), username, password);
+ session.open(false);
+ if (queryPath == null) {
+ String pathFile = commandLine.getOptionValue(TARGET_FILE_ARGS);
+ String path;
+ if (pathFile == null) {
+ LineReader lineReader =
+ JlineUtils.getLineReader(
+ new CliContext(System.in, System.out, System.err,
ExitType.EXCEPTION),
+ username,
+ host,
+ port);
+ path = lineReader.readLine(EXPORT_SCHEMA_CLI_PREFIX + "> please
input path pattern: ");
+ ioTPrinter.println(path);
+ String[] values = path.trim().split(";");
+ for (int i = 0; i < values.length; i++) {
+ if (StringUtils.isBlank(values[i])) {
+ continue;
+ } else {
+ dumpResult(values[i], i);
+ }
+ }
+ } else if (!pathFile.endsWith(".txt")) {
+ ioTPrinter.println("The file name must end with \"txt\"!");
+ hf.printHelp(EXPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ } else {
+ dumpFromPathFile(pathFile);
+ }
+ } else {
+ dumpResult(queryPath, 0);
+ }
+ } catch (IOException e) {
+ ioTPrinter.println("Failed to operate on file, because " +
e.getMessage());
+ exitCode = CODE_ERROR;
+ } catch (ArgsErrorException e) {
+ ioTPrinter.println("Invalid args: " + e.getMessage());
+ exitCode = CODE_ERROR;
+ } catch (IoTDBConnectionException e) {
+ ioTPrinter.println("Connect failed because " + e.getMessage());
+ exitCode = CODE_ERROR;
+ } finally {
+ if (session != null) {
+ try {
+ session.close();
+ } catch (IoTDBConnectionException e) {
+ exitCode = CODE_ERROR;
+ ioTPrinter.println(
+ "Encounter an error when closing session, error is: " +
e.getMessage());
+ }
+ }
+ }
+ System.exit(exitCode);
+ }
+
+ private static void parseSpecialParams(CommandLine commandLine) throws
ArgsErrorException {
+ targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_ARGS_NAME,
commandLine, null);
+ queryPath = commandLine.getOptionValue(TARGET_PATH_ARGS);
+ String timeoutString = commandLine.getOptionValue(TIMEOUT_ARGS);
+ if (timeoutString != null) {
+ timeout = Long.parseLong(timeoutString);
+ }
+ if (targetFile == null) {
+ targetFile = DUMP_FILE_NAME_DEFAULT;
+ }
+ if (!targetDirectory.endsWith("/") && !targetDirectory.endsWith("\\")) {
+ targetDirectory += File.separator;
+ }
+ if (commandLine.getOptionValue(LINES_PER_FILE_ARGS) != null) {
+ linesPerFile =
Integer.parseInt(commandLine.getOptionValue(LINES_PER_FILE_ARGS));
+ }
+ }
+
+ /**
+ * commandline option create.
+ *
+ * @return object Options
+ */
+ private static Options createOptions() {
+ Options options = createNewOptions();
+
+ Option opTargetFile =
+ Option.builder(TARGET_DIR_ARGS)
+ .required()
+ .longOpt(TARGET_DIR_ARGS_NAME)
+ .hasArg()
+ .argName(TARGET_DIR_NAME)
+ .desc("Target File Directory (required)")
+ .build();
+ options.addOption(opTargetFile);
+
+ Option targetPathPattern =
+ Option.builder(TARGET_PATH_ARGS)
+ .longOpt(TARGET_PATH_ARGS_NAME)
+ .hasArg()
+ .argName(TARGET_PATH_NAME)
+ .desc("Export Path Pattern (optional)")
+ .build();
+ options.addOption(targetPathPattern);
+
+ Option targetFileName =
+ Option.builder(TARGET_FILE_ARGS)
+ .longOpt(TARGET_FILE_ARGS_NAME)
+ .hasArg()
+ .argName(TARGET_FILE_NAME)
+ .desc("Export File Name (optional)")
+ .build();
+ options.addOption(targetFileName);
+
+ Option opLinesPerFile =
+ Option.builder(LINES_PER_FILE_ARGS)
+ .longOpt(LINES_PER_FILE_ARGS_NAME)
+ .hasArg()
+ .argName(LINES_PER_FILE_NAME)
+ .desc("Lines per dump file.")
+ .build();
+ options.addOption(opLinesPerFile);
+
+ Option opTimeout =
+ Option.builder(TIMEOUT_ARGS)
+ .longOpt(TIMEOUT_ARGS_NAME)
+ .hasArg()
+ .argName(TIMEOUT_ARGS)
+ .desc(timeout + " Timeout for session query")
+ .build();
+ options.addOption(opTimeout);
+
+ Option opHelp =
+ Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help
information").build();
+ options.addOption(opHelp);
+ return options;
+ }
+
+ /**
+ * This method will be called, if the query commands are written in a sql
file.
+ *
+ * @param pathFile sql file path
+ * @throws IOException exception
+ */
+ private static void dumpFromPathFile(String pathFile) throws IOException {
+ try (BufferedReader reader = new BufferedReader(new FileReader(pathFile)))
{
+ String path;
+ int index = 0;
+ while ((path = reader.readLine()) != null) {
+ dumpResult(path, index);
+ index++;
+ }
+ }
+ }
+
+ /**
+ * Dump files from database to CSV file.
+ *
+ * @param pattern used to be export schema
+ * @param index used to create dump file name
+ */
+ private static void dumpResult(String pattern, int index) {
+ File file = new File(targetDirectory);
+ if (!file.isDirectory()) {
+ file.mkdir();
+ }
+ final String path = targetDirectory + targetFile + index;
+ try {
+ SessionDataSet sessionDataSet =
+ session.executeQueryStatement("show timeseries " + pattern, timeout);
+ writeCsvFile(sessionDataSet, path, sessionDataSet.getColumnNames(),
linesPerFile);
+ sessionDataSet.closeOperationHandle();
+ ioTPrinter.println("Export completely!");
+ } catch (StatementExecutionException | IoTDBConnectionException |
IOException e) {
+ ioTPrinter.println("Cannot dump result because: " + e.getMessage());
+ }
+ }
+
+ @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
+ public static void writeCsvFile(
+ SessionDataSet sessionDataSet, String filePath, List<String> headers,
int linesPerFile)
+ throws IOException, IoTDBConnectionException,
StatementExecutionException {
+ int viewTypeIndex = headers.indexOf(HEADER_VIEW_TYPE);
+ int timeseriesIndex = headers.indexOf(HEADER_TIMESERIES);
+
+ int fileIndex = 0;
+ boolean hasNext = true;
+ while (hasNext) {
+ int i = 0;
+ final String finalFilePath = filePath + "_" + fileIndex + ".csv";
+ final CSVPrinterWrapper csvPrinterWrapper = new
CSVPrinterWrapper(finalFilePath);
+ while (i++ < linesPerFile) {
+ if (sessionDataSet.hasNext()) {
+ if (i == 1) {
+ csvPrinterWrapper.printRecord(HEAD_COLUMNS);
+ }
+ RowRecord rowRecord = sessionDataSet.next();
+ List<Field> fields = rowRecord.getFields();
+ if
(fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE)
+ ||
!fields.get(viewTypeIndex).getStringValue().equals(BASE_VIEW_TYPE)) {
+ continue;
+ }
+ HEAD_COLUMNS.forEach(
+ column -> {
+ Field field = fields.get(headers.indexOf(column));
+ String fieldStringValue = field.getStringValue();
+ if (!"null".equals(field.getStringValue())) {
+ csvPrinterWrapper.print(fieldStringValue);
+ } else {
+ csvPrinterWrapper.print("");
+ }
+ });
+ csvPrinterWrapper.println();
+ } else {
+ hasNext = false;
+ break;
+ }
+ }
+ fileIndex++;
+ csvPrinterWrapper.flush();
+ csvPrinterWrapper.close();
+ }
+ }
+}
diff --git
a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java
b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java
new file mode 100644
index 00000000000..acd55b3eb42
--- /dev/null
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/ImportSchema.java
@@ -0,0 +1,653 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tool;
+
+import org.apache.iotdb.cli.utils.IoTPrinter;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.exception.ArgsErrorException;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE;
+
+/** Import Schema CSV file. */
+public class ImportSchema extends AbstractSchemaTool {
+
+ private static final String FILE_ARGS = "s";
+ private static final String FILE_NAME = "source";
+ private static final String FILE_ARGS_NAME = "sourceDir/sourceFile";
+
+ private static final String FAILED_FILE_ARGS = "fd";
+ private static final String FAILED_FILE_NAME = "fail_dir";
+ private static final String FAILED_FILE_ARGS_NAME = "failDir";
+
+ private static final String ALIGNED_ARGS = "aligned";
+ private static Boolean aligned = false;
+
+ private static final String BATCH_POINT_SIZE_ARGS = "batch";
+ private static final String BATCH_POINT_SIZE_NAME = "batch_size";
+ private static final String BATCH_POINT_SIZE_ARGS_NAME = "batchSize";
+ private static int batchPointSize = 10_000;
+
+ private static final String CSV_SUFFIXS = "csv";
+
+ private static final String LINES_PER_FAILED_FILE_ARGS = "lpf";
+ private static final String LINES_PER_FAILED_FILE_NAME = "lines_per_file";
+ private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "linesPerFile";
+ private static final String IMPORT_SCHEMA_CLI_PREFIX = "ImportSchema";
+ private static int linesPerFailedFile = 10000;
+
+ private static String targetPath;
+ private static String failedFileDirectory = null;
+
+ private static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when
insert csv because ";
+
+ private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
+
+ /**
+ * create the commandline options.
+ *
+ * @return object Options
+ */
+ private static Options createOptions() {
+ Options options = createNewOptions();
+
+ Option opFile =
+ Option.builder(FILE_ARGS)
+ .required()
+ .longOpt(FILE_NAME)
+ .hasArg()
+ .argName(FILE_ARGS_NAME)
+ .desc(
+ "If input a file path, load a csv file, "
+ + "otherwise load all csv file under this directory
(required)")
+ .build();
+ options.addOption(opFile);
+
+ Option opFailedFile =
+ Option.builder(FAILED_FILE_ARGS)
+ .required(false)
+ .longOpt(FAILED_FILE_NAME)
+ .hasArg()
+ .argName(FAILED_FILE_ARGS_NAME)
+ .desc(
+ "Specifying a directory to save failed file, default
YOUR_CSV_FILE_PATH (optional)")
+ .build();
+ options.addOption(opFailedFile);
+
+ Option opAligned =
+ Option.builder(ALIGNED_ARGS)
+ .longOpt(ALIGNED_ARGS)
+ .desc("Whether import schema as aligned timeseries(optional)")
+ .build();
+ options.addOption(opAligned);
+
+ Option opBatchPointSize =
+ Option.builder(BATCH_POINT_SIZE_ARGS)
+ .longOpt(BATCH_POINT_SIZE_NAME)
+ .hasArg()
+ .argName(BATCH_POINT_SIZE_ARGS_NAME)
+ .desc("10000 (only not aligned optional)")
+ .build();
+ options.addOption(opBatchPointSize);
+
+ Option opFailedLinesPerFile =
+ Option.builder(LINES_PER_FAILED_FILE_ARGS)
+ .longOpt(LINES_PER_FAILED_FILE_NAME)
+ .hasArg()
+ .argName(LINES_PER_FAILED_FILE_ARGS_NAME)
+ .desc("Lines per failed file")
+ .build();
+ options.addOption(opFailedLinesPerFile);
+
+ Option opHelp =
+ Option.builder(HELP_ARGS).longOpt(HELP_ARGS).desc("Display help
information").build();
+ options.addOption(opHelp);
+ return options;
+ }
+
+ /**
+ * parse optional params
+ *
+ * @param commandLine
+ */
+ private static void parseSpecialParams(CommandLine commandLine) throws
ArgsErrorException {
+ targetPath = commandLine.getOptionValue(FILE_ARGS);
+ if (commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS) != null) {
+ batchPointSize =
Integer.parseInt(commandLine.getOptionValue(BATCH_POINT_SIZE_ARGS));
+ }
+ if (commandLine.getOptionValue(FAILED_FILE_ARGS) != null) {
+ failedFileDirectory = commandLine.getOptionValue(FAILED_FILE_ARGS);
+ File file = new File(failedFileDirectory);
+ if (!file.isDirectory()) {
+ file.mkdir();
+ failedFileDirectory = file.getAbsolutePath() + File.separator;
+ }
+ }
+ if (commandLine.getOptionValue(ALIGNED_ARGS) != null) {
+ aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS));
+ }
+ if (commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS) != null) {
+ linesPerFailedFile =
Integer.parseInt(commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS));
+ }
+ }
+
+ public static void main(String[] args) throws IoTDBConnectionException {
+ Options options = createOptions();
+ HelpFormatter hf = new HelpFormatter();
+ hf.setOptionComparator(null);
+ hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
+ CommandLine commandLine = null;
+ CommandLineParser parser = new DefaultParser();
+
+ if (args == null || args.length == 0) {
+ ioTPrinter.println("Too few params input, please check the following
hint.");
+ hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (org.apache.commons.cli.ParseException e) {
+ ioTPrinter.println("Parse error: " + e.getMessage());
+ hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ if (commandLine.hasOption(HELP_ARGS)) {
+ hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ try {
+ parseBasicParams(commandLine);
+ String filename = commandLine.getOptionValue(FILE_ARGS);
+ if (filename == null) {
+ hf.printHelp(IMPORT_SCHEMA_CLI_PREFIX, options, true);
+ System.exit(CODE_ERROR);
+ }
+ parseSpecialParams(commandLine);
+ } catch (ArgsErrorException e) {
+ ioTPrinter.println("Args error: " + e.getMessage());
+ System.exit(CODE_ERROR);
+ } catch (Exception e) {
+ ioTPrinter.println("Encounter an error, because: " + e.getMessage());
+ System.exit(CODE_ERROR);
+ }
+ System.exit(importFromTargetPath(host, Integer.parseInt(port), username,
password, targetPath));
+ }
+
+ /**
+ * Specifying a CSV file or a directory including CSV files that you want to
import. This method
+ * can be offered to console cli to implement importing CSV file by command.
+ *
+ * @param host
+ * @param port
+ * @param username
+ * @param password
+ * @param targetPath a CSV file or a directory including CSV files
+ * @return the status code
+ * @throws IoTDBConnectionException
+ */
+ @SuppressWarnings({"squid:S2093"}) // ignore try-with-resources
+ public static int importFromTargetPath(
+ String host, int port, String username, String password, String
targetPath) {
+ try {
+ session = new Session(host, port, username, password, false);
+ session.open(false);
+ File file = new File(targetPath);
+ if (file.isFile()) {
+ importFromSingleFile(file);
+ } else if (file.isDirectory()) {
+ File[] files = file.listFiles();
+ if (files == null) {
+ return CODE_OK;
+ }
+ // 按文件名排序
+ Arrays.sort(files, (f1, f2) -> f1.getName().compareTo(f2.getName()));
+ for (File subFile : files) {
+ if (subFile.isFile()) {
+ importFromSingleFile(subFile);
+ }
+ }
+ } else {
+ ioTPrinter.println("File not found!");
+ return CODE_ERROR;
+ }
+ } catch (IoTDBConnectionException e) {
+ ioTPrinter.println("Encounter an error when connecting to server,
because " + e.getMessage());
+ return CODE_ERROR;
+ } finally {
+ if (session != null) {
+ try {
+ session.close();
+ } catch (IoTDBConnectionException e) {
+ ;
+ }
+ }
+ }
+ return CODE_OK;
+ }
+
+ /**
+ * import the CSV file and load headers and records.
+ *
+ * @param file the File object of the CSV file that you want to import.
+ */
+ private static void importFromSingleFile(File file) {
+ if (file.getName().endsWith(CSV_SUFFIXS)) {
+ try {
+ CSVParser csvRecords = readCsvFile(file.getAbsolutePath());
+ List<String> headerNames = csvRecords.getHeaderNames();
+ Stream<CSVRecord> records = csvRecords.stream();
+ if (headerNames.isEmpty()) {
+ ioTPrinter.println(file.getName() + " : Empty file!");
+ return;
+ }
+ if (!checkHeader(headerNames)) {
+ return;
+ }
+ String failedFilePath = null;
+ if (failedFileDirectory == null) {
+ failedFilePath = file.getAbsolutePath() + ".failed";
+ } else {
+ failedFilePath = failedFileDirectory + file.getName() + ".failed";
+ }
+ writeScheme(file.getName(), headerNames, records, failedFilePath);
+ } catch (IOException | IllegalPathException e) {
+ ioTPrinter.println(
+ file.getName() + " : CSV file read exception because: " +
e.getMessage());
+ }
+ } else {
+ ioTPrinter.println(file.getName() + " : The file name must end with
\"csv\"!");
+ }
+ }
+
+ /**
+ * if the data is aligned by time, the data will be written by this method.
+ *
+ * @param headerNames the header names of CSV file
+ * @param records the records of CSV file
+ * @param failedFilePath the directory to save the failed files
+ */
+ @SuppressWarnings("squid:S3776")
+ private static void writeScheme(
+ String fileName, List<String> headerNames, Stream<CSVRecord> records,
String failedFilePath)
+ throws IllegalPathException {
+ List<String> paths = new ArrayList<>();
+ List<TSDataType> dataTypes = new ArrayList<>();
+ List<TSEncoding> encodings = new ArrayList<>();
+ List<CompressionType> compressors = new ArrayList<>();
+ List<String> pathsWithAlias = new ArrayList<>();
+ List<TSDataType> dataTypesWithAlias = new ArrayList<>();
+ List<TSEncoding> encodingsWithAlias = new ArrayList<>();
+ List<CompressionType> compressorsWithAlias = new ArrayList<>();
+ List<String> measurementAlias = new ArrayList<>();
+
+ AtomicReference<Boolean> hasStarted = new AtomicReference<>(false);
+ AtomicInteger pointSize = new AtomicInteger(0);
+ ArrayList<List<Object>> failedRecords = new ArrayList<>();
+ records.forEach(
+ recordObj -> {
+ boolean failed = false;
+ if (!aligned) {
+ if (Boolean.FALSE.equals(hasStarted.get())) {
+ hasStarted.set(true);
+ } else if (pointSize.get() >= batchPointSize) {
+ try {
+ writeAndEmptyDataSet(
+ paths,
+ dataTypes,
+ encodings,
+ compressors,
+ null,
+ null,
+ null,
+ measurementAlias,
+ 3);
+ writeAndEmptyDataSet(
+ pathsWithAlias,
+ dataTypesWithAlias,
+ encodingsWithAlias,
+ compressorsWithAlias,
+ null,
+ null,
+ null,
+ null,
+ 3);
+ paths.clear();
+ dataTypes.clear();
+ encodings.clear();
+ compressors.clear();
+ measurementAlias.clear();
+ pointSize.set(0);
+ } catch (Exception e) {
+ failedRecords.add((List<Object>) (List<?>) paths);
+ }
+ }
+ } else {
+ paths.clear();
+ dataTypes.clear();
+ encodings.clear();
+ compressors.clear();
+ measurementAlias.clear();
+ }
+ String path =
recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(0)));
+ String alias =
recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(1)));
+ String dataTypeRaw =
recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(2)));
+ TSDataType dataType = typeInfer(dataTypeRaw);
+ String encodingTypeRaw =
recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(3)));
+ TSEncoding encodingType = encodingInfer(encodingTypeRaw);
+ String compressionTypeRaw =
recordObj.get(headerNames.indexOf(HEAD_COLUMNS.get(4)));
+ CompressionType compressionType = compressInfer(compressionTypeRaw);
+ if (StringUtils.isBlank(path) ||
path.trim().startsWith(SYSTEM_DATABASE)) {
+ ioTPrinter.println(
+ String.format(
+ "Line '%s', column '%s': illegal path %s",
+ recordObj.getRecordNumber(), headerNames, path));
+ failedRecords.add(recordObj.stream().collect(Collectors.toList()));
+ failed = true;
+ } else if (ObjectUtils.isEmpty(dataType)) {
+ ioTPrinter.println(
+ String.format(
+ "Line '%s', column '%s': '%s' unknown dataType %n",
+ recordObj.getRecordNumber(), path, dataTypeRaw));
+ failedRecords.add(recordObj.stream().collect(Collectors.toList()));
+ failed = true;
+ } else if (ObjectUtils.isEmpty(encodingType)) {
+ ioTPrinter.println(
+ String.format(
+ "Line '%s', column '%s': '%s' unknown encodingType %n",
+ recordObj.getRecordNumber(), path, encodingTypeRaw));
+ failedRecords.add(recordObj.stream().collect(Collectors.toList()));
+ failed = true;
+ } else if (ObjectUtils.isEmpty(compressionType)) {
+ ioTPrinter.println(
+ String.format(
+ "Line '%s', column '%s': '%s' unknown compressionType %n",
+ recordObj.getRecordNumber(), path, compressionTypeRaw));
+ failedRecords.add(recordObj.stream().collect(Collectors.toList()));
+ failed = true;
+ } else {
+ if (StringUtils.isBlank(alias)) {
+ paths.add(path);
+ dataTypes.add(dataType);
+ encodings.add(encodingType);
+ compressors.add(compressionType);
+ } else {
+ pathsWithAlias.add(path);
+ dataTypesWithAlias.add(dataType);
+ encodingsWithAlias.add(encodingType);
+ compressorsWithAlias.add(compressionType);
+ measurementAlias.add(alias);
+ }
+ pointSize.getAndIncrement();
+ }
+ if (!failed && aligned) {
+ String deviceId = path.substring(0, path.lastIndexOf("."));
+ paths.add(0, path.substring(deviceId.length() + 1));
+ writeAndEmptyDataSetAligned(
+ deviceId, paths, dataTypes, encodings, compressors,
measurementAlias, 3);
+ }
+ });
+ try {
+ if (CollectionUtils.isNotEmpty(paths)) {
+ writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null,
null, null, null, 3);
+ }
+ } catch (Exception e) {
+ paths.forEach(t -> failedRecords.add(Collections.singletonList(t)));
+ }
+ try {
+ if (CollectionUtils.isNotEmpty(pathsWithAlias)) {
+ writeAndEmptyDataSet(
+ pathsWithAlias,
+ dataTypesWithAlias,
+ encodingsWithAlias,
+ compressorsWithAlias,
+ null,
+ null,
+ null,
+ measurementAlias,
+ 3);
+ }
+ } catch (Exception e) {
+ pathsWithAlias.forEach(t ->
failedRecords.add(Collections.singletonList(t)));
+ }
+ pointSize.set(0);
+ if (!failedRecords.isEmpty()) {
+ writeFailedLinesFile(failedFilePath, failedRecords);
+ }
+ if (Boolean.TRUE.equals(hasStarted.get())) {
+ if (!failedRecords.isEmpty()) {
+ ioTPrinter.println(fileName + " : Import completely fail!");
+ } else {
+ ioTPrinter.println(fileName + " : Import completely successful!");
+ }
+ } else {
+ ioTPrinter.println(fileName + " : No records!");
+ }
+ }
+
+ private static boolean checkHeader(List<String> headerNames) {
+ if (CollectionUtils.isNotEmpty(headerNames)
+ && new HashSet<>(headerNames).size() == HEAD_COLUMNS.size()) {
+ List<String> strangers =
+ headerNames.stream().filter(t ->
!HEAD_COLUMNS.contains(t)).collect(Collectors.toList());
+ if (CollectionUtils.isNotEmpty(strangers)) {
+ ioTPrinter.println(
+ "The header of the CSV file to be imported is illegal. The correct
format is \"Timeseries, Alibaba, DataType, Encoding, Compression\"!");
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static void writeFailedLinesFile(
+ String failedFilePath, ArrayList<List<Object>> failedRecords) {
+ int fileIndex = 0;
+ int from = 0;
+ int failedRecordsSize = failedRecords.size();
+ int restFailedRecords = failedRecordsSize;
+ while (from < failedRecordsSize) {
+ int step = Math.min(restFailedRecords, linesPerFailedFile);
+ writeCsvFile(failedRecords.subList(from, from + step), failedFilePath +
"_" + fileIndex++);
+ from += step;
+ restFailedRecords -= step;
+ }
+ }
+
+ private static void writeAndEmptyDataSet(
+ List<String> paths,
+ List<TSDataType> dataTypes,
+ List<TSEncoding> encodings,
+ List<CompressionType> compressors,
+ List<Map<String, String>> propsList,
+ List<Map<String, String>> tagsList,
+ List<Map<String, String>> attributesList,
+ List<String> measurementAliasList,
+ int retryTime)
+ throws StatementExecutionException {
+ try {
+ session.createMultiTimeseries(
+ paths,
+ dataTypes,
+ encodings,
+ compressors,
+ propsList,
+ tagsList,
+ attributesList,
+ measurementAliasList);
+ } catch (IoTDBConnectionException e) {
+ if (retryTime > 0) {
+ try {
+ session.open();
+ } catch (IoTDBConnectionException ex) {
+ ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
+ }
+ writeAndEmptyDataSet(
+ paths,
+ dataTypes,
+ encodings,
+ compressors,
+ propsList,
+ tagsList,
+ attributesList,
+ measurementAliasList,
+ --retryTime);
+ }
+ } catch (StatementExecutionException e) {
+ try {
+ session.close();
+ } catch (IoTDBConnectionException ex) {
+ // do nothing
+ }
+ throw e;
+ }
+ }
+
+ private static void writeAndEmptyDataSetAligned(
+ String deviceId,
+ List<String> measurements,
+ List<TSDataType> dataTypes,
+ List<TSEncoding> encodings,
+ List<CompressionType> compressors,
+ List<String> measurementAliasList,
+ int retryTime) {
+ try {
+ session.createAlignedTimeseries(
+ deviceId, measurements, dataTypes, encodings, compressors,
measurementAliasList);
+ } catch (IoTDBConnectionException e) {
+ if (retryTime > 0) {
+ try {
+ session.open();
+ } catch (IoTDBConnectionException ex) {
+ ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
+ }
+ writeAndEmptyDataSetAligned(
+ deviceId,
+ measurements,
+ dataTypes,
+ encodings,
+ compressors,
+ measurementAliasList,
+ --retryTime);
+ }
+ } catch (StatementExecutionException e) {
+ ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage());
+ try {
+ session.close();
+ } catch (IoTDBConnectionException ex) {
+ // do nothing
+ }
+ System.exit(1);
+ } finally {
+ deviceId = null;
+ measurements.clear();
+ dataTypes.clear();
+ encodings.clear();
+ compressors.clear();
+ measurementAliasList.clear();
+ }
+ }
+
+ /**
+ * read data from the CSV file
+ *
+ * @param path
+ * @return CSVParser csv parser
+ * @throws IOException when reading the csv file failed.
+ */
+ private static CSVParser readCsvFile(String path) throws IOException {
+ return CSVFormat.Builder.create(CSVFormat.DEFAULT)
+ .setHeader()
+ .setSkipHeaderRecord(true)
+ .setQuote('`')
+ .setEscape('\\')
+ .setIgnoreEmptyLines(true)
+ .build()
+ .parse(new InputStreamReader(new FileInputStream(path)));
+ }
+
+ /**
+ * @param typeStr
+ * @return
+ */
+ private static TSDataType typeInfer(String typeStr) {
+ try {
+ if (StringUtils.isNotBlank(typeStr)) {
+ return TSDataType.valueOf(typeStr);
+ }
+ } catch (IllegalArgumentException e) {
+ ;
+ }
+ return null;
+ }
+
+ private static CompressionType compressInfer(String compressionType) {
+ try {
+ if (StringUtils.isNotBlank(compressionType)) {
+ return CompressionType.valueOf(compressionType);
+ }
+ } catch (IllegalArgumentException e) {
+ ;
+ }
+ return null;
+ }
+
+ private static TSEncoding encodingInfer(String encodingType) {
+ try {
+ if (StringUtils.isNotBlank(encodingType)) {
+ return TSEncoding.valueOf(encodingType);
+ }
+ } catch (IllegalArgumentException e) {
+ ;
+ }
+ return null;
+ }
+}
