Caideyipi commented on code in PR #18987:
URL: https://github.com/apache/iotdb/pull/18987#discussion_r4346216987


##########
iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java:
##########
@@ -250,239 +202,236 @@ private static String 
identifierValue(RelationalSqlParser.IdentifierContext cont
     return value.toLowerCase(Locale.ROOT);
   }
 
-  private static String quoteIdentifier(String name) {
-    return "\"" + name.replace("\"", "\"\"") + "\"";
-  }
-
-  private void exportToSqlFile(SessionDataSet sessionDataSet, String table, 
String filePath)
-      throws IOException, IoTDBConnectionException, 
StatementExecutionException {
-    processedRows = 0;
-    lastPrintTime = 0;
-    StringBuilder sqlBuilder;
-    List<String> headers = sessionDataSet.getColumnNames();
-    String prevSql =
-        "insert into "
-            + quoteIdentifier(table)
-            + "("
-            + headers.stream()
-                .map(ExportDataTable::quoteIdentifier)
-                .collect(Collectors.joining(","))
-            + ") values(";
-    SessionDataSet.DataIterator iterator = sessionDataSet.iterator();
-    List<String> columnTypeList = iterator.getColumnTypeList();
-    int totalColumns = columnTypeList.size();
-    int fileIndex = 0;
-    boolean fromOuterLoop = false;
-    while (iterator.next()) {
-      final String finalFilePath = filePath + "_" + fileIndex + ".sql";
-      int countLine = 0;
-      fromOuterLoop = true;
-      try (FileWriter writer = new FileWriter(finalFilePath)) {
-        while (countLine++ < linesPerFile && (fromOuterLoop || 
iterator.next())) {
-          fromOuterLoop = false;
-          sqlBuilder = new StringBuilder();
-          sqlBuilder.append(prevSql);
-          for (int curColumnIndex = 0; curColumnIndex < totalColumns; 
curColumnIndex++) {
-            if (curColumnIndex > 0) {
-              sqlBuilder.append(",");
-            }
-            String columnType = columnTypeList.get(curColumnIndex);
-            String columnValue = iterator.getString(curColumnIndex + 1);
-            sqlBuilder.append(formatSqlValue(columnValue, columnType));
-          }
-          sqlBuilder.append(");\n");
-          writer.write(sqlBuilder.toString());
-          processedRows += 1;
-          if (System.currentTimeMillis() - lastPrintTime > updateTimeInterval) 
{
-            ioTPrinter.printf(Constants.PROCESSED_PROGRESS, processedRows);
-            lastPrintTime = System.currentTimeMillis();
-          }
-        }
-        writer.flush();
-        fileIndex++;
+  private List<String> names(String sql) throws Exception {
+    List<String> names = new ArrayList<>();
+    try (Resource<SessionDataSet> result =
+        own(session.executeQueryStatement(sql, config.timeout))) {
+      while (result.get().hasNext()) {
+        progress.checkActive();
+        names.add(result.get().next().getField(0).getStringValue());
       }
     }
-    ioTPrinter.print("\n");
+    return names;
   }
 
-  private Boolean exportToTsFile(SessionDataSet sessionDataSet, String 
filePath, String table)
-      throws IOException,
-          IoTDBConnectionException,
-          StatementExecutionException,
-          WriteProcessException {
-    processedRows = 0;
-    lastPrintTime = 0;
-    List<String> columnNamesRaw = sessionDataSet.getColumnNames();
-    List<TSDataType> columnTypesRaw =
-        sessionDataSet.getColumnTypes().stream().map(t -> 
getType(t)).collect(Collectors.toList());
-    File f = FSFactoryProducer.getFSFactory().getFile(filePath);
-    if (f.exists()) {
-      Files.delete(f.toPath());
+  @Override
+  void export(ExportTask task) throws Exception {
+    if (!"csv".equals(config.format)) {
+      legalCheck(task.sql());
     }
-    boolean isEmpty = false;
-    Map<String, Integer> deviceColumnIndices = new HashMap<>();
-    List<ColumnSchema> columnSchemas = collectSchemas(columnNamesRaw, 
deviceColumnIndices, table);
-    try (ITsFileWriter tsFileWriter =
-        new TsFileWriterBuilder()
-            .file(f)
-            .tableSchema(new 
org.apache.tsfile.file.metadata.TableSchema(table, columnSchemas))
-            .memoryThreshold(Constants.memoryThreshold)
-            .build()) {
-      List<String> columnNames = new ArrayList<>(columnNamesRaw);
-      List<TSDataType> columnTypes = new ArrayList<>(columnTypesRaw);
-      int timeIndex = columnNamesRaw.indexOf(timeColumn.toLowerCase());
-      if (timeIndex >= 0) {
-        columnNames.remove(timeIndex);
-        columnTypes.remove(timeIndex);
+    progress.taskStarted(task);
+    try (Resource<SessionDataSet> result =
+        own(session.executeQueryStatement(task.sql(), config.timeout))) {
+      switch (config.format) {
+        case "sql":
+          exportSql(result.get(), task);
+          break;
+        case "tsfile":
+          exportTsFile(result.get(), task);
+          break;
+        default:
+          exportCsv(result.get(), task);
       }
-      Tablet tablet = new Tablet(columnNames, columnTypes);
-      if (ObjectUtils.isNotEmpty(tablet)) {
-        writeWithTablets(sessionDataSet, tablet, deviceColumnIndices, 
tsFileWriter);
-      } else {
-        isEmpty = true;
-      }
-    }
-    if (isEmpty) {
-      ioTPrinter.println("!!!Warning:Tablet is empty,no data can be 
exported.");
-      return false;
     }
-    return true;
+    progress.taskCompleted();
   }
 
-  private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath)
-      throws IOException, IoTDBConnectionException, 
StatementExecutionException {
-    processedRows = 0;
-    lastPrintTime = 0;
-    List<String> headers = getCsvHeaders(sessionDataSet, false);
-    int fileIndex = 0;
-    SessionDataSet.DataIterator iterator = sessionDataSet.iterator();
-    List<String> columnTypeList = iterator.getColumnTypeList();
-    int totalColumns = columnTypeList.size();
-    boolean fromOuterloop = false;
-    while (iterator.next()) {
-      final String finalFilePath = filePath + "_" + fileIndex + ".csv";
-      final CSVPrinterWrapper csvPrinterWrapper = new 
CSVPrinterWrapper(finalFilePath);
-      try {
-        csvPrinterWrapper.printRecord(headers);
-        fromOuterloop = true;
-        int countLine = 0;
-        while (countLine++ < linesPerFile && (fromOuterloop || 
iterator.next())) {
-          fromOuterloop = false;
-          for (int curColumnIndex = 0; curColumnIndex < totalColumns; 
curColumnIndex++) {
-            String curType = columnTypeList.get(curColumnIndex);
-            if (curType.equalsIgnoreCase("TIMESTAMP")) {
-              
csvPrinterWrapper.print(timeTrans(iterator.getLong(curColumnIndex + 1)));
-            } else {
-              if (iterator.isNull(curColumnIndex + 1)) {
-                csvPrinterWrapper.print("");
-              } else {
-                String columnValue = iterator.getString(curColumnIndex + 1);
-                if (curType.equalsIgnoreCase("TEXT") || 
curType.equalsIgnoreCase("STRING")) {
-                  csvPrinterWrapper.print("\"" + columnValue + "\"");
-                } else {
-                  csvPrinterWrapper.print(columnValue);
-                }
-              }
+  private void exportCsv(SessionDataSet data, ExportTask task) throws 
Exception {
+    List<String> names = data.getColumnNames();
+    SessionDataSet.DataIterator iterator = data.iterator();
+    List<String> types = iterator.getColumnTypeList();
+    try (Resource<ExportTextWriter> output =
+        own(new ExportTextWriter(config, task, progress, csvHeaders(names, 
types, false)))) {
+      while (iterator.next()) {
+        progress.checkActive();
+        List<Object> values = new ArrayList<>(names.size());
+        for (int i = 0; i < names.size(); i++) {
+          if (iterator.isNull(i + 1)) {
+            values.add("");
+          } else if ("TIMESTAMP".equals(types.get(i))) {
+            values.add(timeTrans(iterator.getLong(i + 1)));
+          } else {
+            String value = iterator.getString(i + 1);
+            if ("TEXT".equals(types.get(i)) || "STRING".equals(types.get(i))) {
+              value = '"' + value + '"';
             }
-          }
-          csvPrinterWrapper.println();
-          processedRows += 1;
-          if (System.currentTimeMillis() - lastPrintTime > updateTimeInterval) 
{
-            ioTPrinter.printf(Constants.PROCESSED_PROGRESS, processedRows);
-            lastPrintTime = System.currentTimeMillis();
+            values.add(value);
           }
         }
-        fileIndex++;
-        csvPrinterWrapper.flush();
-      } finally {
-        csvPrinterWrapper.close();
+        output.get().csvRow(values);
       }
+      output.get().complete();
     }
-    ioTPrinter.print("\n");
   }
 
-  private static void writeWithTablets(
-      SessionDataSet sessionDataSet,
-      Tablet tablet,
-      Map<String, Integer> deviceColumnIndices,
-      ITsFileWriter tsFileWriter)
-      throws IoTDBConnectionException,
-          StatementExecutionException,
-          IOException,
-          WriteProcessException {
-    while (sessionDataSet.hasNext()) {
-      RowRecord rowRecord = sessionDataSet.next();
-      List<Field> fields = rowRecord.getFields();
-      int rowIndex = tablet.getRowSize();
-      tablet.addTimestamp(
-          rowIndex, 
fields.get(deviceColumnIndices.get(timeColumn.toLowerCase())).getLongV());
-      List<IMeasurementSchema> schemas = tablet.getSchemas();
-      for (int i = 0; i < schemas.size(); i++) {
-        IMeasurementSchema measurementSchema = schemas.get(i);
-        // -1 for time not in fields
-        final String measurementName = measurementSchema.getMeasurementName();
-        if (timeColumn.equalsIgnoreCase(measurementName)) {
-          continue;
+  private void exportSql(SessionDataSet data, ExportTask task) throws 
Exception {
+    validateColumns(data, task.table());
+    List<String> headers = data.getColumnNames();
+    SessionDataSet.DataIterator iterator = data.iterator();
+    List<String> types = iterator.getColumnTypeList();
+    List<String> quotedHeaders = new ArrayList<>();
+    for (String header : headers) {
+      quotedHeaders.add(identifier(header));
+    }
+    String prefix =
+        "insert into "
+            + identifier(task.table())
+            + "("
+            + String.join(",", quotedHeaders)
+            + ") values(";
+    try (Resource<ExportTextWriter> output =
+        own(new ExportTextWriter(config, task, progress, null))) {
+      while (iterator.next()) {
+        progress.checkActive();
+        StringBuilder sql = new StringBuilder(prefix);
+        for (int i = 0; i < headers.size(); i++) {
+          if (i > 0) {
+            sql.append(',');
+          }
+          sql.append(sqlValue(iterator, types.get(i), i + 1));
         }
-        Object value =
-            fields
-                .get(deviceColumnIndices.get(measurementName))
-                .getObjectValue(measurementSchema.getType());
-        tablet.addValue(measurementName, rowIndex, value);
+        output.get().sqlRow(sql.append(");\n").toString());
       }
+      output.get().complete();
+    }
+  }
 
-      if (tablet.getRowSize() == tablet.getMaxRowNumber()) {
-        writeToTsFile(tsFileWriter, tablet);
-        processedRows += tablet.getRowSize();
-        tablet.initBitMaps();
-        tablet.reset();
-      }
-      if (System.currentTimeMillis() - lastPrintTime > updateTimeInterval) {
-        ioTPrinter.printf(Constants.PROCESSED_PROGRESS, processedRows);
-        lastPrintTime = System.currentTimeMillis();
+  private Map<String, ColumnSchema> validateColumns(SessionDataSet data, 
String table)
+      throws Exception {
+    Map<String, ColumnSchema> schemas = new LinkedHashMap<>();
+    try (Resource<SessionDataSet> result =
+        own(session.executeQueryStatement("DESCRIBE " + identifier(table), 
config.timeout))) {
+      while (result.get().hasNext()) {
+        progress.checkActive();
+        RowRecord row = result.get().next();
+        String name = row.getField(0).getStringValue();
+        if (!"time".equals(name)) {
+          schemas.put(
+              name,
+              new ColumnSchemaBuilder()
+                  .name(name)
+                  
.dataType(TSDataType.valueOf(row.getField(1).getStringValue()))
+                  
.category(ColumnCategory.valueOf(row.getField(2).getStringValue()))
+                  .build());
+        }
       }
     }
-    if (tablet.getRowSize() != 0) {
-      writeToTsFile(tsFileWriter, tablet);
-      processedRows += tablet.getRowSize();
-      if (System.currentTimeMillis() - lastPrintTime > updateTimeInterval) {
-        ioTPrinter.printf(Constants.PROCESSED_PROGRESS, processedRows);
-        lastPrintTime = System.currentTimeMillis();
+    List<String> columns = data.getColumnNames();
+    List<String> types = data.getColumnTypes();
+    if (!columns.contains("time")) {

Review Comment:
   Could this require at least one projected FIELD column as well? `SELECT 
time, deviceId FROM metrics` passes when `deviceId` is a TAG, but SQL export 
then emits an INSERT with only time/TAG columns, which IoTDB rejects (`No Field 
column present`). That makes an accepted SQL export impossible to re-import.



##########
iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java:
##########
@@ -222,22 +174,22 @@ public Void 
visitTableName(RelationalSqlParser.TableNameContext context) {
         }
       }.visit(parser.singleStatement());
     } catch (ParseCancellationException e) {
-      throw new StatementExecutionException(message);
+      throw invalid;
     }
-    // Multiple source relations do not have an unambiguous destination table.
+    // A query reading several relations has no single destination table.
     if (sources.size() != 1) {

Review Comment:
   Could this also validate the projection, rather than only counting source 
tables? A computed expression aliased to an existing column name and returning 
the same type passes the later name/type checks, then SQL/TsFile export writes 
the computed value back under that original column. A backup round-trip can 
therefore silently alter data.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to