This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch iotdb in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 330fa5b3461bc38a57bfb722c70eafdb0ce24454 Author: CloudWise-Lukemiao <[email protected]> AuthorDate: Fri Aug 23 10:24:26 2024 +0800 Fixed testing bug (#201) * Fixed testing bug * Fixed testing bug * Fixed ut failed * Fixed ut failed * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments * Optimize code based on review comments --------- Co-authored-by: luke.miao <[email protected]> --- java/tools/pom.xml | 18 +++ .../assembly/resources/conf/logback-cvs2tsfile.xml | 5 +- .../src/assembly/resources/tools/csv2tsfile.bat | 5 +- java/tools/src/assembly/tools.xml | 3 +- .../java/org/apache/tsfile/tools/SchemaParser.java | 38 ++++-- .../java/org/apache/tsfile/tools/TsFileTool.java | 140 +++++++++++++++++---- .../org/apache/tsfile/tools/TsfiletoolsTest.java | 75 ++++++++++- 7 files changed, 240 insertions(+), 44 deletions(-) diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 5b346794..57b2941d 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -43,11 +43,19 @@ <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> <dependency> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile</artifactId> <version>1.2.0-b31fb57c-SNAPSHOT</version> </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> @@ -60,6 +68,16 @@ </dependencies> <build> <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <usedDependencies combine.children="append"> + <!-- We just need this dependency to prevent the compiler from freaking out on unused imports --> + <usedDependency>ch.qos.logback:logback-classic</usedDependency> + </usedDependencies> + </configuration> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> diff --git a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml index d4d2ed8f..84a56a6c 100644 --- a/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml +++ b/java/tools/src/assembly/resources/conf/logback-cvs2tsfile.xml @@ -16,6 +16,7 @@ under the License. --> <configuration scan="true" scanPeriod="60 seconds"> + <statusListener class="ch.qos.logback.core.status.NopStatusListener"/> <appender class="ch.qos.logback.core.ConsoleAppender" name="stdout"> <Target>System.out</Target> <encoder> @@ -23,7 +24,7 @@ <charset>utf-8</charset> </encoder> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> + <level>INFO</level> </filter> </appender> <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="tools"> @@ -46,4 +47,6 @@ <appender-ref ref="stdout"/> <appender-ref ref="tools"/> </root> + <logger name="org.apache.tsfile.common.conf.TSFileDescriptor" level="OFF"/> + <logger name="org.apache.tsfile.write.TsFileWriter" level="OFF"/> </configuration> diff --git a/java/tools/src/assembly/resources/tools/csv2tsfile.bat b/java/tools/src/assembly/resources/tools/csv2tsfile.bat index d8bc1dff..ec03e3e1 100644 --- a/java/tools/src/assembly/resources/tools/csv2tsfile.bat +++ b/java/tools/src/assembly/resources/tools/csv2tsfile.bat @@ -39,11 +39,10 @@ set CLASSPATH="%TSFILE_HOME%\lib\*" if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.tsfile.tools.TsFileTool set TSFILE_CONF=%TSFILE_HOME%\conf -set "tsfile_params=-Dlogback.configurationFile=!IOTDB_CLI_CONF!\logback-cvs2tsfile.xml" -start /B "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*) > nul 2>&1" +set "tsfile_params=-Dlogback.configurationFile=!TSFILE_CONF!\logback-cvs2tsfile.xml" +start /B /WAIT "" cmd /C "("%JAVA_HOME%\bin\java" -DTSFILE_HOME=!TSFILE_HOME! !tsfile_params! !JAVA_OPTS! -cp !CLASSPATH! !MAIN_CLASS! %*)" exit /b - :err echo JAVA_HOME environment variable must be set! set ret_code=1 diff --git a/java/tools/src/assembly/tools.xml b/java/tools/src/assembly/tools.xml index ccd96051..d03bca13 100644 --- a/java/tools/src/assembly/tools.xml +++ b/java/tools/src/assembly/tools.xml @@ -25,7 +25,7 @@ <format>dir</format> <format>zip</format> </formats> - <includeBaseDirectory>false</includeBaseDirectory> + <includeBaseDirectory>true</includeBaseDirectory> <dependencySets> <dependencySet> <outputDirectory>lib</outputDirectory> @@ -45,6 +45,7 @@ <file> <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.sh</source> <destName>tools/csv2tsfile.sh</destName> + <fileMode>0755</fileMode> </file> <file> <source>${maven.multiModuleProjectDirectory}/java/tools/src/assembly/resources/tools/csv2tsfile.bat</source> diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java index 0bb9816e..779cfbb1 100644 --- a/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java +++ b/java/tools/src/main/java/org/apache/tsfile/tools/SchemaParser.java @@ -23,16 +23,17 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class SchemaParser { public static class Schema { - String tableName; - String timePrecision; + String tableName = ""; + String timePrecision = "ms"; boolean hasHeader = true; - String separator; + String separator = ","; String nullFormat; - String timeColumn; + String timeColumn = ""; int timeColumnIndex = -1; List<IDColumns> idColumns = new ArrayList<>(); List<Column> csvColumns = new ArrayList<>(); @@ -155,7 +156,12 @@ public class SchemaParser { } else if (line.startsWith("time_precision=")) { schema.timePrecision = extractValue(line); } else if (line.startsWith("has_header=")) { - schema.hasHeader = Boolean.parseBoolean(extractValue(line)); + String has_header = extractValue(line); + if (has_header.equals("true") || has_header.equals("false")) { + schema.hasHeader = Boolean.parseBoolean(has_header); + } else { + throw new IllegalArgumentException("The data format of has_header is incorrect"); + } } else if (line.startsWith("separator=")) { schema.separator = extractValue(line); } else if (line.startsWith("null_format=")) { @@ -208,7 +214,7 @@ public class SchemaParser { for (IDColumns idColumn : idColumnsList) { if (!idColumn.isDefault) { for (int j = 0; j < columnList.size(); j++) { - if (columnList.get(j).name.equals(idColumn.name)) { + if (Objects.equals(columnList.get(j).name, idColumn.name)) { idColumn.csvColumnIndex = j; break; } @@ -244,21 +250,33 @@ public class SchemaParser { if (!schema.timePrecision.equals("us") && !schema.timePrecision.equals("ms") && !schema.timePrecision.equals("ns")) { - throw new IllegalArgumentException("timePrecision must be us,ms or ns"); + throw new IllegalArgumentException("The time_precision parameter only supports ms,us,ns"); } if (!schema.separator.equals(",") && !schema.separator.equals("tab") && !schema.separator.equals(";")) { throw new IllegalArgumentException("separator must be \",\", tab, or \";\""); } - if (schema.timeColumnIndex < 0) { - throw new IllegalArgumentException("time_column is required"); - } if (schema.tableName.isEmpty()) { throw new IllegalArgumentException("table_name is required"); } + if (schema.idColumns.isEmpty()) { + throw new IllegalArgumentException("id_columns is required"); + } if (schema.csvColumns.isEmpty()) { throw new IllegalArgumentException("csv_columns is required"); } + if (schema.timeColumn.isEmpty()) { + throw new IllegalArgumentException("time_column is required"); + } else if (schema.timeColumnIndex < 0) { + throw new IllegalArgumentException( + "The value " + schema.timeColumn + " of time_column is not in csv_columns"); + } + for (IDColumns idColumn : schema.idColumns) { + if (idColumn.csvColumnIndex < 0 && !idColumn.isDefault) { + throw new IllegalArgumentException( + "The value " + idColumn.name + " of id_columns is not in csv_columns"); + } + } } } diff --git a/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java index 0d5c1cfe..2c62982e 100644 --- a/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java +++ b/java/tools/src/main/java/org/apache/tsfile/tools/TsFileTool.java @@ -35,11 +35,14 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; @@ -97,6 +100,7 @@ public class TsFileTool { } catch (InterruptedException e) { LOGGER.error("Failed to await termination", e); } + LOGGER.info("The " + inputDirectoryStr + " directory or file has completed execution"); } } @@ -140,8 +144,14 @@ public class TsFileTool { return new TableSchema(tableName, measurementSchemas, columnTypes); } - private static boolean writeTsFile(String fileName, List<String> lineList) { - final File tsFile = new File(outputDirectoryStr, fileName); + private static boolean writeTsFile( + String sourceFilePath, String fileName, List<String> lineList) { + String inputFileAbsolutePath = new File(inputDirectoryStr).getAbsolutePath(); + String soureFlieName = new File(sourceFilePath).getName(); + String fileOutPutDirStr = + outputDirectoryStr + + sourceFilePath.replace(inputFileAbsolutePath, "").replace(soureFlieName, ""); + final File tsFile = new File(fileOutPutDirStr, fileName); TsFileWriter writer = null; try { writer = new TsFileWriter(tsFile); @@ -158,20 +168,25 @@ public class TsFileTool { return false; } } catch (Exception e) { - e.printStackTrace(); - LOGGER.error("Failed to write file: " + tsFile); + LOGGER.error("Failed to write file: " + tsFile.getAbsolutePath(), e); return false; } finally { if (writer != null) { try { writer.close(); } catch (IOException e) { - e.printStackTrace(); + LOGGER.error("Failed to close file: " + tsFile.getAbsolutePath(), e); } } } } + private static void deleteFile(File tsfile) { + if (!tsfile.delete()) { + LOGGER.error(tsfile.getAbsolutePath() + " delete failed"); + } + } + private static Tablet genTablet( TableSchema tableSchema, List<String> lineList, Map<String, Object> defaultMap) { int num = lineList.size(); @@ -188,9 +203,9 @@ public class TsFileTool { map.put(column.name, i); } try { + List<String[]> parsedLines = sortAndParseLines(lineList); for (int i = 0; i < num; i++) { - String line = lineList.get(i); - String[] lineArray = line.split(schema.separator); + String[] lineArray = parsedLines.get(i); long timestamp = DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( lineArray[schema.timeColumnIndex], schema.timePrecision); @@ -219,11 +234,30 @@ public class TsFileTool { tablet.rowSize = num; return tablet; } catch (Exception e) { - LOGGER.error("Failed to parse csv file"); + LOGGER.error("Failed to parse csv file", e); } return null; } + public static List<String[]> sortAndParseLines(List<String> data) { + List<String[]> parsedLines = new ArrayList<>(data.size()); + + for (String line : data) { + parsedLines.add(line.split(schema.separator)); + } + parsedLines.sort( + (o1, o2) -> { + long time1 = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + o1[schema.timeColumnIndex], schema.timePrecision); + long time2 = + DateTimeUtils.convertTimestampOrDatetimeStrToLongWithDefaultZone( + o2[schema.timeColumnIndex], schema.timePrecision); + return Long.compare(time1, time2); + }); + return parsedLines; + } + public static Object getValue(TSDataType dataType, String i, Tablet.ColumnType columnType) { switch (dataType) { case INT64: @@ -266,15 +300,46 @@ public class TsFileTool { private static void cpFile(String sourceFilePath, String targetDirectoryPath) { try { - Files.createDirectories(Paths.get(targetDirectoryPath)); + String inputFileAbsolutePath = new File(inputDirectoryStr).getAbsolutePath(); + String soureFlieName = new File(sourceFilePath).getName(); + String fileOutPutDirStr = + targetDirectoryPath + + sourceFilePath.replace(inputFileAbsolutePath, "").replace(soureFlieName, ""); + Files.createDirectories(Paths.get(fileOutPutDirStr)); Path sourcePath = Paths.get(sourceFilePath); - Path targetPath = Paths.get(targetDirectoryPath, sourcePath.getFileName().toString()); + Path targetPath = Paths.get(fileOutPutDirStr, sourcePath.getFileName().toString()); Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { LOGGER.error("Failed to copy file: " + sourceFilePath, e); } } + public static void writeToNewCSV( + String headerLine, String fileAbsolutePath, List<String> data, String newFileName) { + if (schema.hasHeader && StringUtils.isNotEmpty(headerLine)) { + data.add(0, headerLine); + } + String inputFileAbsolutePath = new File(inputDirectoryStr).getAbsolutePath(); + String soureFlieName = new File(fileAbsolutePath).getName(); + String fileOutPutDirStr = + failedDirectoryStr + + fileAbsolutePath.replace(inputFileAbsolutePath, "").replace(soureFlieName, ""); + try { + Files.createDirectories(Paths.get(fileOutPutDirStr)); + } catch (IOException e) { + throw new RuntimeException(e); + } + String path = Paths.get(fileOutPutDirStr, newFileName).toFile().getAbsolutePath(); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(path))) { + for (String line : data) { + writer.write(line); + writer.newLine(); + } + } catch (IOException e) { + LOGGER.error("Error writing to CSV file", e); + } + } + private static void processFile(File inputFile, ExecutorService executor) { AtomicInteger fileCounter = new AtomicInteger(1); String fileName = FilenameUtils.getBaseName(inputFile.getName()); @@ -289,6 +354,7 @@ public class TsFileTool { int index = 0; List<String> lineList = new ArrayList<>(); boolean isSingleFile = true; + String headerLine = null; while ((line = reader.readLine()) != null) { if (index == 0) { if (schema.timeColumnIndex == -1) { @@ -308,6 +374,7 @@ public class TsFileTool { } if (schema.hasHeader && index == 0) { + headerLine = line; index++; continue; } @@ -318,6 +385,7 @@ public class TsFileTool { isSingleFile = false; if (chunkLines > 0) { submitChunk( + headerLine, lineList, fileCounter.getAndIncrement(), executor, @@ -330,6 +398,7 @@ public class TsFileTool { } else { lineList.add(line); submitChunk( + headerLine, lineList, fileCounter.getAndIncrement(), executor, @@ -347,6 +416,7 @@ public class TsFileTool { } if (lineList.size() > 0) { submitChunk( + headerLine, lineList, fileCounter.getAndIncrement(), executor, @@ -356,11 +426,12 @@ public class TsFileTool { } } catch (IOException e) { - e.printStackTrace(); + LOGGER.error("Error reading file", e); } } private static void submitChunk( + String headerLine, List<String> lineList, int fileNumber, ExecutorService executor, @@ -371,12 +442,28 @@ public class TsFileTool { () -> { boolean isSuccess; if (isSingleFile) { - isSuccess = writeTsFile(fileName + ".tsfile", lineList); + isSuccess = writeTsFile(fileAbsolutePath, fileName + ".tsfile", lineList); } else { - isSuccess = writeTsFile(fileName + "_" + fileNumber + ".tsfile", lineList); + isSuccess = + writeTsFile(fileAbsolutePath, fileName + "_" + fileNumber + ".tsfile", lineList); } if (!isSuccess) { - cpFile(fileAbsolutePath, failedDirectoryStr); + if (isSingleFile) { + cpFile(fileAbsolutePath, failedDirectoryStr); + File tsfile = new File(outputDirectoryStr, fileName + ".tsfile"); + deleteFile(tsfile); + } else { + writeToNewCSV( + headerLine, fileAbsolutePath, lineList, fileName + "_" + fileNumber + ".csv"); + File tsfile = new File(outputDirectoryStr, fileName + "_" + fileNumber + ".tsfile"); + deleteFile(tsfile); + } + } else { + String tsFileName = fileName + "_" + fileNumber + ".tsfile"; + if (isSingleFile) { + tsFileName = fileName + ".tsfile"; + } + LOGGER.info(tsFileName + " successfully generated"); } }); } @@ -402,7 +489,7 @@ public class TsFileTool { if (cmd.hasOption("h")) { printHelp(options); - return; + System.exit(0); } if (cmd.hasOption("s")) { @@ -428,7 +515,7 @@ public class TsFileTool { failedDirectoryStr = "failed"; } } catch (ParseException e) { - e.printStackTrace(); + LOGGER.error("Error parsing command line options", e); } } @@ -469,23 +556,26 @@ public class TsFileTool { } private static boolean validateParams() { - if (inputDirectoryStr == null - || inputDirectoryStr.isEmpty() - || outputDirectoryStr == null - || outputDirectoryStr.isEmpty() - || schemaPathStr == null - || schemaPathStr.isEmpty()) { - LOGGER.error("Missing required parameters. Please provide --source, --target, and --schema."); + if (inputDirectoryStr == null || inputDirectoryStr.isEmpty()) { + LOGGER.error("Missing required parameters.--source/-s is a required"); + return false; + } + if (outputDirectoryStr == null || outputDirectoryStr.isEmpty()) { + LOGGER.error("Missing required parameters. --target/-t is a required"); + return false; + } + if (schemaPathStr == null || schemaPathStr.isEmpty()) { + LOGGER.error("Missing required parameters. --schema is a required"); return false; } File sourceDir = new File(inputDirectoryStr); if (!sourceDir.exists()) { - LOGGER.error("Source directory(" + sourceDir + ") does not exist."); + LOGGER.error(sourceDir + " directory or file does not exist."); return false; } File schemaFile = new File(schemaPathStr); if (!schemaFile.exists()) { - LOGGER.error("Schema file(" + schemaPathStr + ") does not exist."); + LOGGER.error(schemaPathStr + " schema file does not exist."); return false; } if (THREAD_COUNT <= 0) { diff --git a/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java index 455ab1c5..ddbe32e8 100644 --- a/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java +++ b/java/tools/src/test/java/org/apache/tsfile/tools/TsfiletoolsTest.java @@ -31,8 +31,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -45,8 +47,12 @@ import static org.junit.Assert.assertTrue; public class TsfiletoolsTest { private final String testDir = "target" + File.separator + "csvTest"; private final String csvFile = testDir + File.separator + "data.csv"; + + private final String wrongCsvFile = testDir + File.separator + "dataWrong.csv"; private final String schemaFile = testDir + File.separator + "schemaFile.txt"; + private final String failedDir = testDir + File.separator + "failed"; + float[] tmpResult2 = new float[20]; float[] tmpResult3 = new float[20]; float[] tmpResult5 = new float[20]; @@ -55,6 +61,7 @@ public class TsfiletoolsTest { public void setUp() { new File(testDir).mkdirs(); genCsvFile(20); + genWrongCsvFile(100); genSchemaFile(); } @@ -71,12 +78,18 @@ public class TsfiletoolsTest { writer.write("null_format=\\N"); writer.newLine(); writer.newLine(); + writer.write("id_columns"); + writer.newLine(); + writer.write("tmp1"); + writer.newLine(); writer.write("time_column=time"); writer.newLine(); writer.write("csv_columns"); writer.newLine(); writer.write("time INT64,"); writer.newLine(); + writer.write("tmp1 TEXT,"); + writer.newLine(); writer.write("tmp2 FLOAT,"); writer.newLine(); writer.write("tmp3 FLOAT,"); @@ -85,20 +98,50 @@ public class TsfiletoolsTest { writer.newLine(); writer.write("tmp5 FLOAT"); } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException("Failed to generate schema file", e); + } + } + + public void genWrongCsvFile(int rows) { + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(wrongCsvFile))) { + writer.write("time,tmp1,tmp2,tmp3,tmp4,tmp5"); + writer.newLine(); + Random random = new Random(); + long timestamp = System.currentTimeMillis(); + + for (int i = 0; i < rows; i++) { + timestamp = timestamp + i; + String tmp1 = "s1"; + float tmp2 = random.nextFloat(); + float tmp3 = random.nextFloat(); + float tmp4 = random.nextFloat(); + float tmp5 = random.nextFloat(); + if (i % 99 == 0) { + writer.write( + timestamp + "aa" + "," + tmp1 + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," + tmp5); + } else { + writer.write(timestamp + "," + tmp1 + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," + tmp5); + } + + writer.newLine(); + } + } catch (IOException e) { + throw new RuntimeException("Failed to generate wrong CSV file", e); } } public void genCsvFile(int rows) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile))) { - writer.write("time,tmp2,tmp3,tmp4,tmp5"); + writer.write("time,tmp1,tmp2,tmp3,tmp4,tmp5"); writer.newLine(); Random random = new Random(); long timestamp = System.currentTimeMillis(); for (int i = 0; i < rows; i++) { timestamp = timestamp + i; + String tmp1 = "s1"; float tmp2 = random.nextFloat(); float tmp3 = random.nextFloat(); float tmp4 = random.nextFloat(); @@ -106,11 +149,11 @@ public class TsfiletoolsTest { tmpResult2[i] = tmp2; tmpResult3[i] = tmp3; tmpResult5[i] = tmp5; - writer.write(timestamp + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," + tmp5); + writer.write(timestamp + "," + tmp1 + "," + tmp2 + "," + tmp3 + "," + tmp4 + "," + tmp5); writer.newLine(); } } catch (IOException e) { - e.printStackTrace(); + throw new RuntimeException("Failed to generate CSV file", e); } } @@ -155,4 +198,28 @@ public class TsfiletoolsTest { assertEquals(20, cnt); } } + + @Test + public void testCsvToTsfileFailed() { + String scFilePath = new File(schemaFile).getAbsolutePath(); + String csvFilePath = new File(wrongCsvFile).getAbsolutePath(); + String targetPath = new File(testDir).getAbsolutePath(); + String fd = new File(failedDir).getAbsolutePath(); + String[] args = + new String[] { + "-s" + csvFilePath, "-schema" + scFilePath, "-t" + targetPath, "-fail_dir" + fd + }; + TsFileTool.main(args); + assertTrue(new File(failedDir + File.separator + "dataWrong.csv").exists()); + try (BufferedReader br = + new BufferedReader(new FileReader(failedDir + File.separator + "dataWrong.csv"))) { + int num = 0; + while (br.readLine() != null) { + num++; + } + assertEquals(101, num); + } catch (IOException e) { + throw new RuntimeException("IOException occurred while reading file", e); + } + } }
