This is an automated email from the ASF dual-hosted git repository. hxd pushed a commit to branch tsfileTools in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 60eeb83a61045850f5779c1383bb034f0b08e49c Author: xiangdong huang <[email protected]> AuthorDate: Fri Feb 14 16:36:41 2020 +0800 fix bugs of datahealthChecker --- .../apache/iotdb/db/tools/DataHealthChecker.java | 68 +++++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java b/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java index 7c918bf..8231f94 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/DataHealthChecker.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.tools; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -46,8 +48,12 @@ import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; public class DataHealthChecker { public static void main(String[] args) { String dataFolder = "data/data/";//the dataFoler has two subfolders: sequence and unsequence + String progressFile = "progress.log"; if (args.length == 1) { dataFolder = args[0]; + } if (args.length == 2) { + dataFolder = args[0]; + progressFile = args[1]; } System.out.println("data folder path:" + dataFolder); File folder = FSFactoryProducer.getFSFactory().getFile(dataFolder); @@ -56,35 +62,81 @@ public class DataHealthChecker { System.out.println("the input is not a data folder"); return; } + File[] subFiles = folder.listFiles(File::isDirectory); Arrays.sort(subFiles, (x, y) -> x.getName().compareTo(y.getName())); if (subFiles.length > 2) { System.out.println("[ERROR] there are some unknown folder under the data folder"); } + + int total = 0; + for (File seqOrUnSeq : subFiles) { + for (File sgFolder : seqOrUnSeq.listFiles(File::isDirectory)) { + for (File partitionFolder : sgFolder.listFiles(File::isDirectory)) { + //check the resource file first + total += partitionFolder.listFiles(x -> x.getName().endsWith(".tsfile")).length; + } + } + } + + FileWriter writer = null; + try { + writer = new FileWriter(progressFile); + System.out.println("The progress is recorded in " + new File(progressFile).getAbsolutePath()); + } catch (IOException e) { + e.printStackTrace(); + return; + } + + int current =0; for (File seqOrUnSeq : subFiles) { for(File sgFolder : seqOrUnSeq.listFiles(File::isDirectory)) { for (File partitionFolder : sgFolder.listFiles(File::isDirectory)) { //check the resource file first for (File tsfile : partitionFolder.listFiles(x -> x.getName().endsWith(".tsfile"))) { - //check resource - TsFileResource tsResource = new TsFileResource(SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath())); try { - tsResource.deSerialize(); - if (tsResource.getStartTimeMap().isEmpty()) { - System.out.print(String.format("[ERROR] Resource file %s has no devices", tsfile.getAbsolutePath()+".resource")); + writer.write(String.format("%d/%d\n", ++current, total)); + writer.flush(); + } catch (IOException e) { + e.printStackTrace(); + try { + writer.close(); + } catch (IOException ex) { } - } catch (Exception e) { - System.out.print(String.format("[ERROR] Resource file %s is broken: %s", tsfile.getAbsolutePath()+".resource", e.getMessage())); + return; + } + //check resource + if (SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath()+".resource").exists()) { + TsFileResource tsResource = new TsFileResource( + SystemFileFactory.INSTANCE.getFile(tsfile.getAbsolutePath())); + try { + tsResource.deSerialize(); + if (tsResource.getStartTimeMap().isEmpty()) { + System.out.print(String.format("[ERROR] Resource file %s has no devices", + tsfile.getAbsolutePath() + ".resource")); + } + } catch (Exception e) { + System.out.print(String.format("[ERROR] Resource file %s is broken: %s", + tsfile.getAbsolutePath() + ".resource", e.getMessage())); + } + } else { + System.out.print(String.format("[ERROR] TsFile %s has no resource file", + tsfile.getAbsolutePath() )); } //check data checkTsFile(tsfile); } for (File otherFile : partitionFolder.listFiles(x -> ! (x.getName().endsWith(".tsfile") || x.getName().endsWith(".tsfile.resource")))) { - System.out.print(String.format("[ERROR] Bad file %s ", otherFile.getAbsolutePath())); + System.out.println(String.format("[ERROR] Bad file %s ", otherFile.getAbsolutePath())); } } } } + try { + writer.close(); + } catch (IOException e) { + e.printStackTrace(); + } } private static void checkTsFile(File tsfile) {
