This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rewriteTool
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rewriteTool by this push:
new fe0112a542 add -f param in rewriteFileTool
fe0112a542 is described below
commit fe0112a5422806c28f86b2be3000582711bc8ed2
Author: Alima777 <[email protected]>
AuthorDate: Wed Jun 1 11:30:39 2022 +0800
add -f param in rewriteFileTool
---
rewriteBadFile/pom.xml | 62 ----------
rewriteFileTool/pom.xml | 64 +++++++++++
.../assembly/resources/sbin/rewrite-file-tool.sh | 2 +-
.../src/assembly/resources/sbin/validate-tsfile.sh | 0
.../src/assembly/rewriteFileTool.xml | 34 +++---
.../java/org/apache/iotdb/RewriteFileTool.java | 128 +++++++++++++--------
.../org/apache/iotdb/TsFileValidationTool.java | 0
7 files changed, 163 insertions(+), 127 deletions(-)
diff --git a/rewriteBadFile/pom.xml b/rewriteBadFile/pom.xml
deleted file mode 100644
index 77b9ead6b9..0000000000
--- a/rewriteBadFile/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <parent>
- <artifactId>iotdb-parent</artifactId>
- <groupId>org.apache.iotdb</groupId>
- <version>0.12.6-SNAPSHOT</version>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>rewriteBadFile</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-server</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>tsfile</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-session</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>${maven.assembly.version}</version>
- <executions>
- <!-- Package binaries-->
- <execution>
- <id>client-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
-
<descriptor>src/assembly/rewriteBadFile.xml</descriptor>
- </descriptors>
- <appendAssemblyId>false</appendAssemblyId>
- <archive>
- <manifest>
-
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
-
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
- </manifest>
- </archive>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
- </properties>
-</project>
diff --git a/rewriteFileTool/pom.xml b/rewriteFileTool/pom.xml
new file mode 100644
index 0000000000..ce6ea590ec
--- /dev/null
+++ b/rewriteFileTool/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.12.6-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>rewriteBadFile</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>tsfile</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>iotdb-session</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <executions>
+ <!-- Package binaries-->
+ <execution>
+ <id>client-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assembly/rewriteFileTool.xml</descriptor>
+ </descriptors>
+ <appendAssemblyId>false</appendAssemblyId>
+ <archive>
+ <manifest>
+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
+</project>
diff --git a/rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh
b/rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh
similarity index 96%
rename from rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh
rename to rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh
index 4f0d45c931..988ce60e69 100644
--- a/rewriteBadFile/src/assembly/resources/sbin/rewrite-badFile-tool.sh
+++ b/rewriteFileTool/src/assembly/resources/sbin/rewrite-file-tool.sh
@@ -42,7 +42,7 @@ for f in ${IOTDB_HOME}/lib/*.jar; do
CLASSPATH=${CLASSPATH}":"$f
done
-MAIN_CLASS=org.apache.iotdb.RewriteBadFileTool
+MAIN_CLASS=org.apache.iotdb.RewriteFileTool
"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
exit $?
diff --git a/rewriteBadFile/src/assembly/resources/sbin/validate-tsfile.sh
b/rewriteFileTool/src/assembly/resources/sbin/validate-tsfile.sh
similarity index 100%
rename from rewriteBadFile/src/assembly/resources/sbin/validate-tsfile.sh
rename to rewriteFileTool/src/assembly/resources/sbin/validate-tsfile.sh
diff --git a/rewriteBadFile/src/assembly/rewriteBadFile.xml
b/rewriteFileTool/src/assembly/rewriteFileTool.xml
similarity index 65%
rename from rewriteBadFile/src/assembly/rewriteBadFile.xml
rename to rewriteFileTool/src/assembly/rewriteFileTool.xml
index dfbd548135..bb8424af84 100644
--- a/rewriteBadFile/src/assembly/rewriteBadFile.xml
+++ b/rewriteFileTool/src/assembly/rewriteFileTool.xml
@@ -20,21 +20,21 @@
-->
<assembly>
- <id>rewriteBadFile</id>
- <formats>
- <format>dir</format>
- <format>zip</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>lib</outputDirectory>
- </dependencySet>
- </dependencySets>
- <fileSets>
- <fileSet>
- <directory>src/assembly/resources</directory>
- <outputDirectory>${file.separator}</outputDirectory>
- </fileSet>
- </fileSets>
+ <id>RewriteFileTool</id>
+ <formats>
+ <format>dir</format>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>src/assembly/resources</directory>
+ <outputDirectory>${file.separator}</outputDirectory>
+ </fileSet>
+ </fileSets>
</assembly>
diff --git
a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
similarity index 74%
rename from
rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
rename to rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
index e47c064534..5563dd6b5f 100644
--- a/rewriteBadFile/src/main/java/org/apache/iotdb/RewriteBadFileTool.java
+++ b/rewriteFileTool/src/main/java/org/apache/iotdb/RewriteFileTool.java
@@ -57,13 +57,20 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-public class RewriteBadFileTool {
+/**
+ * This tool reads tsFiles and rewrites it chunk by chunk. It constructs
tablet and invokes
+ * insertTablet() for every page in chunk, in case chunk is too large. `Move`
command is used to
+ * unload files in iotdb, and mods files are moved manually.
+ */
+public class RewriteFileTool {
// backup data dir path
- private static String backUpDirPath = "backup";
+ private static String backUpDirPath;
// validation file path
- private static String validationFilePath = "TsFile_validation_view.txt";
+ private static String validationFilePath;
+ // tsfile list path
+ private static String tsfileListPath;
// output file path
- private static String outputLogFilePath = "TsFile_rewrite_view.txt";
+ private static String outputLogFilePath;
private static final String HostIP = "localhost";
private static final String rpcPort = "6667";
@@ -75,8 +82,8 @@ public class RewriteBadFileTool {
private static PrintWriter pw;
/**
- * -b=[path of backUp directory] -v=[path of validation file] -o=[path of
output log] -m=[whether
- * move the file]
+ * -b=[path of backUp directory] -vf=[path of validation file]/-f=[path of
tsfile list] -o=[path
+ * of output log]
*/
public static void main(String[] args) throws IOException {
if (!checkArgs(args)) {
@@ -84,20 +91,22 @@ public class RewriteBadFileTool {
}
pw = new PrintWriter(new FileWriter(outputLogFilePath));
try {
- moveAndRewriteBadFile();
- } catch (IoTDBConnectionException
- | IOException
- | StatementExecutionException
- | InterruptedException e) {
+ if (validationFilePath != null) {
+ readValidationFile(validationFilePath);
+ }
+ if (tsfileListPath != null) {
+ readTsFileList(tsfileListPath);
+ }
+ } catch (Exception e) {
+ printBoth(e.getMessage());
e.printStackTrace();
} finally {
pw.close();
}
}
- public static void moveAndRewriteBadFile()
- throws IOException, IoTDBConnectionException,
StatementExecutionException,
- InterruptedException {
+ public static void readValidationFile(String validationFilePath)
+ throws IOException, IoTDBConnectionException {
Session session = new Session(HostIP, rpcPort, user, password);
session.open(false);
@@ -108,40 +117,57 @@ public class RewriteBadFileTool {
continue;
}
String badFilePath = line.replace("-- Find the bad file ", "");
+ unloadAndReWriteWrongTsFile(badFilePath, session);
+ }
+ bufferedReader.close();
+ session.close();
+ printBoth("Finish rewriting all bad files.");
+ }
+
+ public static void readTsFileList(String tsfileListPath)
+ throws IoTDBConnectionException, IOException {
+ Session session = new Session(HostIP, rpcPort, user, password);
+ session.open(false);
+
+ BufferedReader bufferedReader = new BufferedReader(new
FileReader(tsfileListPath));
+ String badFilePath;
+ while ((badFilePath = bufferedReader.readLine()) != null) {
+ unloadAndReWriteWrongTsFile(badFilePath, session);
+ }
+ bufferedReader.close();
+ session.close();
+ printBoth("Finish rewriting all bad files.");
+ }
- printBoth(String.format("Start moving %s to backup dir.", badFilePath));
- session.executeNonQueryStatement(String.format("move '%s' '%s'",
badFilePath, backUpDirPath));
- String[] dirs = badFilePath.split("/");
+ public static void unloadAndReWriteWrongTsFile(String filename, Session
session) {
+ printBoth(String.format("Start moving %s to backup dir.", filename));
+ try {
+ session.executeNonQueryStatement(String.format("move '%s' '%s'",
filename, backUpDirPath));
+ String[] dirs = filename.split("/");
String targetFilePath = backUpDirPath + File.separator +
dirs[dirs.length - 1];
File targetFile = new File(targetFilePath);
// move mods file
- File modsFile = new File(badFilePath + ModificationFile.FILE_SUFFIX);
+ File modsFile = new File(filename + ModificationFile.FILE_SUFFIX);
if (modsFile.exists()) {
fsFactory.moveFile(modsFile, new File(targetFilePath +
ModificationFile.FILE_SUFFIX));
}
- printBoth(String.format("Finish unloading %s.", badFilePath));
+ printBoth(String.format("Finish unloading %s.", filename));
+
// rewriteFile
- try {
- printBoth(String.format("Start rewriting %s to iotdb.", badFilePath));
- if (targetFile.exists()) {
- rewriteWrongTsFile(targetFilePath, session);
- targetFile.renameTo(new File(targetFilePath + "." + "finish"));
- } else {
- printBoth("---- Meet error in rewriting, " + targetFilePath + " does
not exist.");
- }
- } catch (Throwable e) {
- e.printStackTrace();
- printBoth("---- Meet error in rewriting " + targetFilePath + ", " +
e.getMessage());
+ printBoth(String.format("Start rewriting %s to iotdb.", filename));
+ if (targetFile.exists()) {
+ rewriteWrongTsFile(targetFilePath, session);
+ targetFile.renameTo(new File(targetFilePath + "." + "finish"));
+ } else {
+ printBoth("---- Meet error in rewriting, " + targetFilePath + " does
not exist.");
}
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ e.printStackTrace();
+ printBoth("---- Meet error in unloading " + filename + ", " +
e.getMessage());
}
- bufferedReader.close();
- session.close();
- printBoth("Finish rewriting all bad files.");
}
- public static void rewriteWrongTsFile(String filename, Session session)
- throws IoTDBConnectionException, StatementExecutionException,
IOException,
- IllegalPathException {
+ public static void rewriteWrongTsFile(String filename, Session session) {
// read mods file
List<Modification> modifications = null;
if (FSFactoryProducer.getFSFactory()
@@ -231,28 +257,36 @@ public class RewriteBadFileTool {
MetaMarker.handleUnexpectedMarker(marker);
}
}
+ } catch (IllegalPathException
+ | IOException
+ | IoTDBConnectionException
+ | StatementExecutionException e) {
+ printBoth("---- Meet error in rewriting " + filename + ", " +
e.getMessage());
+ printBoth(e.getMessage());
+ e.printStackTrace();
}
}
private static boolean checkArgs(String[] args) {
- if (args.length != 3) {
- System.out.println(
- "Param incorrect, -b=[path of backUp directory] -v=[path of
validation file] -o=[path of output file].");
- return false;
- }
+ String paramConfig =
+ "-b=[path of backUp directory] -vf=[path of validation file]/-f=[path
of tsfile list] -o=[path of output log]";
for (String arg : args) {
if (arg.startsWith("-b")) {
- backUpDirPath = arg.split("=")[1];
- } else if (arg.startsWith("-v")) {
- validationFilePath = arg.split("=")[1];
+ backUpDirPath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-vf")) {
+ validationFilePath = arg.substring(arg.indexOf('=') + 1);
+ } else if (arg.startsWith("-f")) {
+ tsfileListPath = arg.substring(arg.indexOf('=') + 1);
} else if (arg.startsWith("-o")) {
- outputLogFilePath = arg.split("=")[1];
+ outputLogFilePath = arg.substring(arg.indexOf('=') + 1);
} else {
- System.out.println(
- "Param incorrect, -b=[path of backUp directory] -v=[path of
validation file] -o=[path of output file].");
+ System.out.println("Param incorrect!" + paramConfig);
return false;
}
}
+ if (backUpDirPath == null || (validationFilePath == null && tsfileListPath
== null)) {
+ System.out.println("Param incorrect!" + paramConfig);
+ }
return true;
}
diff --git
a/rewriteBadFile/src/main/java/org/apache/iotdb/TsFileValidationTool.java
b/rewriteFileTool/src/main/java/org/apache/iotdb/TsFileValidationTool.java
similarity index 100%
rename from
rewriteBadFile/src/main/java/org/apache/iotdb/TsFileValidationTool.java
rename to
rewriteFileTool/src/main/java/org/apache/iotdb/TsFileValidationTool.java
