This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch add_checking_tool
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/add_checking_tool by this push:
new cef27a0 add test
cef27a0 is described below
commit cef27a02f5c35852e8472f1cfc0a03fa212a7fcf
Author: jt <[email protected]>
AuthorDate: Sun Mar 3 12:15:08 2019 +0800
add test
---
.../java/org/apache/iotdb/db/tools/WalChecker.java | 26 ++--
.../db/writelog/transfer/PhysicalPlanCodec.java | 4 +-
.../writelog/transfer/PhysicalPlanLogTransfer.java | 2 +-
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 133 +++++++++++++++++++++
.../apache/iotdb/db/writelog/PerformanceTest.java | 2 +-
.../iotdb/db/writelog/io/LogWriterReaderTest.java | 2 +-
6 files changed, 156 insertions(+), 13 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
index ed96f25..5ae9025 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java
@@ -22,9 +22,9 @@ package org.apache.iotdb.db.tools;
import static
org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode.WAL_FILE_NAME;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.iotdb.db.exception.SysCheckException;
import org.apache.iotdb.db.writelog.io.RAFLogReader;
@@ -47,7 +47,12 @@ public class WalChecker {
this.walFolder = walFolder;
}
- public void doCheck() throws SysCheckException {
+ /**
+ * check the root wal dir and find the damaged files
+ * @return a list of damaged files.
+ * @throws SysCheckException if the root wal dir does not exist.
+ */
+ public List<File> doCheck() throws SysCheckException {
File walFolderFile = new File(walFolder);
if(!walFolderFile.exists() || !walFolderFile.isDirectory()) {
throw new SysCheckException(String.format("%s is not a directory",
walFolder));
@@ -56,10 +61,10 @@ public class WalChecker {
File[] storageWalFolders = walFolderFile.listFiles();
if (storageWalFolders == null || storageWalFolders.length == 0) {
LOGGER.info("No sub-directories under the given directory, check ends");
- return;
+ return Collections.emptyList();
}
- List<File> failedFile = new ArrayList<>();
+ List<File> failedFiles = new ArrayList<>();
for (int dirIndex = 0; dirIndex < storageWalFolders.length; dirIndex++) {
File storageWalFolder = storageWalFolders[dirIndex];
LOGGER.debug("Checking the No.{} directory {}", dirIndex,
storageWalFolder.getName());
@@ -76,7 +81,7 @@ public class WalChecker {
logReader.next();
}
} catch (IOException e) {
- failedFile.add(walFile);
+ failedFiles.add(walFile);
LOGGER.error("{} fails the check because", walFile.getAbsoluteFile(),
e);
} finally {
if( logReader != null) {
@@ -84,11 +89,15 @@ public class WalChecker {
}
}
}
+ return failedFiles;
+ }
- if (failedFile.isEmpty()) {
+ // a temporary method which should be in the integrated self-check module in
the future
+ public static void report(List<File> failedFiles) {
+ if (failedFiles.isEmpty()) {
LOGGER.info("Check finished. There is no damaged file");
} else {
- LOGGER.error("There are {} files failed the check. They are {}",
failedFile.size(), failedFile);
+ LOGGER.error("There are {} failed files. They are {}",
failedFiles.size(), failedFiles);
}
}
@@ -103,6 +112,7 @@ public class WalChecker {
}
WalChecker checker = new WalChecker(args[0]);
- checker.doCheck();
+ List<File> files = checker.doCheck();
+ report(files);
}
}
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanCodec.java
b/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanCodec.java
index 54ba9e5..b611a4a 100644
---
a/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanCodec.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanCodec.java
@@ -57,9 +57,9 @@ public enum PhysicalPlanCodec {
this.codec = codec;
}
- public static PhysicalPlanCodec fromOpcode(int opcode) {
+ public static PhysicalPlanCodec fromOpcode(int opcode) throws IOException {
if (!codecMap.containsKey(opcode)) {
- throw new UnsupportedOperationException(
+ throw new IOException(
"SystemLogOperator [" + opcode + "] is not supported. ");
}
return codecMap.get(opcode);
diff --git
a/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanLogTransfer.java
b/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanLogTransfer.java
index 4fb4792..76a6e46 100644
---
a/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanLogTransfer.java
+++
b/iotdb/src/main/java/org/apache/iotdb/db/writelog/transfer/PhysicalPlanLogTransfer.java
@@ -27,7 +27,7 @@ public class PhysicalPlanLogTransfer {
private PhysicalPlanLogTransfer(){}
- public static byte[] operatorToLog(PhysicalPlan plan) throws
WALOverSizedException {
+ public static byte[] operatorToLog(PhysicalPlan plan) throws IOException {
Codec<PhysicalPlan> codec;
switch (plan.getOperatorType()) {
case INSERT:
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
new file mode 100644
index 0000000..0c3ca29
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import static
org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode.WAL_FILE_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.exception.SysCheckException;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.writelog.io.LogWriter;
+import org.apache.iotdb.db.writelog.transfer.PhysicalPlanLogTransfer;
+import org.junit.Test;
+
+public class WalCheckerTest {
+
+ @Test
+ public void testNoDir() {
+ WalChecker checker = new WalChecker("no such dir");
+ boolean caught = false;
+ try {
+ checker.doCheck();
+ } catch (SysCheckException e) {
+ caught = true;
+ }
+ assertTrue(caught);
+ }
+
+ @Test
+ public void testEmpty() throws IOException, SysCheckException {
+ File tempRoot = new File("root");
+ tempRoot.mkdir();
+
+ try {
+ WalChecker checker = new WalChecker(tempRoot.getAbsolutePath());
+ assertTrue(checker.doCheck().isEmpty());
+ } finally {
+ FileUtils.deleteDirectory(tempRoot);
+ }
+ }
+
+ @Test
+ public void testNormalCheck() throws IOException, SysCheckException {
+ File tempRoot = new File("root");
+ tempRoot.mkdir();
+
+ try {
+ for (int i = 0; i < 5; i++) {
+ File subDir = new File(tempRoot, "storage_group" + i);
+ subDir.mkdir();
+ LogWriter logWriter = new LogWriter(subDir.getPath() + File.separator
+ + WAL_FILE_NAME);
+
+ List<byte[]> binaryPlans = new ArrayList<>();
+ String deviceId = "device1";
+ List<String> measurements = Arrays.asList("s1", "s2", "s3");
+ List<String> values = Arrays.asList("5", "6", "7");
+ for (int j = 0; j < 10; j++) {
+ binaryPlans.add(PhysicalPlanLogTransfer
+ .operatorToLog(new InsertPlan(deviceId, j, measurements,
values)));
+ }
+ logWriter.write(binaryPlans);
+ logWriter.force();
+
+ logWriter.close();
+ }
+
+ WalChecker checker = new WalChecker(tempRoot.getAbsolutePath());
+ assertTrue(checker.doCheck().isEmpty());
+ } finally {
+ FileUtils.deleteDirectory(tempRoot);
+ }
+ }
+
+ @Test
+ public void testAbnormalCheck() throws IOException, SysCheckException {
+ File tempRoot = new File("root");
+ tempRoot.mkdir();
+
+ try {
+ for (int i = 0; i < 5; i++) {
+ File subDir = new File(tempRoot, "storage_group" + i);
+ subDir.mkdir();
+ LogWriter logWriter = new LogWriter(subDir.getPath() + File.separator
+ + WAL_FILE_NAME);
+
+ List<byte[]> binaryPlans = new ArrayList<>();
+ String deviceId = "device1";
+ List<String> measurements = Arrays.asList("s1", "s2", "s3");
+ List<String> values = Arrays.asList("5", "6", "7");
+ for (int j = 0; j < 10; j++) {
+ binaryPlans.add(PhysicalPlanLogTransfer
+ .operatorToLog(new InsertPlan(deviceId, j, measurements,
values)));
+ }
+ if (i > 2) {
+ binaryPlans.add("not a wal".getBytes());
+ }
+ logWriter.write(binaryPlans);
+ logWriter.force();
+
+ logWriter.close();
+ }
+
+ WalChecker checker = new WalChecker(tempRoot.getAbsolutePath());
+ assertEquals(2, checker.doCheck().size());
+ } finally {
+ FileUtils.deleteDirectory(tempRoot);
+ }
+ }
+}
diff --git
a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
index e3b81c4..ce0af29 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
@@ -216,7 +216,7 @@ public class PerformanceTest {
}
@Test
- public void SQLEncodingComparisonTest() throws WALOverSizedException {
+ public void SQLEncodingComparisonTest() throws IOException {
String sql = "INSERT INTO root.logTestDevice(time,s1,s2,s3,s4) "
+ "VALUES (100,1.0,15,\"str\",false)";
InsertPlan bwInsertPlan = new InsertPlan(1, "root.logTestDevice", 100,
diff --git
a/iotdb/src/test/java/org/apache/iotdb/db/writelog/io/LogWriterReaderTest.java
b/iotdb/src/test/java/org/apache/iotdb/db/writelog/io/LogWriterReaderTest.java
index 4ba32ef..d68478e 100644
---
a/iotdb/src/test/java/org/apache/iotdb/db/writelog/io/LogWriterReaderTest.java
+++
b/iotdb/src/test/java/org/apache/iotdb/db/writelog/io/LogWriterReaderTest.java
@@ -42,7 +42,7 @@ public class LogWriterReaderTest {
List<PhysicalPlan> plans = new ArrayList<>();
@Before
- public void prepare() throws WALOverSizedException {
+ public void prepare() throws IOException {
if (new File(filePath).exists()) {
new File(filePath).delete();
}
