This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch improve_wal
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/improve_wal by this push:
new 59e6bca reduce index size
59e6bca is described below
commit 59e6bcacb7d3246468f39533c5e8f0a96b53771a
Author: jt <[email protected]>
AuthorDate: Fri Oct 16 09:42:09 2020 +0800
reduce index size
---
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 2 +-
.../writelog/manager/MultiFileLogNodeManager.java | 3 +-
.../db/writelog/node/DifferentialWriteLogNode.java | 44 ++++++++++++++++------
3 files changed, 34 insertions(+), 15 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index 499898d..01c8214 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -305,7 +305,7 @@ public abstract class PhysicalPlan {
public static PhysicalPlan create(ByteBuffer buffer,
Queue<PhysicalPlan> planWindow) throws IOException,
IllegalPathException {
- int baseIndex = buffer.getInt();
+ short baseIndex = buffer.getShort();
int typeNum = buffer.get();
if (typeNum >= PhysicalPlanType.values().length) {
throw new IOException("unrecognized log type " + typeNum);
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java
b/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java
index 83e8687..b96b227 100644
---
a/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java
+++
b/server/src/main/java/org/apache/iotdb/db/writelog/manager/MultiFileLogNodeManager.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.db.writelog.node.DifferentialWriteLogNode;
-import org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +46,7 @@ public class MultiFileLogNodeManager implements
WriteLogNodeManager, IService {
private ScheduledExecutorService executorService;
private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private final void forceTask() {
+ private void forceTask() {
if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
logger.warn("system mode is read-only, the force flush WAL task is
stopped");
return;
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/node/DifferentialWriteLogNode.java
b/server/src/main/java/org/apache/iotdb/db/writelog/node/DifferentialWriteLogNode.java
index 7a70c31..fc4f9ee 100644
---
a/server/src/main/java/org/apache/iotdb/db/writelog/node/DifferentialWriteLogNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/writelog/node/DifferentialWriteLogNode.java
@@ -28,7 +28,6 @@ import java.util.Queue;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.db.writelog.io.DifferentialSingleFileLogReader;
import org.apache.iotdb.db.writelog.io.ILogReader;
@@ -37,10 +36,17 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * DifferentialWriteLogNode maintains a reduced plan window (reduced means the
plans do not
+ * contain unnecessary fields for differentiation, e.g., values and timestamps
in InsertPlan).
+ * When a plan is to be serialized, if a similar log can be found with in the
window, identical
+ * fields (like deviceId, measurementIds, dataTypes) will be referenced from
the similar log and
+ * remove the necessity of serializing them more than once.
+ */
public class DifferentialWriteLogNode extends ExclusiveWriteLogNode {
private static final Logger logger =
LoggerFactory.getLogger(DifferentialWriteLogNode.class);
- // TODO: make WINDOW_LENGTH a config
+ // we can only use a linear search now, so the window length should not be
too large
public static final int WINDOW_LENGTH = 2000;
private Queue<PhysicalPlan> planWindow;
@@ -57,7 +63,7 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
@Override
void putLog(PhysicalPlan plan) {
logBuffer.mark();
- Pair<PhysicalPlan, Integer> similarPlanIndex = findSimilarPlan(plan);
+ Pair<PhysicalPlan, Short> similarPlanIndex = findSimilarPlan(plan);
try {
serialize(plan, similarPlanIndex);
} catch (BufferOverflowException e) {
@@ -76,7 +82,7 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
planWindow.clear();
}
- private void serialize(PhysicalPlan plan, Pair<PhysicalPlan, Integer>
similarPlanIndex) {
+ private void serialize(PhysicalPlan plan, Pair<PhysicalPlan, Short>
similarPlanIndex) {
if (similarPlanIndex == null) {
serializeNonDifferentially(plan);
} else {
@@ -85,12 +91,12 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
}
private void serializeNonDifferentially(PhysicalPlan plan) {
- logBuffer.putInt(-1);
+ logBuffer.putShort((short) -1);
plan.serialize(logBuffer);
}
private void serializeDifferentially(PhysicalPlan plan,
- Pair<PhysicalPlan, Integer> similarPlanIndex) {
+ Pair<PhysicalPlan, Short> similarPlanIndex) {
if (plan instanceof InsertPlan) {
serializeDifferentially(((InsertPlan) plan), ((InsertPlan)
similarPlanIndex.left),
similarPlanIndex.right);
@@ -99,13 +105,14 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
}
}
- private void serializeDifferentially(InsertPlan plan, InsertPlan base, int
index) {
- logBuffer.putInt(index);
+ private void serializeDifferentially(InsertPlan plan, InsertPlan base, short
index) {
+ logBuffer.putShort(index);
plan.serialize(logBuffer, base);
}
- private Pair<PhysicalPlan, Integer> findSimilarPlan(PhysicalPlan plan) {
- int index = -1;
+ private Pair<PhysicalPlan, Short> findSimilarPlan(PhysicalPlan plan) {
+ short index = -1;
+ // linear search, is there any way better?
for (PhysicalPlan next : planWindow) {
index++;
if (isPlanSimilarEnough(plan, next)) {
@@ -119,13 +126,16 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
if (!planA.getClass().equals(planB.getClass())) {
return false;
}
- if (planA instanceof InsertTabletPlan && planB instanceof
InsertTabletPlan) {
- return isPlanSimilarEnough(((InsertTabletPlan) planA),
((InsertTabletPlan) planB));
+ // we only do differentiation for InsertPlans now, as they are the majority
+ if (planA instanceof InsertPlan && planB instanceof InsertPlan) {
+ return isPlanSimilarEnough(((InsertPlan) planA), ((InsertPlan) planB));
}
return false;
}
private boolean isPlanSimilarEnough(InsertPlan planA, InsertPlan planB) {
+ // data types are also compared because the timeseries may be deleted and
recreated with
+ // different types between two insertions
return planA.getDeviceId().equals(planB.getDeviceId()) &&
Arrays.equals(planA.getMeasurements(), planB.getMeasurements()) &&
Arrays.equals(planA.getDataTypes(), planB.getDataTypes());
@@ -138,4 +148,14 @@ public class DifferentialWriteLogNode extends
ExclusiveWriteLogNode {
Comparator.comparingInt(f ->
Integer.parseInt(f.getName().replace(WAL_FILE_NAME, ""))));
return new MultiFileLogReader(logFiles,
DifferentialSingleFileLogReader::new);
}
+
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
}
