Luo Chen has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1796
Change subject: Allow change insert/delete mode for anti-matter LSM index
......................................................................
Allow change insert/delete mode for anti-matter LSM index
Currently, when we bulk load an lsm index, the leaf factory of the index
only allows to insert tuples, but not deleting tuples (i.e.,
inserting anti-matter tuples). However, the new change of creating
secondary index requires inserting both matter and anti-matter tuples.
This change exposes set insert/delete mode API for tuple writer factory
when we bulk load an lsm index component. This is achived basically
the same way as bulk loading an ExternalBTree.
Change-Id: I6665f56a5d2183697197298fa24824eeb827686a
---
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
D
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriterFactory.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
13 files changed, 231 insertions(+), 124 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/96/1796/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c641dc1..3f92a3c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -35,11 +35,9 @@
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import
org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
@@ -50,6 +48,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
import
org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
@@ -449,7 +448,7 @@
public class LSMTwoPCBTreeBulkLoader implements IIndexBulkLoader,
ITwoPCIndexBulkLoader {
private final ILSMDiskComponent component;
private final IIndexBulkLoader componentBulkLoader;
- private final ITreeIndexTupleWriterFactory frameTupleWriterFactory;
+ private final ILSMTreeTupleWriterFactory frameTupleWriterFactory;
private final boolean isTransaction;
@@ -463,8 +462,8 @@
component = createBulkLoadTarget();
}
- frameTupleWriterFactory =
- ((LSMBTreeDiskComponent)
component).getBTree().getLeafFrameFactory().getTupleWriterFactory();
+ frameTupleWriterFactory = (ILSMTreeTupleWriterFactory)
((LSMBTreeDiskComponent) component).getBTree()
+ .getLeafFrameFactory().getTupleWriterFactory();
componentBulkLoader =
createComponentBulkLoader(component, fillFactor,
verifyInput, numElementsHint, false, true);
@@ -499,9 +498,9 @@
// calling delete
@Override
public void delete(ITupleReference tuple) throws HyracksDataException {
- ((LSMBTreeRefrencingTupleWriterFactory)
frameTupleWriterFactory).setMode(IndexOperation.DELETE);
+ frameTupleWriterFactory.setDelete(true);
componentBulkLoader.add(tuple);
- ((LSMBTreeRefrencingTupleWriterFactory)
frameTupleWriterFactory).setMode(IndexOperation.INSERT);
+ frameTupleWriterFactory.setDelete(false);
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
deleted file mode 100644
index 24ae68c..0000000
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeRefrencingTupleWriterFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.btree.tuples;
-
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-
-/**
- * This is a TupleWriterFactory that can be used to reference the last writer
it created
- * and switch between actual and antimatter writer modes
- * @author alamouda
- *
- */
-public class LSMBTreeRefrencingTupleWriterFactory extends
TypeAwareTupleWriterFactory{
- private static final long serialVersionUID = 1L;
- private final ITypeTraits[] typeTraits;
- private final int numKeyFields;
- private boolean isDelete;
- private LSMBTreeTupleWriter createdTupleWriter;
-
- public LSMBTreeRefrencingTupleWriterFactory(ITypeTraits[] typeTraits, int
numKeyFields, boolean isDelete) {
- super(typeTraits);
- this.typeTraits = typeTraits;
- this.numKeyFields = numKeyFields;
- this.isDelete = isDelete;
- }
-
- @Override
- public ITreeIndexTupleWriter createTupleWriter() {
- createdTupleWriter = new LSMBTreeTupleWriter(typeTraits, numKeyFields,
isDelete);
- return createdTupleWriter;
- }
-
- public void setMode(IndexOperation op){
- if(op == IndexOperation.INSERT){
- this.isDelete = false;
- if(createdTupleWriter != null){
- this.createdTupleWriter.setAntimatter(false);
- }
- } else if(op == IndexOperation.DELETE){
- this.isDelete = true;
- if(createdTupleWriter != null){
- this.createdTupleWriter.setAntimatter(true);
- }
- }
- }
-
- public LSMBTreeTupleWriter getCreatedTupleWriter() {
- return createdTupleWriter;
- }
-}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
index 3c60f15..c083f17 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriter.java
@@ -23,20 +23,21 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
-public class LSMBTreeTupleWriter extends TypeAwareTupleWriter {
- private boolean isAntimatter;
+public class LSMBTreeTupleWriter extends TypeAwareTupleWriter implements
ILSMTreeTupleWriter {
+ private boolean isDelete;
private final int numKeyFields;
- public LSMBTreeTupleWriter(ITypeTraits[] typeTraits, int numKeyFields,
boolean isAntimatter) {
+ public LSMBTreeTupleWriter(ITypeTraits[] typeTraits, int numKeyFields,
boolean isDelete) {
super(typeTraits);
this.numKeyFields = numKeyFields;
- this.isAntimatter = isAntimatter;
+ this.isDelete = isDelete;
}
@Override
public int bytesRequired(ITupleReference tuple) {
- if (isAntimatter) {
+ if (isDelete) {
// Only requires space for the key fields.
return super.bytesRequired(tuple, 0, numKeyFields);
} else {
@@ -57,19 +58,19 @@
@Override
protected int getNullFlagsBytes(int numFields) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) numFields + 1.0) / 8.0);
+ return (int) Math.ceil((numFields + 1.0) / 8.0);
}
@Override
protected int getNullFlagsBytes(ITupleReference tuple) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0);
}
@Override
public int writeTuple(ITupleReference tuple, byte[] targetBuf, int
targetOff) {
int bytesWritten = -1;
- if (isAntimatter) {
+ if (isDelete) {
bytesWritten = super.writeTupleFields(tuple, 0, numKeyFields,
targetBuf, targetOff);
setAntimatterBit(targetBuf, targetOff);
} else {
@@ -83,8 +84,8 @@
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
- // Allow using the same writer for both delete and insert tuples
- public void setAntimatter(boolean isAntimatter) {
- this.isAntimatter = isAntimatter;
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
index 5ee2b81..f6147be 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/tuples/LSMBTreeTupleWriterFactory.java
@@ -20,14 +20,15 @@
package org.apache.hyracks.storage.am.lsm.btree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
-public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
+public class LSMBTreeTupleWriterFactory extends TypeAwareTupleWriterFactory
implements ILSMTreeTupleWriterFactory {
private static final long serialVersionUID = 1L;
- private final int numKeyFields;
- private final boolean isDelete;
+ protected final int numKeyFields;
+ protected boolean isDelete;
public LSMBTreeTupleWriterFactory(ITypeTraits[] typeTraits, int
numKeyFields, boolean isDelete) {
super(typeTraits);
@@ -36,7 +37,13 @@
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
+ public ILSMTreeTupleWriter createTupleWriter() {
return new LSMBTreeTupleWriter(typeTraits, numKeyFields, isDelete);
}
+
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
+
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 0e5fcdb..70d82a2 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -39,7 +39,6 @@
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
import
org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
import
org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
-import
org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
import
org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -52,6 +51,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import
org.apache.hyracks.storage.am.lsm.common.tuples.LSMTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -74,16 +74,21 @@
new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, true);
LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory =
new LSMBTreeCopyTupleWriterFactory(typeTraits,
cmpFactories.length);
+ LSMTreeRefrencingTupleWriterFactory bulkLoadTupleWriterFactory = new
LSMTreeRefrencingTupleWriterFactory(
+ new LSMBTreeTupleWriterFactory(filterTypeTraits,
cmpFactories.length, false));
+
ITreeIndexFrameFactory insertLeafFrameFactory = new
BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new
BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory deleteLeafFrameFactory = new
BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new
BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
+ ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new
BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory);
TreeIndexFactory<BTree> diskBTreeFactory =
new BTreeFactory(ioManager, diskBufferCache,
diskFileMapProvider, freePageManagerFactory,
interiorFrameFactory, copyTupleLeafFrameFactory,
cmpFactories, typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new
BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
- freePageManagerFactory, interiorFrameFactory,
insertLeafFrameFactory, cmpFactories, typeTraits.length);
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory =
+ new BTreeFactory(ioManager, diskBufferCache,
diskFileMapProvider, freePageManagerFactory,
+ interiorFrameFactory, bulkLoadLeafFrameFactory,
cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = needKeyDupCheck
? new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields) : null;
@@ -125,8 +130,8 @@
ITreeIndexFrameFactory deleteLeafFrameFactory = new
BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new
BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
// This is the tuple writer that can do both inserts and deletes
- LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory =
- new LSMBTreeRefrencingTupleWriterFactory(typeTraits,
cmpFactories.length, false);
+ LSMTreeRefrencingTupleWriterFactory referencingTupleWriterFactory =
new LSMTreeRefrencingTupleWriterFactory(
+ new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false));
// This is the leaf frame factory for transaction components since it
// can be used for both inserts and deletes
ITreeIndexFrameFactory dualLeafFrameFactory = new
BTreeNSMLeafFrameFactory(referencingTupleWriterFactory);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
new file mode 100644
index 0000000..d896075
--- /dev/null
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+
+/**
+ * This interface allows to write both insert (matter) and delete
(anti-matter) tuples
+ *
+ * @author luochen
+ *
+ */
+public interface ILSMTreeTupleWriter extends ITreeIndexTupleWriter {
+
+ void setDelete(boolean isDelete);
+
+}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriterFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriterFactory.java
new file mode 100644
index 0000000..c206980
--- /dev/null
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTreeTupleWriterFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+
+public interface ILSMTreeTupleWriterFactory extends
ITreeIndexTupleWriterFactory {
+
+ @Override
+ public ILSMTreeTupleWriter createTupleWriter();
+
+ public void setDelete(boolean isDelete);
+}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java
new file mode 100644
index 0000000..d527cda
--- /dev/null
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/tuples/LSMTreeRefrencingTupleWriterFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.tuples;
+
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
+
+/**
+ * This is a TupleWriterFactory that can be used to reference the last writer
it created
+ * and switch between actual and antimatter writer modes
+ *
+ */
+public class LSMTreeRefrencingTupleWriterFactory implements
ILSMTreeTupleWriterFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final ILSMTreeTupleWriterFactory factory;
+ private ILSMTreeTupleWriter createdTupleWriter;
+
+ public LSMTreeRefrencingTupleWriterFactory(ILSMTreeTupleWriterFactory
factory) {
+ this.factory = factory;
+ }
+
+ @Override
+ public ILSMTreeTupleWriter createTupleWriter() {
+ createdTupleWriter = factory.createTupleWriter();
+ return createdTupleWriter;
+ }
+
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.factory.setDelete(isDelete);
+ if (this.createdTupleWriter != null) {
+ this.createdTupleWriter.setDelete(isDelete);
+ }
+ }
+
+}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
index 3e586e6..3d32775 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriter.java
@@ -22,14 +22,15 @@
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
-public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter {
- private final boolean isAntimatter;
+public class LSMRTreeTupleWriter extends RTreeTypeAwareTupleWriter implements
ILSMTreeTupleWriter {
+ private boolean isDelete;
- public LSMRTreeTupleWriter(ITypeTraits[] typeTraits, boolean isAntimatter)
{
+ public LSMRTreeTupleWriter(ITypeTraits[] typeTraits, boolean isDelete) {
super(typeTraits);
- this.isAntimatter = isAntimatter;
+ this.isDelete = isDelete;
}
@Override
@@ -45,7 +46,7 @@
@Override
public int writeTuple(ITupleReference tuple, byte[] targetBuf, int
targetOff) {
int bytesWritten = super.writeTuple(tuple, targetBuf, targetOff);
- if (isAntimatter) {
+ if (isDelete) {
setAntimatterBit(targetBuf, targetOff);
}
return bytesWritten;
@@ -54,13 +55,13 @@
@Override
protected int getNullFlagsBytes(int numFields) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) numFields + 1.0) / 8.0);
+ return (int) Math.ceil((numFields + 1.0) / 8.0);
}
@Override
protected int getNullFlagsBytes(ITupleReference tuple) {
// +1.0 is for matter/antimatter bit.
- return (int) Math.ceil(((double) tuple.getFieldCount() + 1.0) / 8.0);
+ return (int) Math.ceil((tuple.getFieldCount() + 1.0) / 8.0);
}
protected void setAntimatterBit(byte[] targetBuf, int targetOff) {
@@ -68,4 +69,9 @@
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
+
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
index 0c21194..2394067 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactory.java
@@ -20,14 +20,15 @@
package org.apache.hyracks.storage.am.lsm.rtree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
-public class LSMRTreeTupleWriterFactory extends TypeAwareTupleWriterFactory {
+public class LSMRTreeTupleWriterFactory extends TypeAwareTupleWriterFactory
implements ILSMTreeTupleWriterFactory {
private static final long serialVersionUID = 1L;
private final ITypeTraits[] typeTraits;
- private final boolean isDelete;
+ private boolean isDelete;
public LSMRTreeTupleWriterFactory(ITypeTraits[] typeTraits, boolean
isDelete) {
super(typeTraits);
@@ -36,7 +37,12 @@
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
+ public ILSMTreeTupleWriter createTupleWriter() {
return new LSMRTreeTupleWriter(typeTraits, isDelete);
}
+
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
index c18823a..0ab3ec3 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterFactoryForPointMBR.java
@@ -20,26 +20,37 @@
package org.apache.hyracks.storage.am.lsm.rtree.tuples;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriterFactory;
-public class LSMRTreeTupleWriterFactoryForPointMBR extends
TypeAwareTupleWriterFactory {
+public class LSMRTreeTupleWriterFactoryForPointMBR extends
TypeAwareTupleWriterFactory
+ implements ILSMTreeTupleWriterFactory {
private static final long serialVersionUID = 1L;
private final int keyFieldCount;
private final int valueFieldCount;
private final boolean antimatterAware;
+ private boolean isDelete;
public LSMRTreeTupleWriterFactoryForPointMBR(ITypeTraits[] typeTraits, int
keyFieldCount, int valueFieldCount,
- boolean antimatterAware) {
+ boolean antimatterAware, boolean isDelete) {
super(typeTraits);
this.keyFieldCount = keyFieldCount;
this.valueFieldCount = valueFieldCount;
this.antimatterAware = antimatterAware;
+ this.isDelete = isDelete;
}
@Override
- public ITreeIndexTupleWriter createTupleWriter() {
- return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount,
valueFieldCount, antimatterAware);
+ public ILSMTreeTupleWriter createTupleWriter() {
+ return new LSMRTreeTupleWriterForPointMBR(typeTraits, keyFieldCount,
valueFieldCount, antimatterAware,
+ isDelete);
}
+
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
+
}
\ No newline at end of file
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
index 5de4d20..ca5a446 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleWriterForPointMBR.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMTreeTupleWriter;
import org.apache.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
@@ -40,16 +41,17 @@
* the stored point MBR [0.4, 0.3, 1] and generates a tuple reference which is
externally shown as [0.4, 0.3, 0.4, 0.3, 1].
*/
-public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter {
+public class LSMRTreeTupleWriterForPointMBR extends RTreeTypeAwareTupleWriter
implements ILSMTreeTupleWriter {
private final int inputKeyFieldCount; //double field count for mbr
secondary key of an input tuple
private final int valueFieldCount; //value(or payload or primary key)
field count (same for an input tuple and a stored tuple)
private final int inputTotalFieldCount; //total field count (key + value
fields) of an input tuple.
private final int storedKeyFieldCount; //double field count to be stored
for the mbr secondary key
private final int storedTotalFieldCount; //total field count (key + value
fields) of a stored tuple.
private final boolean antimatterAware;
+ private boolean isDelete;
public LSMRTreeTupleWriterForPointMBR(ITypeTraits[] typeTraits, int
keyFieldCount, int valueFieldCount,
- boolean antimatterAware) {
+ boolean antimatterAware, boolean isDelete) {
super(typeTraits);
this.inputKeyFieldCount = keyFieldCount;
this.valueFieldCount = valueFieldCount;
@@ -75,7 +77,8 @@
@Override
public ITreeIndexTupleReference createTupleReference() {
- return new LSMRTreeTupleReferenceForPointMBR(typeTraits,
inputKeyFieldCount, valueFieldCount, antimatterAware);
+ return new LSMRTreeTupleReferenceForPointMBR(typeTraits,
inputKeyFieldCount, valueFieldCount,
+ antimatterAware);
}
@Override
@@ -87,7 +90,7 @@
targetBuf[runner++] = (byte) 0;
}
- // write field slots for variable length fields which applies only to
value fields in RTree
+ // write field slots for variable length fields which applies only to
value fields in RTree
for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) {
if (!typeTraits[i].isFixedLength()) {
runner +=
VarLenIntEncoderDecoder.encode(tuple.getFieldLength(i), targetBuf, runner);
@@ -106,10 +109,16 @@
}
//set antimatter bit if necessary
+ //this is used when we flush an in-memory rtree into disk, and insert
anti-matter tuples from in-memory buddy btree into disk rtree
if (antimatterAware) {
if (tuple instanceof ILSMTreeTupleReference &&
((ILSMTreeTupleReference) tuple).isAntimatter()) {
setAntimatterBit(targetBuf, targetOff);
}
+ }
+
+ //this is used during creating secondary index operation, where we
explicitly insert some antimatter tuple
+ if (isDelete) {
+ setAntimatterBit(targetBuf, targetOff);
}
return runner - targetOff;
@@ -123,10 +132,12 @@
"writeTupleFields(ITupleReference, int, int, byte[], int) not
implemented for RTreeTypeAwareTupleWriterForPointMBR class.");
}
+ @Override
protected int getNullFlagsBytes(ITupleReference tuple) {
- return (int) Math.ceil((double) (storedTotalFieldCount +
(antimatterAware ? 1 : 0)) / 8.0);
+ return (int) Math.ceil((storedTotalFieldCount + (antimatterAware ? 1 :
0)) / 8.0);
}
+ @Override
protected int getFieldSlotsBytes(ITupleReference tuple) {
int fieldSlotBytes = 0;
for (int i = inputKeyFieldCount; i < inputTotalFieldCount; i++) {
@@ -146,4 +157,9 @@
// Set leftmost bit to 1.
targetBuf[targetOff] = (byte) (targetBuf[targetOff] | (1 << 7));
}
+
+ @Override
+ public void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
index d390be0..0a0d123 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/utils/LSMRTreeUtils.java
@@ -50,6 +50,7 @@
import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import
org.apache.hyracks.storage.am.lsm.common.tuples.LSMTreeRefrencingTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
@@ -92,7 +93,7 @@
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
if (isPointMBR) {
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, false);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, false, false);
} else {
rtreeLeafFrameTupleWriterFactory =
rtreeInteriorFrameTupleWriterFactory;
}
@@ -154,17 +155,22 @@
new LSMRTreeTupleWriterFactory(typeTraits, false);
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory;
ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory;
+ ITreeIndexTupleWriterFactory rtreeLeafFrameBulkLoadWriterFactory;
if (isPointMBR) {
int keyFieldCount = rtreeCmpFactories.length;
int valueFieldCount = btreeComparatorFactories.length -
keyFieldCount;
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, true);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, true, false);
rtreeLeafFrameCopyTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, true);
-
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, true, false);
+ rtreeLeafFrameBulkLoadWriterFactory = new
LSMTreeRefrencingTupleWriterFactory(
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, true, false));
} else {
rtreeLeafFrameTupleWriterFactory = new
LSMRTreeTupleWriterFactory(typeTraits, false);
rtreeLeafFrameCopyTupleWriterFactory = new
LSMRTreeCopyTupleWriterFactory(typeTraits);
+ rtreeLeafFrameBulkLoadWriterFactory =
+ new LSMTreeRefrencingTupleWriterFactory(new
LSMRTreeTupleWriterFactory(typeTraits, false));
+
}
LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new
LSMRTreeTupleWriterFactory(typeTraits, true);
@@ -172,6 +178,8 @@
rtreeInteriorFrameTupleWriterFactory, valueProviderFactories,
rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory rtreeLeafFrameFactory = new
RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory,
valueProviderFactories, rtreePolicyType, isPointMBR);
+ ITreeIndexFrameFactory rtreeLeafFrameBulkLoadFactory = new
RTreeNSMLeafFrameFactory(
+ rtreeLeafFrameBulkLoadWriterFactory, valueProviderFactories,
rtreePolicyType, isPointMBR);
ITreeIndexFrameFactory btreeInteriorFrameFactory = new
BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
ITreeIndexFrameFactory btreeLeafFrameFactory = new
BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
@@ -183,7 +191,7 @@
typeTraits.length, isPointMBR);
TreeIndexFactory<RTree> bulkLoadRTreeFactory = new
RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
- freePageManagerFactory, rtreeInteriorFrameFactory,
rtreeLeafFrameFactory, rtreeCmpFactories,
+ freePageManagerFactory, rtreeInteriorFrameFactory,
rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories,
typeTraits.length, isPointMBR);
// The first field is for the sorted curve (e.g. Hilbert curve), and
the
@@ -242,7 +250,7 @@
ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
if (isPointMBR) {
rtreeLeafFrameTupleWriterFactory =
- new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, false);
+ new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits,
keyFieldCount, valueFieldCount, false, false);
} else {
rtreeLeafFrameTupleWriterFactory =
rtreeInteriorFrameTupleWriterFactory;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1796
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6665f56a5d2183697197298fa24824eeb827686a
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Luo Chen <[email protected]>
