Refactor setup utilities for indexes into the index package. Leave redirection code so that the operations are still easy to find.
Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/b659de12 Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/b659de12 Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/b659de12 Branch: refs/heads/hadoop-rdf Commit: b659de12c952d52a38b93649a016bbece108ec74 Parents: a6b5b55 Author: Andy Seaborne <[email protected]> Authored: Fri Jan 9 15:56:47 2015 +0000 Committer: Andy Seaborne <[email protected]> Committed: Fri Jan 9 15:56:47 2015 +0000 ---------------------------------------------------------------------- .../com/hp/hpl/jena/tdb/index/SetupIndex.java | 173 +++++++++++++++++++ .../java/com/hp/hpl/jena/tdb/sys/SetupTDB.java | 162 +++++++---------- jena-tdb/src/main/java/tdb/CmdRewriteIndex.java | 4 +- .../index/bplustree/TestBPlusTreeRewriter.java | 4 +- 4 files changed, 237 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/b659de12/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/SetupIndex.java ---------------------------------------------------------------------- diff --git a/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/SetupIndex.java b/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/SetupIndex.java new file mode 100644 index 0000000..a60d491 --- /dev/null +++ b/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/index/SetupIndex.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hp.hpl.jena.tdb.index; + +import com.hp.hpl.jena.tdb.base.block.BlockMgr ; +import com.hp.hpl.jena.tdb.base.block.BlockMgrFactory ; +import com.hp.hpl.jena.tdb.base.file.FileSet ; +import com.hp.hpl.jena.tdb.base.file.Location ; +import com.hp.hpl.jena.tdb.base.record.RecordFactory ; +import com.hp.hpl.jena.tdb.index.bplustree.BPlusTree ; +import com.hp.hpl.jena.tdb.index.bplustree.BPlusTreeParams ; +import com.hp.hpl.jena.tdb.sys.Names ; +import com.hp.hpl.jena.tdb.sys.SystemTDB ; + +public class SetupIndex { + + /** Create a B+Tree using defaults */ + public static RangeIndex createBPTree(FileSet fileset, + RecordFactory factory) + { + int readCacheSize = SystemTDB.BlockReadCacheSize ; + int writeCacheSize = SystemTDB.BlockWriteCacheSize ; + int blockSize = SystemTDB.BlockSize ; + if ( fileset.isMem() ) + { + readCacheSize = 0 ; + writeCacheSize = 0 ; + blockSize = SystemTDB.BlockSizeTest ; + } + + return createBPTreeByBlockSize(fileset, blockSize, readCacheSize, writeCacheSize, factory) ; + } + + /** Create a B+Tree by BlockSize */ + public static RangeIndex createBPTreeByBlockSize(FileSet fileset, + int blockSize, + int readCacheSize, int writeCacheSize, + RecordFactory factory) + { + return createBPTree(fileset, -1, blockSize, readCacheSize, writeCacheSize, factory) ; + } + + /** Create a B+Tree by Order */ + public static RangeIndex createBPTreeByOrder(FileSet fileset, + int order, + int readCacheSize, int writeCacheSize, + RecordFactory factory) + { + return createBPTree(fileset, order, -1, readCacheSize, writeCacheSize, factory) ; + } + + /** Knowing all the parameters, create a B+Tree */ + public static BPlusTree createBPTree(FileSet fileset, int order, int blockSize, + int readCacheSize, int writeCacheSize, + RecordFactory factory) + { + // ---- Checking + if (blockSize < 0 && order < 0) throw new IllegalArgumentException("Neither blocksize nor order specified") ; + if (blockSize >= 0 && order < 0) order = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; + if (blockSize >= 0 && order >= 0) + { + int order2 = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; + if (order != order2) throw new IllegalArgumentException("Wrong order (" + order + "), calculated = " + + order2) ; + } + + // Iffy - does not allow for slop. + if (blockSize < 0 && order >= 0) + { + // Only in-memory. + blockSize = BPlusTreeParams.calcBlockSize(order, factory) ; + } + + BPlusTreeParams params = new BPlusTreeParams(order, factory) ; + BlockMgr blkMgrNodes = BlockMgrFactory.create(fileset, Names.bptExtTree, blockSize, readCacheSize, writeCacheSize) ; + BlockMgr blkMgrRecords = BlockMgrFactory.create(fileset, Names.bptExtRecords, blockSize, readCacheSize, writeCacheSize) ; + return BPlusTree.create(params, blkMgrNodes, blkMgrRecords) ; + } + + public static Index makeIndex(Location location, String indexName, + int blkSize, + int dftKeyLength, int dftValueLength, + int readCacheSize,int writeCacheSize) + { + return makeRangeIndex(location, indexName, blkSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; + } + + public static RangeIndex makeRangeIndex(Location location, String indexName, + int blkSize, + int dftKeyLength, int dftValueLength, + int readCacheSize,int writeCacheSize) + { + FileSet fs = new FileSet(location, indexName) ; + RangeIndex rIndex = makeBPlusTree(fs, blkSize, readCacheSize, writeCacheSize, dftKeyLength, dftValueLength) ; + return rIndex ; + } + + public static RangeIndex makeBPlusTree(FileSet fs, int blkSize, + int readCacheSize, int writeCacheSize, + int dftKeyLength, int dftValueLength) + { + RecordFactory recordFactory = makeRecordFactory(dftKeyLength, dftValueLength) ; + int order = BPlusTreeParams.calcOrder(blkSize, recordFactory.recordLength()) ; + RangeIndex rIndex = createBPTree(fs, order, blkSize, readCacheSize, writeCacheSize, recordFactory) ; + return rIndex ; + } + + public static RecordFactory makeRecordFactory(int keyLen, int valueLen) + { + return new RecordFactory(keyLen, valueLen) ; + } + // + // /** Make a NodeTable without cache and inline wrappers */ + // public static NodeTable makeNodeTableBase(Location location, String indexNode2Id, String indexId2Node) + // { + // if (location.isMem()) + // return NodeTableFactory.createMem() ; + // + // // -- make id to node mapping -- Names.indexId2Node + // FileSet fsIdToNode = new FileSet(location, indexId2Node) ; + // + // ObjectFile stringFile = makeObjectFile(fsIdToNode) ; + // + // // -- make node to id mapping -- Names.indexNode2Id + // // Make index of id to node (data table) + // + // // No caching at the index level - we use the internal caches of the node table. + // Index nodeToId = makeIndex(location, indexNode2Id, LenNodeHash, SizeOfNodeId, -1 ,-1) ; + // + // // -- Make the node table using the components established above. + // NodeTable nodeTable = new NodeTableNative(nodeToId, stringFile) ; + // return nodeTable ; + // } + // + // /** Make a NodeTable with cache and inline wrappers */ + // public static NodeTable makeNodeTable(Location location) + // { + // return makeNodeTable(location, + // Names.indexNode2Id, SystemTDB.Node2NodeIdCacheSize, + // Names.indexId2Node, SystemTDB.NodeId2NodeCacheSize, + // SystemTDB.NodeMissCacheSize) ; + // } + // + // /** Make a NodeTable with cache and inline wrappers */ + // public static NodeTable makeNodeTable(Location location, + // String indexNode2Id, int nodeToIdCacheSize, + // String indexId2Node, int idToNodeCacheSize, + // int nodeMissCacheSize) + // { + // NodeTable nodeTable = makeNodeTableBase(location, indexNode2Id, indexId2Node) ; + // nodeTable = NodeTableCache.create(nodeTable, nodeToIdCacheSize, idToNodeCacheSize, nodeMissCacheSize) ; + // nodeTable = NodeTableInline.create(nodeTable) ; + // return nodeTable ; + // } + // +} + http://git-wip-us.apache.org/repos/asf/jena/blob/b659de12/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/sys/SetupTDB.java ---------------------------------------------------------------------- diff --git a/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/sys/SetupTDB.java b/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/sys/SetupTDB.java index 72268de..0c160c1 100644 --- a/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/sys/SetupTDB.java +++ b/jena-tdb/src/main/java/com/hp/hpl/jena/tdb/sys/SetupTDB.java @@ -23,8 +23,6 @@ import org.slf4j.Logger ; import com.hp.hpl.jena.tdb.TDB ; import com.hp.hpl.jena.tdb.TDBException ; -import com.hp.hpl.jena.tdb.base.block.BlockMgr ; -import com.hp.hpl.jena.tdb.base.block.BlockMgrFactory ; import com.hp.hpl.jena.tdb.base.file.FileFactory ; import com.hp.hpl.jena.tdb.base.file.FileSet ; import com.hp.hpl.jena.tdb.base.file.Location ; @@ -32,8 +30,8 @@ import com.hp.hpl.jena.tdb.base.objectfile.ObjectFile ; import com.hp.hpl.jena.tdb.base.record.RecordFactory ; import com.hp.hpl.jena.tdb.index.Index ; import com.hp.hpl.jena.tdb.index.RangeIndex ; +import com.hp.hpl.jena.tdb.index.SetupIndex ; import com.hp.hpl.jena.tdb.index.bplustree.BPlusTree ; -import com.hp.hpl.jena.tdb.index.bplustree.BPlusTreeParams ; import com.hp.hpl.jena.tdb.setup.StoreParams ; import com.hp.hpl.jena.tdb.store.NodeId ; import com.hp.hpl.jena.tdb.store.tupletable.TupleIndex ; @@ -143,7 +141,7 @@ public class SetupTDB int writeCacheSize = params.getBlockWriteCacheSize() ; // Value part is null (zero length) - RangeIndex rIndex = makeRangeIndex(location, indexName, params.getBlockSize(), keyLength, 0, readCacheSize, writeCacheSize) ; + RangeIndex rIndex = SetupIndex.makeRangeIndex(location, indexName, params.getBlockSize(), keyLength, 0, readCacheSize, writeCacheSize) ; TupleIndex tupleIndex = new TupleIndexRecord(primary.length(), new ColumnMap(primary, indexOrder), indexOrder, rIndex.getRecordFactory(), rIndex) ; return tupleIndex ; } @@ -154,7 +152,7 @@ public class SetupTDB int dftKeyLength, int dftValueLength, int readCacheSize,int writeCacheSize) { - return makeRangeIndex(location, indexName, blkSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; + return SetupIndex.makeIndex(location, indexName, blkSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; } public static RangeIndex makeRangeIndex(Location location, String indexName, @@ -162,69 +160,64 @@ public class SetupTDB int dftKeyLength, int dftValueLength, int readCacheSize,int writeCacheSize) { - FileSet fs = new FileSet(location, indexName) ; - RangeIndex rIndex = makeBPlusTree(fs, blkSize, readCacheSize, writeCacheSize, dftKeyLength, dftValueLength) ; - return rIndex ; + return SetupIndex.makeRangeIndex(location, indexName, blkSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; } public static RangeIndex makeBPlusTree(FileSet fs, int blkSize, int readCacheSize, int writeCacheSize, int dftKeyLength, int dftValueLength) { - RecordFactory recordFactory = makeRecordFactory(dftKeyLength, dftValueLength) ; - int order = BPlusTreeParams.calcOrder(blkSize, recordFactory.recordLength()) ; - RangeIndex rIndex = createBPTree(fs, order, blkSize, readCacheSize, writeCacheSize, recordFactory) ; - return rIndex ; + return SetupIndex.makeBPlusTree(fs, blkSize, readCacheSize, writeCacheSize, dftKeyLength, dftValueLength) ; } public static RecordFactory makeRecordFactory(int keyLen, int valueLen) - { - return new RecordFactory(keyLen, valueLen) ; - } -// -// /** Make a NodeTable without cache and inline wrappers */ -// public static NodeTable makeNodeTableBase(Location location, String indexNode2Id, String indexId2Node) -// { -// if (location.isMem()) -// return NodeTableFactory.createMem() ; -// -// // -- make id to node mapping -- Names.indexId2Node -// FileSet fsIdToNode = new FileSet(location, indexId2Node) ; -// -// ObjectFile stringFile = makeObjectFile(fsIdToNode) ; -// -// // -- make node to id mapping -- Names.indexNode2Id -// // Make index of id to node (data table) -// -// // No caching at the index level - we use the internal caches of the node table. -// Index nodeToId = makeIndex(location, indexNode2Id, LenNodeHash, SizeOfNodeId, -1 ,-1) ; -// -// // -- Make the node table using the components established above. -// NodeTable nodeTable = new NodeTableNative(nodeToId, stringFile) ; -// return nodeTable ; -// } -// -// /** Make a NodeTable with cache and inline wrappers */ -// public static NodeTable makeNodeTable(Location location) -// { -// return makeNodeTable(location, -// Names.indexNode2Id, SystemTDB.Node2NodeIdCacheSize, -// Names.indexId2Node, SystemTDB.NodeId2NodeCacheSize, -// SystemTDB.NodeMissCacheSize) ; -// } -// -// /** Make a NodeTable with cache and inline wrappers */ -// public static NodeTable makeNodeTable(Location location, -// String indexNode2Id, int nodeToIdCacheSize, -// String indexId2Node, int idToNodeCacheSize, -// int nodeMissCacheSize) -// { -// NodeTable nodeTable = makeNodeTableBase(location, indexNode2Id, indexId2Node) ; -// nodeTable = NodeTableCache.create(nodeTable, nodeToIdCacheSize, idToNodeCacheSize, nodeMissCacheSize) ; -// nodeTable = NodeTableInline.create(nodeTable) ; -// return nodeTable ; -// } -// + { + return SetupIndex.makeRecordFactory(keyLen, valueLen) ; + } + // + // /** Make a NodeTable without cache and inline wrappers */ + // public static NodeTable makeNodeTableBase(Location location, String indexNode2Id, String indexId2Node) + // { + // if (location.isMem()) + // return NodeTableFactory.createMem() ; + // + // // -- make id to node mapping -- Names.indexId2Node + // FileSet fsIdToNode = new FileSet(location, indexId2Node) ; + // + // ObjectFile stringFile = makeObjectFile(fsIdToNode) ; + // + // // -- make node to id mapping -- Names.indexNode2Id + // // Make index of id to node (data table) + // + // // No caching at the index level - we use the internal caches of the node table. + // Index nodeToId = makeIndex(location, indexNode2Id, LenNodeHash, SizeOfNodeId, -1 ,-1) ; + // + // // -- Make the node table using the components established above. + // NodeTable nodeTable = new NodeTableNative(nodeToId, stringFile) ; + // return nodeTable ; + // } + // + // /** Make a NodeTable with cache and inline wrappers */ + // public static NodeTable makeNodeTable(Location location) + // { + // return makeNodeTable(location, + // Names.indexNode2Id, SystemTDB.Node2NodeIdCacheSize, + // Names.indexId2Node, SystemTDB.NodeId2NodeCacheSize, + // SystemTDB.NodeMissCacheSize) ; + // } + // + // /** Make a NodeTable with cache and inline wrappers */ + // public static NodeTable makeNodeTable(Location location, + // String indexNode2Id, int nodeToIdCacheSize, + // String indexId2Node, int idToNodeCacheSize, + // int nodeMissCacheSize) + // { + // NodeTable nodeTable = makeNodeTableBase(location, indexNode2Id, indexId2Node) ; + // nodeTable = NodeTableCache.create(nodeTable, nodeToIdCacheSize, idToNodeCacheSize, nodeMissCacheSize) ; + // nodeTable = NodeTableInline.create(nodeTable) ; + // return nodeTable ; + // } + // // XXX Move to FileFactory public static ObjectFile makeObjectFile(FileSet fsIdToNode) @@ -235,66 +228,31 @@ public class SetupTDB } /** Create a B+Tree using defaults */ - public static RangeIndex createBPTree(FileSet fileset, - RecordFactory factory) - { - int readCacheSize = SystemTDB.BlockReadCacheSize ; - int writeCacheSize = SystemTDB.BlockWriteCacheSize ; - int blockSize = SystemTDB.BlockSize ; - if ( fileset.isMem() ) - { - readCacheSize = 0 ; - writeCacheSize = 0 ; - blockSize = SystemTDB.BlockSizeTest ; - } - - return createBPTreeByBlockSize(fileset, blockSize, readCacheSize, writeCacheSize, factory) ; + public static RangeIndex createBPTree(FileSet fileset, RecordFactory factory) { + return SetupIndex.createBPTree(fileset, factory) ; } /** Create a B+Tree by BlockSize */ public static RangeIndex createBPTreeByBlockSize(FileSet fileset, int blockSize, int readCacheSize, int writeCacheSize, - RecordFactory factory) - { - return createBPTree(fileset, -1, blockSize, readCacheSize, writeCacheSize, factory) ; + RecordFactory factory) { + return SetupIndex.createBPTreeByBlockSize(fileset, blockSize, readCacheSize, writeCacheSize, factory) ; } /** Create a B+Tree by Order */ public static RangeIndex createBPTreeByOrder(FileSet fileset, int order, int readCacheSize, int writeCacheSize, - RecordFactory factory) - { - return createBPTree(fileset, order, -1, readCacheSize, writeCacheSize, factory) ; + RecordFactory factory) { + return SetupIndex.createBPTreeByOrder(fileset, order, readCacheSize, writeCacheSize, factory) ; } /** Knowing all the parameters, create a B+Tree */ public static BPlusTree createBPTree(FileSet fileset, int order, int blockSize, int readCacheSize, int writeCacheSize, - RecordFactory factory) - { - // ---- Checking - if (blockSize < 0 && order < 0) throw new IllegalArgumentException("Neither blocksize nor order specified") ; - if (blockSize >= 0 && order < 0) order = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; - if (blockSize >= 0 && order >= 0) - { - int order2 = BPlusTreeParams.calcOrder(blockSize, factory.recordLength()) ; - if (order != order2) throw new IllegalArgumentException("Wrong order (" + order + "), calculated = " - + order2) ; - } - - // Iffy - does not allow for slop. - if (blockSize < 0 && order >= 0) - { - // Only in-memory. - blockSize = BPlusTreeParams.calcBlockSize(order, factory) ; - } - - BPlusTreeParams params = new BPlusTreeParams(order, factory) ; - BlockMgr blkMgrNodes = BlockMgrFactory.create(fileset, Names.bptExtTree, blockSize, readCacheSize, writeCacheSize) ; - BlockMgr blkMgrRecords = BlockMgrFactory.create(fileset, Names.bptExtRecords, blockSize, readCacheSize, writeCacheSize) ; - return BPlusTree.create(params, blkMgrNodes, blkMgrRecords) ; + RecordFactory factory) { + return SetupIndex.createBPTree(fileset, order, blockSize, readCacheSize, writeCacheSize, factory) ; } } http://git-wip-us.apache.org/repos/asf/jena/blob/b659de12/jena-tdb/src/main/java/tdb/CmdRewriteIndex.java ---------------------------------------------------------------------- diff --git a/jena-tdb/src/main/java/tdb/CmdRewriteIndex.java b/jena-tdb/src/main/java/tdb/CmdRewriteIndex.java index a8353bf..a256bcc 100644 --- a/jena-tdb/src/main/java/tdb/CmdRewriteIndex.java +++ b/jena-tdb/src/main/java/tdb/CmdRewriteIndex.java @@ -30,11 +30,11 @@ import com.hp.hpl.jena.tdb.base.file.Location ; import com.hp.hpl.jena.tdb.base.record.Record ; import com.hp.hpl.jena.tdb.base.record.RecordFactory ; import com.hp.hpl.jena.tdb.index.RangeIndex ; +import com.hp.hpl.jena.tdb.index.SetupIndex ; import com.hp.hpl.jena.tdb.index.bplustree.BPlusTree ; import com.hp.hpl.jena.tdb.index.bplustree.BPlusTreeParams ; import com.hp.hpl.jena.tdb.index.bplustree.BPlusTreeRewriter ; import com.hp.hpl.jena.tdb.sys.Names ; -import com.hp.hpl.jena.tdb.sys.SetupTDB ; import com.hp.hpl.jena.tdb.sys.SystemTDB ; /** Rewrite one index */ @@ -107,7 +107,7 @@ public class CmdRewriteIndex BlockMgr blkMgrRecords ; int blockSize = SystemTDB.BlockSize ; - RangeIndex rangeIndex = SetupTDB.makeRangeIndex(srcLoc, indexName, blockSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; + RangeIndex rangeIndex = SetupIndex.makeRangeIndex(srcLoc, indexName, blockSize, dftKeyLength, dftValueLength, readCacheSize, writeCacheSize) ; BPlusTree bpt = (BPlusTree)rangeIndex ; bptParams = bpt.getParams() ; recordFactory = bpt.getRecordFactory() ; http://git-wip-us.apache.org/repos/asf/jena/blob/b659de12/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/index/bplustree/TestBPlusTreeRewriter.java ---------------------------------------------------------------------- diff --git a/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/index/bplustree/TestBPlusTreeRewriter.java b/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/index/bplustree/TestBPlusTreeRewriter.java index 9cf549e..c0e644e 100644 --- a/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/index/bplustree/TestBPlusTreeRewriter.java +++ b/jena-tdb/src/test/java/com/hp/hpl/jena/tdb/index/bplustree/TestBPlusTreeRewriter.java @@ -34,8 +34,8 @@ import com.hp.hpl.jena.tdb.base.block.BlockMgrFactory ; import com.hp.hpl.jena.tdb.base.file.FileSet ; import com.hp.hpl.jena.tdb.base.record.Record ; import com.hp.hpl.jena.tdb.base.record.RecordFactory ; +import com.hp.hpl.jena.tdb.index.SetupIndex ; import com.hp.hpl.jena.tdb.sys.Names ; -import com.hp.hpl.jena.tdb.sys.SetupTDB ; public class TestBPlusTreeRewriter extends BaseTest { @@ -169,7 +169,7 @@ public class TestBPlusTreeRewriter extends BaseTest static List<Record> createData2(int ORDER, int N, RecordFactory recordFactory) { // Use a B+Tree - so original data can be unsorted. - BPlusTree bpt = SetupTDB.createBPTree(FileSet.mem(), ORDER, -1, -1, -1, recordFactory) ; + BPlusTree bpt = SetupIndex.createBPTree(FileSet.mem(), ORDER, -1, -1, -1, recordFactory) ; //BPlusTreeParams.checkAll() ; // 200 -> runt leaf problem.
