Forcing bulk indexing to NOT use the fast keyvalue store.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8b5b12e7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8b5b12e7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8b5b12e7 Branch: refs/heads/master Commit: 8b5b12e778189ab1b27131db93c5e726f138e49c Parents: 5833291 Author: Aaron McCurry <[email protected]> Authored: Mon Mar 2 21:08:40 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Mar 2 21:08:40 2015 -0500 ---------------------------------------------------------------------- .../manager/writer/BlurIndexSimpleWriter.java | 11 ++++++++- .../blur/store/hdfs_v2/JoinDirectory.java | 9 ++++++- .../blur/store/hdfs_v2/StoreDirection.java | 26 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b5b12e7/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java index 8ed070b..66f8b01 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java @@ -63,6 +63,7 @@ import org.apache.blur.server.IndexSearcherCloseableSecureBase; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; import org.apache.blur.server.cache.ThriftCache; +import org.apache.blur.store.hdfs_v2.StoreDirection; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.TableDescriptor; @@ -804,16 +805,24 @@ public class BlurIndexSimpleWriter extends BlurIndex { } else { final IndexAction indexAction = bulkEntry.getIndexAction(); if (blockUntilComplete) { - process(indexAction); + StoreDirection.LONG_TERM.set(true); + try { + process(indexAction); + } finally { + StoreDirection.LONG_TERM.set(false); + } } else { Thread thread = new Thread(new Runnable() { @Override public void run() { try { + StoreDirection.LONG_TERM.set(true); process(indexAction); } catch (IOException e) { LOG.error("Shard [{0}/{1}] Id [{2}] Unknown error while trying to finish the bulk updates.", e, table, shard, bulkId); + } finally { + StoreDirection.LONG_TERM.set(false); } } }); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b5b12e7/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java index 6619e3f..4182391 100644 --- a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/JoinDirectory.java @@ -93,7 +93,7 @@ public class JoinDirectory extends Directory implements LastModified, HdfsSymlin @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { - if (Thread.currentThread().getName().startsWith(BlurConstants.SHARED_MERGE_SCHEDULER_PREFIX)) { + if (shouldBeLongTermStorage()) { addLongTermSyncFile(name); return _longTermStorage.createOutput(name, context); } @@ -101,6 +101,13 @@ public class JoinDirectory extends Directory implements LastModified, HdfsSymlin return _shortTermStorage.createOutput(name, context); } + private boolean shouldBeLongTermStorage() { + if (Thread.currentThread().getName().startsWith(BlurConstants.SHARED_MERGE_SCHEDULER_PREFIX)) { + return true; + } + return StoreDirection.LONG_TERM.get(); + } + private void addShortTermSyncFile(String name) { _shortTermSyncFiles.add(name); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b5b12e7/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/StoreDirection.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/StoreDirection.java b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/StoreDirection.java new file mode 100644 index 0000000..2dacc8a --- /dev/null +++ b/blur-store/src/main/java/org/apache/blur/store/hdfs_v2/StoreDirection.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.store.hdfs_v2; + +public class StoreDirection { + public static ThreadLocal<Boolean> LONG_TERM = new ThreadLocal<Boolean>() { + @Override + protected Boolean initialValue() { + return false; + } + }; +}
