Modified blur service and added the snapshot methods, generated the code, hooked the controller and shard servers for the new methods and stubbed the BlurNRTIndex
Signed-off-by: Aaron McCurry <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d512053c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d512053c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d512053c Branch: refs/heads/master Commit: d512053c53e65182c184204a6b1fd720e4c37320 Parents: 079113a Author: Rahul Challapalli <[email protected]> Authored: Fri Aug 16 23:19:08 2013 -0700 Committer: Aaron McCurry <[email protected]> Committed: Sat Aug 17 12:53:33 2013 -0400 ---------------------------------------------------------------------- .../apache/blur/manager/writer/BlurIndex.java | 7 + .../blur/manager/writer/BlurIndexReader.java | 17 + .../blur/manager/writer/BlurNRTIndex.java | 16 + .../blur/thrift/BlurControllerServer.java | 68 + .../org/apache/blur/thrift/BlurShardServer.java | 63 + .../org/apache/blur/thrift/generated/Blur.java | 2809 ++++++++++++++++++ .../src/main/scripts/interface/Blur.thrift | 14 + .../main/scripts/interface/gen-html/Blur.html | 19 +- .../main/scripts/interface/gen-html/index.html | 3 + .../org/apache/blur/thrift/generated/Blur.java | 2809 ++++++++++++++++++ .../src/main/scripts/interface/gen-js/Blur.js | 533 ++++ .../scripts/interface/gen-perl/Blur/Blur.pm | 713 +++++ .../src/main/scripts/interface/gen-rb/blur.rb | 181 ++ 13 files changed, 7251 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java index 344e886..19b26b6 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndex.java @@ -17,6 +17,7 @@ package org.apache.blur.manager.writer; * limitations under the License. */ import java.io.IOException; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -51,6 +52,12 @@ public abstract class BlurIndex { public abstract AtomicBoolean isClosed(); public abstract void optimize(int numberOfSegmentsPerShard) throws IOException; + + public abstract void createSnapshot(String name) throws IOException; + + public abstract void removeSnapshot(String name) throws IOException; + + public abstract List<String> getSnapshots() throws IOException; public long getRecordCount() throws IOException { IndexSearcherClosable searcher = getIndexReader(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java index f3733e6..6017e0a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java @@ -19,6 +19,7 @@ package org.apache.blur.manager.writer; import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import java.io.IOException; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -29,6 +30,7 @@ import org.apache.blur.lucene.warmup.TraceableDirectory; import org.apache.blur.server.IndexSearcherClosable; import org.apache.blur.server.ShardContext; import org.apache.blur.server.TableContext; +import org.apache.blur.thrift.BException; import org.apache.blur.thrift.generated.Row; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.index.BlurIndexWriter; @@ -130,4 +132,19 @@ public class BlurIndexReader extends BlurIndex { public AtomicBoolean isClosed() { return _isClosed; } + + @Override + public void createSnapshot(String name) throws IOException { + throw new RuntimeException("Read-only shard"); + } + + @Override + public void removeSnapshot(String name) throws IOException { + throw new RuntimeException("Read-only shard"); + } + + @Override + public List<String> getSnapshots() throws IOException { + throw new RuntimeException("Read-only shard"); + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java index 993074a..ed0569a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java @@ -264,4 +264,20 @@ public class BlurNRTIndex extends BlurIndex { } } + @Override + public void createSnapshot(String name) throws IOException { + // TODO Auto-generated method stub + } + + @Override + public void removeSnapshot(String name) throws IOException { + // TODO Auto-generated method stub + } + + @Override + public List<String> getSnapshots() throws IOException { + // TODO Auto-generated method stub + return null; + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index cdac2f1..822508c 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -830,6 +830,74 @@ public class BlurControllerServer extends TableAdmin implements Iface { } } } + + @Override + public void createSnapshot(final String table, final String name) throws BlurException, TException { + checkTable(table); + try { + scatter(getCluster(table), new BlurCommand<Void>() { + @Override + public Void call(Client client) throws BlurException, TException { + client.createSnapshot(table, name); + return null; + } + }); + } catch (Exception e) { + LOG.error("Unknown error while trying to create a snapshot of table [{0}] snapshot name", e, table, name); + throw new BException("Unknown error while trying to create a snapshot of table [{0}] snapshot name", e, table, name); + } + } + + @Override + public void removeSnapshot(final String table, final String name) throws BlurException, TException { + checkTable(table); + try { + scatter(getCluster(table), new BlurCommand<Void>() { + @Override + public Void call(Client client) throws BlurException, TException { + client.removeSnapshot(table, name); + return null; + } + }); + } catch (Exception e) { + LOG.error("Unknown error while trying to remove a snapshot of table [{0}] snapshot name", e, table, name); + throw new BException("Unknown error while trying to remove a snapshot of table [{0}] snapshot name", e, table, name); + } + } + + @Override + public Map<String, List<String>> listSnapshots(final String table) throws BlurException, TException { + checkTable(table); + try { + return scatterGather(getCluster(table), new BlurCommand<Map<String, List<String>>>() { + @Override + public Map<String, List<String>> call(Client client) throws BlurException, TException { + return client.listSnapshots(table); + } + }, new Merger<Map<String, List<String>>>() { + @Override + public Map<String, List<String>> merge(BlurExecutorCompletionService<Map<String, List<String>>> service) throws BlurException { + Map<String, List<String>> result = new HashMap<String, List<String>>(); + while (service.getRemainingCount() > 0) { + Future<Map<String, List<String>>> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS, true); + Map<String, List<String>> snapshotsOnAShardServer = service.getResultThrowException(future); + for (Entry<String, List<String>> entry : snapshotsOnAShardServer.entrySet()) { + List<String> snapshots = result.get(entry.getKey()); + if (snapshots == null) { + snapshots = new ArrayList<String>(); + result.put(entry.getKey(), snapshots); + } + snapshots.addAll(entry.getValue()); + } + } + return result; + } + }); + } catch (Exception e) { + LOG.error("Unknown error while trying to get the list of snapshots for table [{0}]", e, table); + throw new BException("Unknown error while trying to get the list of snapshots for table [{0}]", e, table); + } + } public void setNodeName(String nodeName) { _nodeName = nodeName; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d512053c/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index edd3d80..61cf775 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -21,12 +21,14 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIV import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLongArray; @@ -41,6 +43,8 @@ import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.writer.BlurIndex; import org.apache.blur.server.ShardServerContext; import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.commands.BlurCommand; +import org.apache.blur.thrift.generated.Blur.Client; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; @@ -56,10 +60,12 @@ import org.apache.blur.thrift.generated.SimpleQuery; import org.apache.blur.thrift.generated.Status; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.utils.BlurConstants; +import org.apache.blur.utils.BlurExecutorCompletionService; import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.QueryCache; import org.apache.blur.utils.QueryCacheEntry; import org.apache.blur.utils.QueryCacheKey; +import org.apache.blur.utils.ForkJoin.Merger; public class BlurShardServer extends TableAdmin implements Iface { @@ -397,6 +403,63 @@ public class BlurShardServer extends TableAdmin implements Iface { throw new BException(e.getMessage(), e); } } + + @Override + public void createSnapshot(final String table, final String name) throws BlurException, TException { + try { + Map<String, BlurIndex> indexes = _indexServer.getIndexes(table); + for (Entry<String, BlurIndex> entry : indexes.entrySet()) { + BlurIndex index = entry.getValue(); + index.createSnapshot(name); + } + } catch (Exception e) { + LOG.error("Unknown error while trying to getting indexes for table [" + table + "]", e); + if (e instanceof BlurException) { + throw (BlurException) e; + } + throw new BException(e.getMessage(), e); + } + } + + @Override + public void removeSnapshot(final String table, final String name) throws BlurException, TException { + try { + Map<String, BlurIndex> indexes = _indexServer.getIndexes(table); + for (Entry<String, BlurIndex> entry : indexes.entrySet()) { + BlurIndex index = entry.getValue(); + index.removeSnapshot(name); + } + } catch (Exception e) { + LOG.error("Unknown error while trying to getting indexes for table [" + table + "]", e); + if (e instanceof BlurException) { + throw (BlurException) e; + } + throw new BException(e.getMessage(), e); + } + } + + @Override + public Map<String, List<String>> listSnapshots(final String table) throws BlurException, TException { + Map<String, List<String>> snapshots = new HashMap<String, List<String>>(); + try { + Map<String, BlurIndex> indexes = _indexServer.getIndexes(table); + for (Entry<String, BlurIndex> entry : indexes.entrySet()) { + BlurIndex index = entry.getValue(); + List<String> shardSnapshots = index.getSnapshots(); + if (shardSnapshots == null) { + shardSnapshots = new ArrayList<String>(); + } + snapshots.put(entry.getKey(), shardSnapshots); + } + return snapshots; + } catch (Exception e) { + LOG.error("Unknown error while trying to getting indexes for table [" + table + "]", e); + if (e instanceof BlurException) { + throw (BlurException) e; + } + throw new BException(e.getMessage(), e); + } + } public int getDataFetchThreadCount() { return _dataFetchThreadCount;
