Repository: incubator-blur Updated Branches: refs/heads/master 87d29a17d -> 89e10a0dd
Java collections now work in arguments as well as return types. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/89e10a0d Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/89e10a0d Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/89e10a0d Branch: refs/heads/master Commit: 89e10a0dd77d3a10072f027793b1b52a329a3f2f Parents: 87d29a1 Author: Aaron McCurry <[email protected]> Authored: Thu Oct 2 13:40:04 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Thu Oct 2 13:40:04 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/command/TermsCommand.java | 17 +- .../blur/command/example/UsingTermCommand.java | 6 +- .../apache/blur/command/DocFreqCommandTest.java | 4 - .../blur/command/IntegrationTestSuite.java | 4 +- .../blur/command/TermsCommandIntTests.java | 4 +- .../apache/blur/command/TermsCommandTest.java | 41 ++-- .../apache/blur/command/ArgumentOverlay.java | 8 +- .../apache/blur/command/BaseCommandManager.java | 10 +- .../org/apache/blur/command/BlurObject.java | 32 +++ .../apache/blur/command/BlurObjectSerDe.java | 210 +++++++++++++++++++ .../org/apache/blur/command/CommandRunner.java | 21 +- .../org/apache/blur/command/CommandUtil.java | 59 ++++-- .../blur/command/ControllerClusterContext.java | 7 +- .../blur/thrift/BlurControllerServer.java | 9 +- .../org/apache/blur/thrift/BlurShardServer.java | 10 +- .../blur/command/ShardCommandManagerTest.java | 10 +- 16 files changed, 365 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/main/java/org/apache/blur/command/TermsCommand.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/TermsCommand.java b/blur-command/src/main/java/org/apache/blur/command/TermsCommand.java index 3b5e116..3bc2dc6 100644 --- a/blur-command/src/main/java/org/apache/blur/command/TermsCommand.java +++ b/blur-command/src/main/java/org/apache/blur/command/TermsCommand.java @@ -36,7 +36,7 @@ import com.google.common.collect.Sets; * License for the specific language governing permissions and limitations under * the License. */ -public class TermsCommand extends ClusterServerReadCommandSingleTable<BlurArray> { +public class TermsCommand extends ClusterServerReadCommandSingleTable<List<String>> { private static final String NAME = "terms"; @RequiredArgument @@ -49,19 +49,18 @@ public class TermsCommand extends ClusterServerReadCommandSingleTable<BlurArray> private String startWith = ""; @Override - public BlurArray execute(IndexContext context) throws IOException { - return new BlurArray(terms(context.getIndexReader(), fieldName, startWith, size)); + public List<String> execute(IndexContext context) throws IOException { + return terms(context.getIndexReader(), fieldName, startWith, size); } - @SuppressWarnings("unchecked") @Override - public BlurArray combine(CombiningContext context, Map<? extends Location<?>, BlurArray> results) throws IOException, - InterruptedException { + public List<String> combine(CombiningContext context, Map<? extends Location<?>, List<String>> results) + throws IOException, InterruptedException { SortedSet<String> terms = Sets.newTreeSet(); - for (BlurArray t : results.values()) { - terms.addAll((List<String>) t.asList()); + for (List<String> t : results.values()) { + terms.addAll(t); } - return new BlurArray(Lists.newArrayList(terms).subList(0, Math.min((int) size, terms.size()))); + return Lists.newArrayList(terms).subList(0, Math.min((int) size, terms.size())); } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/main/java/org/apache/blur/command/example/UsingTermCommand.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/example/UsingTermCommand.java b/blur-command/src/main/java/org/apache/blur/command/example/UsingTermCommand.java index 8962896..5ea9e66 100644 --- a/blur-command/src/main/java/org/apache/blur/command/example/UsingTermCommand.java +++ b/blur-command/src/main/java/org/apache/blur/command/example/UsingTermCommand.java @@ -17,8 +17,8 @@ package org.apache.blur.command.example; * limitations under the License. */ import java.io.IOException; +import java.util.List; -import org.apache.blur.command.BlurArray; import org.apache.blur.command.TermsCommand; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.BlurException; @@ -29,7 +29,7 @@ public class UsingTermCommand { TermsCommand command = new TermsCommand(); command.setTable("test"); command.setFieldName("fam0.col0"); - BlurArray blurArray = command.run("localhost:40020"); - System.out.println(blurArray); + List<String> terms = command.run("localhost:40010"); + System.out.println(terms); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/test/java/org/apache/blur/command/DocFreqCommandTest.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/DocFreqCommandTest.java b/blur-command/src/test/java/org/apache/blur/command/DocFreqCommandTest.java index 5980a48..d2547ca 100644 --- a/blur-command/src/test/java/org/apache/blur/command/DocFreqCommandTest.java +++ b/blur-command/src/test/java/org/apache/blur/command/DocFreqCommandTest.java @@ -3,14 +3,10 @@ package org.apache.blur.command; import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.util.Map; import org.junit.BeforeClass; import org.junit.Test; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java b/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java index a5a8024..2fdacf3 100644 --- a/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java +++ b/blur-command/src/test/java/org/apache/blur/command/IntegrationTestSuite.java @@ -4,11 +4,9 @@ import java.io.IOException; import java.util.List; import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.FacetTests; import org.apache.blur.thrift.SuiteCluster; -import org.apache.blur.thrift.TermsTests; -import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; import org.junit.After; import org.junit.ClassRule; import org.junit.rules.ExternalResource; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/test/java/org/apache/blur/command/TermsCommandIntTests.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/TermsCommandIntTests.java b/blur-command/src/test/java/org/apache/blur/command/TermsCommandIntTests.java index f3b14eb..3f5c09d 100644 --- a/blur-command/src/test/java/org/apache/blur/command/TermsCommandIntTests.java +++ b/blur-command/src/test/java/org/apache/blur/command/TermsCommandIntTests.java @@ -41,9 +41,9 @@ public class TermsCommandIntTests extends BaseClusterTest { command.setTable(tableName); command.setFieldName("test.col1"); - BlurArray blurArray = command.run(getClient()); + List<String> terms = command.run(getClient()); List<String> list = Lists.newArrayList("value"); - assertEquals(list, blurArray.asList()); + assertEquals(list, terms); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-command/src/test/java/org/apache/blur/command/TermsCommandTest.java ---------------------------------------------------------------------- diff --git a/blur-command/src/test/java/org/apache/blur/command/TermsCommandTest.java b/blur-command/src/test/java/org/apache/blur/command/TermsCommandTest.java index b7b74d0..73161f6 100644 --- a/blur-command/src/test/java/org/apache/blur/command/TermsCommandTest.java +++ b/blur-command/src/test/java/org/apache/blur/command/TermsCommandTest.java @@ -3,6 +3,7 @@ package org.apache.blur.command; import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.List; import java.util.Map; import org.junit.BeforeClass; @@ -37,69 +38,69 @@ public class TermsCommandTest { @Test public void basicTermsShouldReturn() throws IOException { - BlurArray returned = getExecuteResult(ctx, "val", null, null); - BlurArray expected = new BlurArray(Lists.newArrayList("val")); + List<String> returned = getExecuteResult(ctx, "val", null, null); + List<String> expected = Lists.newArrayList("val"); assertEquals(expected, returned); } @Test public void sizeOfTermsRequestShouldBeRespected() throws IOException { - BlurArray returned = getExecuteResult(ctx, "alpha", (short) 7, null); - BlurArray expected = new BlurArray(Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg")); + List<String> returned = getExecuteResult(ctx, "alpha", (short) 7, null); + List<String> expected = Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg"); assertEquals(expected, returned); } @Test public void sizeShouldDefaultToTen() throws IOException { - BlurArray returned = getExecuteResult(ctx, "alpha", null, null); - BlurArray expected = new BlurArray(Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj")); + List<String> returned = getExecuteResult(ctx, "alpha", null, null); + List<String> expected = Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj"); assertEquals(expected, returned); } @Test public void combineSizeShouldDefaultToTen() throws IOException, InterruptedException { - Map<Shard, BlurArray> execResults = Maps.newHashMap(); - execResults.put(new Shard("t1", "s1"), new BlurArray(Lists.newArrayList("aa", "cc", "ee", "gg", "ii"))); - execResults.put(new Shard("t1", "s2"), new BlurArray(Lists.newArrayList("bb", "dd", "ff", "hh", "jj"))); + Map<Shard, List<String>> execResults = Maps.newHashMap(); + execResults.put(new Shard("t1", "s1"), Lists.newArrayList("aa", "cc", "ee", "gg", "ii")); + execResults.put(new Shard("t1", "s2"), Lists.newArrayList("bb", "dd", "ff", "hh", "jj")); - BlurArray expected = new BlurArray(Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj")); + List<String> expected = Lists.newArrayList("aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj"); TermsCommand cmd = new TermsCommand(); - BlurArray returned = cmd.combine(new TestCombiningContext(), execResults); + List<String> returned = cmd.combine(new TestCombiningContext(), execResults); assertEquals(expected, returned); } @Test public void combineShouldRespectSize() throws IOException, InterruptedException { - Map<Shard, BlurArray> execResults = Maps.newHashMap(); - execResults.put(new Shard("t1", "s1"), new BlurArray(Lists.newArrayList("aa", "cc"))); - execResults.put(new Shard("t1", "s2"), new BlurArray(Lists.newArrayList("bb", "dd"))); + Map<Shard, List<String>> execResults = Maps.newHashMap(); + execResults.put(new Shard("t1", "s1"), Lists.newArrayList("aa", "cc")); + execResults.put(new Shard("t1", "s2"), Lists.newArrayList("bb", "dd")); - BlurArray expected = new BlurArray(Lists.newArrayList("aa", "bb")); + List<String> expected = Lists.newArrayList("aa", "bb"); TermsCommand cmd = new TermsCommand(); cmd.setSize((short) 2); - BlurArray returned = cmd.combine(new TestCombiningContext(), execResults); + List<String> returned = cmd.combine(new TestCombiningContext(), execResults); assertEquals(expected, returned); } @Test public void combineEmptyShouldGiveNiceEmptyList() throws IOException, InterruptedException { - Map<Shard, BlurArray> execResults = Maps.newHashMap(); - BlurArray expected = new BlurArray(Lists.newArrayList()); + Map<Shard, List<String>> execResults = Maps.newHashMap(); + List<String> expected = Lists.newArrayList(); TermsCommand cmd = new TermsCommand(); - BlurArray returned = cmd.combine(new TestCombiningContext(), execResults); + List<String> returned = cmd.combine(new TestCombiningContext(), execResults); assertEquals(expected, returned); } - private BlurArray getExecuteResult(IndexContext context, String field, Short size, String startsWith) + private List<String> getExecuteResult(IndexContext context, String field, Short size, String startsWith) throws IOException { TermsCommand cmd = new TermsCommand(); cmd.setFieldName(field); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/ArgumentOverlay.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/ArgumentOverlay.java b/blur-core/src/main/java/org/apache/blur/command/ArgumentOverlay.java index 9cf4a6a..e051b93 100644 --- a/blur-core/src/main/java/org/apache/blur/command/ArgumentOverlay.java +++ b/blur-core/src/main/java/org/apache/blur/command/ArgumentOverlay.java @@ -17,22 +17,22 @@ package org.apache.blur.command; import java.lang.reflect.Field; +import java.util.Map; import org.apache.blur.command.annotation.OptionalArgument; import org.apache.blur.command.annotation.RequiredArgument; public class ArgumentOverlay { - private BlurObject _args; + private final Map<String, ? extends Object> _args; - public ArgumentOverlay(BlurObject args) { - _args = args; + public ArgumentOverlay(BlurObject args, BlurObjectSerDe serDe) { + _args = serDe.deserialize(args); } public <T> Command<T> setup(Command<T> command) { Class<?> clazz = command.getClass(); setupInternal(clazz, command); - return command; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java index e41a32b..7b73ab0 100644 --- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java @@ -33,6 +33,7 @@ import org.apache.blur.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.thrift.generated.Arguments; +import org.apache.blur.thrift.generated.BlurException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -78,6 +79,7 @@ public abstract class BaseCommandManager implements Closeable { protected final long _pollingPeriod = TimeUnit.SECONDS.toMillis(15); protected final Map<Path, BigInteger> _commandPathLastChange = new ConcurrentHashMap<Path, BigInteger>(); protected final Configuration _configuration; + protected final BlurObjectSerDe _serDe = new BlurObjectSerDe(); public BaseCommandManager(String tmpPath, String commandPath, int workerThreadCount, int driverThreadCount, long connectionTimeout, Configuration configuration) throws IOException { @@ -482,8 +484,12 @@ public abstract class BaseCommandManager implements Closeable { return command.getReturnType(); } - protected Arguments toArguments(Command<?> command) { - return CommandUtil.toArguments(command); + protected Arguments toArguments(Command<?> command) throws IOException { + try { + return CommandUtil.toArguments(command, _serDe); + } catch (BlurException e) { + throw new IOException(e); + } } protected void validate(Command<?> command) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/BlurObject.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BlurObject.java b/blur-core/src/main/java/org/apache/blur/command/BlurObject.java index f4d37df..e61899d 100644 --- a/blur-core/src/main/java/org/apache/blur/command/BlurObject.java +++ b/blur-core/src/main/java/org/apache/blur/command/BlurObject.java @@ -320,4 +320,36 @@ public class BlurObject { return _valueMap.get(name); } + public static boolean supportedType(Object o) { + if (o instanceof String) { + return true; + } else if (o instanceof String) { + return true; + } else if (o instanceof Short) { + return true; + } else if (o instanceof Long) { + return true; + } else if (o instanceof Integer) { + return true; + } else if (o instanceof Float) { + return true; + } else if (o instanceof Double) { + return true; + } else if (o instanceof byte[]) { + return true; + } else if (o instanceof Boolean) { + return true; + } else if (o instanceof BlurObject) { + return true; + } else if (o instanceof BlurArray) { + return true; + } else { + return false; + } + } + + public boolean hasKey(String key) { + return _valueMap.containsKey(key); + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/BlurObjectSerDe.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BlurObjectSerDe.java b/blur-core/src/main/java/org/apache/blur/command/BlurObjectSerDe.java new file mode 100644 index 0000000..ca6f776 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/command/BlurObjectSerDe.java @@ -0,0 +1,210 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.command; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +public class BlurObjectSerDe { + + private static final String SET = "set"; + private static final String VALUE = "v"; + private static final String KEY = "k"; + private static final String MAP = "map"; + private static final String LIST = "list"; + private static final String TYPE = "serdetype"; + + // TODO make real unit test + + // public static <E> void main(String[] args) { + // BlurObjectSerDe serde = new BlurObjectSerDe(); + // Map<String, Long> m = new HashMap<String, Long>(); + // m.put("sub", 100L); + // Map<String, Object> map = new HashMap<String, Object>(); + // map.put("a", "a"); + // map.put("b", Arrays.asList("a", "b", "c")); + // map.put("c", m); + // + // Set<Integer> set = new HashSet<Integer>(); + // set.add(3); + // set.add(5); + // map.put("d", set); + // + // BlurObject bo1 = serde.serialize(map); + // System.out.println(bo1.toString(1)); + // + // Map<String, ? extends Object> map2 = serde.deserialize(bo1); + // System.out.println(map2); + // + // Set<?> set2 = (Set<?>) map.get("d"); + // System.out.println(set2); + // } + + public BlurObject serialize(Map<String, ? extends Object> args) { + BlurObject result = new BlurObject(); + for (Entry<String, ? extends Object> e : args.entrySet()) { + result.put(e.getKey(), toSupportedThriftObject(e.getValue())); + } + return result; + } + + public Object toSupportedThriftObject(Object o) { + if (BlurObject.supportedType(o)) { + return o; + } else { + return convertToSupportedType(o); + } + } + + @SuppressWarnings("unchecked") + private BlurObject convertToSupportedType(Object o) { + if (o instanceof Set) { + return toBlurObject((Set<?>) o); + } else if (o instanceof Map) { + return toBlurObject((Map<Object, Object>) o); + } else if (o instanceof List) { + return toBlurObject((List<Object>) o); + } else { + return toBlurObjectFromUnknown(o); + } + } + + private BlurObject toBlurObject(Map<Object, Object> o) { + BlurObject blurObject = new BlurObject(); + blurObject.put(TYPE, MAP); + BlurArray blurArray = new BlurArray(); + Set<Entry<Object, Object>> entrySet = o.entrySet(); + for (Entry<Object, Object> e : entrySet) { + BlurObject entry = new BlurObject(); + entry.put(KEY, toSupportedThriftObject(e.getKey())); + entry.put(VALUE, toSupportedThriftObject(e.getValue())); + blurArray.put(entry); + } + blurObject.put(MAP, blurArray); + return blurObject; + } + + private BlurObject toBlurObject(List<Object> o) { + BlurObject blurObject = new BlurObject(); + blurObject.put(TYPE, LIST); + BlurArray blurArray = new BlurArray(); + for (Object t : o) { + blurArray.put(toSupportedThriftObject(t)); + } + blurObject.put(LIST, blurArray); + return blurObject; + } + + private BlurObject toBlurObject(Set<?> o) { + BlurObject blurObject = new BlurObject(); + blurObject.put(TYPE, SET); + BlurArray blurArray = new BlurArray(); + for (Object t : o) { + blurArray.put(toSupportedThriftObject(t)); + } + blurObject.put(SET, blurArray); + return blurObject; + } + + private BlurObject toBlurObjectFromUnknown(Object o) { + throw new RuntimeException("Not implemented."); + } + + public Map<String, ? extends Object> deserialize(BlurObject blurObject) { + Map<String, Object> map = new HashMap<String, Object>(); + Iterator<String> keys = blurObject.keys(); + while (keys.hasNext()) { + String key = keys.next(); + map.put(key, fromSupportedThriftObject(blurObject.get(key))); + } + return map; + } + + public Object fromSupportedThriftObject(Object object) { + if (object instanceof BlurObject) { + BlurObject bo = (BlurObject) object; + if (isCustomObject(bo)) { + return convertFromSupportedType(bo); + } + } + return object; + } + + private Object convertFromSupportedType(BlurObject blurObject) { + String type = blurObject.getString(TYPE); + if (type.equals(LIST)) { + return toList(blurObject); + } else if (type.equals(MAP)) { + return toMap(blurObject); + } else if (type.equals(SET)) { + return toSet(blurObject); + } else { + return toUnknownObject(blurObject); + } + } + + private Object toSet(BlurObject blurObject) { + BlurArray blurArray = blurObject.getBlurArray(SET); + Set<Object> set = new HashSet<Object>(); + int length = blurArray.length(); + for (int i = 0; i < length; i++) { + set.add(fromSupportedThriftObject(blurArray.get(i))); + } + return set; + } + + private Object toUnknownObject(BlurObject blurObject) { + throw new RuntimeException("Not implemented."); + } + + private Map<? extends Object, ? extends Object> toMap(BlurObject blurObject) { + Map<Object, Object> result = new HashMap<Object, Object>(); + BlurArray blurArray = blurObject.getBlurArray(MAP); + int length = blurArray.length(); + for (int i = 0; i < length; i++) { + BlurObject bo = blurArray.getBlurObject(i); + Object key = bo.get(KEY); + Object value = bo.get(VALUE); + result.put(fromSupportedThriftObject(key), fromSupportedThriftObject(value)); + } + return result; + } + + private List<? extends Object> toList(BlurObject blurObject) { + BlurArray blurArray = blurObject.getBlurArray(LIST); + List<Object> list = new ArrayList<Object>(); + int length = blurArray.length(); + for (int i = 0; i < length; i++) { + list.add(fromSupportedThriftObject(blurArray.get(i))); + } + return list; + } + + private boolean isCustomObject(BlurObject bo) { + if (bo.hasKey(TYPE)) { + return true; + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/CommandRunner.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/CommandRunner.java b/blur-core/src/main/java/org/apache/blur/command/CommandRunner.java index b9df674..b281f05 100644 --- a/blur-core/src/main/java/org/apache/blur/command/CommandRunner.java +++ b/blur-core/src/main/java/org/apache/blur/command/CommandRunner.java @@ -83,14 +83,13 @@ public class CommandRunner { return (Map<Shard, T>) runInternal((Command<?>) command, connection); } - public static <T> Map<Server, T> run(ServerReadCommand<?, T> command) throws IOException, BlurException, - TException { + public static <T> Map<Server, T> run(ServerReadCommand<?, T> command) throws IOException, BlurException, TException { Iface client = BlurClient.getClient(); return run(command, getConnection(client)); } - public static <T> Map<Server, T> run(ServerReadCommand<?, T> command, String connectionStr) - throws IOException, BlurException, TException { + public static <T> Map<Server, T> run(ServerReadCommand<?, T> command, String connectionStr) throws IOException, + BlurException, TException { Iface client = BlurClient.getClient(connectionStr); return run(command, getConnection(client)); } @@ -101,8 +100,8 @@ public class CommandRunner { } @SuppressWarnings("unchecked") - public static <T> Map<Server, T> run(ServerReadCommand<?, T> command, Connection... connection) - throws IOException, BlurException, TException { + public static <T> Map<Server, T> run(ServerReadCommand<?, T> command, Connection... connection) throws IOException, + BlurException, TException { return (Map<Server, T>) runInternal((Command<?>) command, connection); } @@ -132,8 +131,8 @@ public class CommandRunner { } @SuppressWarnings("unchecked") - public static <T> T run(ClusterServerReadCommand<T> command, String connectioStr) throws IOException, - BlurException, TException { + public static <T> T run(ClusterServerReadCommand<T> command, String connectioStr) throws IOException, BlurException, + TException { return (T) runInternal((Command<?>) command, getConnection(BlurClient.getClient(connectioStr))); } @@ -201,8 +200,10 @@ public class CommandRunner { ClientPool clientPool = BlurClientManager.getClientPool(); Client client = clientPool.getClient(connection); try { - Response response = client.execute(command.getName(), CommandUtil.toArguments(command)); - return CommandUtil.fromThriftResponseToObject(response); + BlurObjectSerDe serde = new BlurObjectSerDe(); + Response response = client.execute(command.getName(), CommandUtil.toArguments(command, serde)); + Object thriftObject = CommandUtil.fromThriftResponseToObject(response); + return serde.fromSupportedThriftObject(thriftObject); } finally { clientPool.returnClient(connection, client); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/CommandUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/CommandUtil.java b/blur-core/src/main/java/org/apache/blur/command/CommandUtil.java index 3f7bf6a..bb87f2d 100644 --- a/blur-core/src/main/java/org/apache/blur/command/CommandUtil.java +++ b/blur-core/src/main/java/org/apache/blur/command/CommandUtil.java @@ -2,6 +2,7 @@ package org.apache.blur.command; import java.lang.reflect.Field; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -34,24 +35,48 @@ import org.apache.blur.thrift.generated.ValueObject._Fields; public class CommandUtil { - public static org.apache.blur.thrift.generated.Response fromObjectToThrift(Response response) throws BlurException { + public static org.apache.blur.thrift.generated.Response fromObjectToThrift(Response response, BlurObjectSerDe serDe) + throws BlurException { org.apache.blur.thrift.generated.Response converted = new org.apache.blur.thrift.generated.Response(); + if (response.isAggregatedResults()) { - converted.setValue(toValueObject(response.getServerResult())); + Object result = response.getServerResult(); + Object supportedThriftObject = serDe.toSupportedThriftObject(result); + converted.setValue(toValueObject(supportedThriftObject)); } else { Map<Server, Object> serverResults = response.getServerResults(); if (serverResults == null) { - Map<org.apache.blur.thrift.generated.Shard, ValueObject> fromObjectToThrift = fromObjectToThrift(response - .getShardResults()); + Map<Shard, Object> shardResults = response.getShardResults(); + Map<Shard, Object> supportedThriftObjectShardResults = toThriftSupportedObjects(shardResults, serDe); + Map<org.apache.blur.thrift.generated.Shard, ValueObject> fromObjectToThrift = fromObjectToThrift(supportedThriftObjectShardResults); converted.setShardToValue(fromObjectToThrift); } else { - Map<org.apache.blur.thrift.generated.Server, ValueObject> fromObjectToThrift = fromObjectToThrift(serverResults); + Map<Server, Object> supportedThriftObjectServerResults = toThriftSupportedObjects(serverResults, serDe); + Map<org.apache.blur.thrift.generated.Server, ValueObject> fromObjectToThrift = fromObjectToThrift(supportedThriftObjectServerResults); converted.setServerToValue(fromObjectToThrift); } } return converted; } + public static <T> Map<T, Object> toThriftSupportedObjects(Map<T, Object> map, BlurObjectSerDe serDe) { + Map<T, Object> results = new HashMap<T, Object>(); + for (Entry<T, Object> e : map.entrySet()) { + Object supportedThriftObject = serDe.toSupportedThriftObject(e.getValue()); + results.put(e.getKey(), supportedThriftObject); + } + return results; + } + + public static <T> Map<T, Object> fromThriftSupportedObjects(Map<T, Object> map, BlurObjectSerDe serDe) { + Map<T, Object> results = new HashMap<T, Object>(); + for (Entry<T, Object> e : map.entrySet()) { + Object fromSupportedThriftObject = serDe.fromSupportedThriftObject(e.getValue()); + results.put(e.getKey(), fromSupportedThriftObject); + } + return results; + } + @SuppressWarnings("unchecked") public static <T, R> Map<R, ValueObject> fromObjectToThrift(Map<T, Object> map) throws BlurException { Map<R, ValueObject> result = new HashMap<R, ValueObject>(); @@ -109,8 +134,6 @@ public class CommandUtil { valueObject.setValue(toValue(o)); } else if (o instanceof BlurObject || o instanceof BlurArray) { valueObject.setBlurObject(ObjectArrayPacking.pack(o)); - } else if (o instanceof Set) { - throw new RuntimeException("Not implemented."); } else { valueObject.setValue(toValue(o)); } @@ -188,14 +211,22 @@ public class CommandUtil { } } - public static Arguments toArguments(Command<?> command) { + public static Arguments toArguments(Command<?> command, BlurObjectSerDe serde) throws BlurException { Class<?> clazz = command.getClass(); + Map<String, Object> args = new HashMap<String, Object>(); + addArguments(clazz, args, command); + BlurObject blurObject = serde.serialize(args); Arguments arguments = new Arguments(); - addArguments(clazz, arguments, command); + Iterator<String> keys = blurObject.keys(); + while (keys.hasNext()) { + String key = keys.next(); + Object object = blurObject.get(key); + arguments.putToValues(key, toValueObject(object)); + } return arguments; } - private static void addArguments(Class<?> clazz, Arguments arguments, Command<?> command) { + private static void addArguments(Class<?> clazz, Map<String, Object> arguments, Command<?> command) { if (!(clazz.equals(Command.class))) { addArguments(clazz.getSuperclass(), arguments, command); } @@ -208,7 +239,7 @@ public class CommandUtil { try { Object o = field.get(command); if (o != null) { - arguments.putToValues(name, CommandUtil.toValueObject(o)); + arguments.put(name, o); } else { throw new IllegalArgumentException("Field [" + name + "] is required."); } @@ -216,8 +247,6 @@ public class CommandUtil { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); - } catch (BlurException e) { - throw new RuntimeException(e); } } @@ -228,14 +257,12 @@ public class CommandUtil { try { Object o = field.get(command); if (o != null) { - arguments.putToValues(name, CommandUtil.toValueObject(o)); + arguments.put(name, o); } } catch (IllegalArgumentException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); - } catch (BlurException e) { - throw new RuntimeException(e); } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/command/ControllerClusterContext.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/ControllerClusterContext.java b/blur-core/src/main/java/org/apache/blur/command/ControllerClusterContext.java index 3e53067..29c4c70 100644 --- a/blur-core/src/main/java/org/apache/blur/command/ControllerClusterContext.java +++ b/blur-core/src/main/java/org/apache/blur/command/ControllerClusterContext.java @@ -53,6 +53,7 @@ public class ControllerClusterContext extends ClusterContext implements Closeabl private final Map<Server, Client> _clientMap; private final ControllerCommandManager _manager; private final LayoutFactory _layoutFactory; + private final BlurObjectSerDe _serDe = new BlurObjectSerDe(); public ControllerClusterContext(TableContextFactory tableContextFactory, LayoutFactory layoutFactory, ControllerCommandManager manager) throws IOException { @@ -121,7 +122,8 @@ public class ControllerClusterContext extends ClusterContext implements Closeabl @Override public Map<Shard, T> call() throws Exception { Response response = waitForResponse(client, command, arguments); - Map<Shard, Object> shardToValue = CommandUtil.fromThriftToObjectShard(response.getShardToValue()); + Map<Shard, Object> shardToThriftValue = CommandUtil.fromThriftToObjectShard(response.getShardToValue()); + Map<Shard, Object> shardToValue = CommandUtil.fromThriftSupportedObjects(shardToThriftValue, _serDe); return (Map<Shard, T>) shardToValue; } }); @@ -207,7 +209,8 @@ public class ControllerClusterContext extends ClusterContext implements Closeabl public T call() throws Exception { Response response = waitForResponse(client, command, arguments); ValueObject valueObject = response.getValue(); - return (T) CommandUtil.toObject(valueObject); + Object thriftObject = CommandUtil.toObject(valueObject); + return (T) _serDe.fromSupportedThriftObject(thriftObject); } }); futureMap.put(server, future); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index 32967a1..477e0d0 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.blur.command.ArgumentOverlay; import org.apache.blur.command.BlurObject; +import org.apache.blur.command.BlurObjectSerDe; import org.apache.blur.command.CommandUtil; import org.apache.blur.command.ControllerCommandManager; import org.apache.blur.command.ExecutionId; @@ -208,6 +209,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { private Timer _tableContextWarmupTimer; private long _tableLayoutTimeoutNanos = TimeUnit.SECONDS.toNanos(30); private ControllerCommandManager _commandManager; + private BlurObjectSerDe _serDe = new BlurObjectSerDe(); public void init() throws KeeperException, InterruptedException { setupZookeeper(); @@ -1518,8 +1520,9 @@ public class BlurControllerServer extends TableAdmin implements Iface { throws BlurException, TException { try { BlurObject args = CommandUtil.toBlurObject(arguments); - Response response = _commandManager.execute(getTableContextFactory(), getLayoutFactory(), commandName, new ArgumentOverlay(args)); - return CommandUtil.fromObjectToThrift(response); + Response response = _commandManager.execute(getTableContextFactory(), getLayoutFactory(), commandName, + new ArgumentOverlay(args, _serDe)); + return CommandUtil.fromObjectToThrift(response, _serDe); } catch (Exception e) { if (e instanceof org.apache.blur.command.TimeoutException) { throw new TimeoutException(((org.apache.blur.command.TimeoutException) e).getExecutionId().getId()); @@ -1624,7 +1627,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { TimeoutException, TException { try { Response response = _commandManager.reconnect(new ExecutionId(executionId)); - return CommandUtil.fromObjectToThrift(response); + return CommandUtil.fromObjectToThrift(response, _serDe); } catch (Exception e) { if (e instanceof org.apache.blur.command.TimeoutException) { throw new TimeoutException(((org.apache.blur.command.TimeoutException) e).getExecutionId().getId()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index c323d4e..c6e2a2f 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.command.ArgumentOverlay; import org.apache.blur.command.BlurObject; +import org.apache.blur.command.BlurObjectSerDe; import org.apache.blur.command.CommandUtil; import org.apache.blur.command.ExecutionId; import org.apache.blur.command.Response; @@ -90,6 +91,7 @@ public class BlurShardServer extends TableAdmin implements Iface { private String _cluster = BlurConstants.BLUR_CLUSTER; private int _dataFetchThreadCount = 32; private ShardCommandManager _commandManager; + private BlurObjectSerDe _serDe = new BlurObjectSerDe(); public void init() throws BlurException { _queryCache = new QueryCache("shard-cache", _maxQueryCacheElements, _maxTimeToLive); @@ -610,8 +612,8 @@ public class BlurShardServer extends TableAdmin implements Iface { } }; BlurObject args = CommandUtil.toBlurObject(arguments); - Response response = _commandManager.execute(tableContextFactory, commandName, new ArgumentOverlay(args)); - return CommandUtil.fromObjectToThrift(response); + Response response = _commandManager.execute(tableContextFactory, commandName, new ArgumentOverlay(args, _serDe)); + return CommandUtil.fromObjectToThrift(response, _serDe); } catch (Exception e) { if (e instanceof org.apache.blur.command.TimeoutException) { throw new TimeoutException(((org.apache.blur.command.TimeoutException) e).getExecutionId().getId()); @@ -633,7 +635,7 @@ public class BlurShardServer extends TableAdmin implements Iface { TimeoutException, TException { try { Response response = _commandManager.reconnect(new ExecutionId(executionId)); - return CommandUtil.fromObjectToThrift(response); + return CommandUtil.fromObjectToThrift(response, _serDe); } catch (Exception e) { if (e instanceof org.apache.blur.command.TimeoutException) { throw new TimeoutException(((org.apache.blur.command.TimeoutException) e).getExecutionId().getId()); @@ -650,7 +652,7 @@ public class BlurShardServer extends TableAdmin implements Iface { public void refresh() throws TException { ShardServerContext.resetSearchers(); } - + @Override public List<CommandDescriptor> listInstalledCommands() throws BlurException, TException { try { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/89e10a0d/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index f8ef2e5..29e3e8b 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -134,7 +134,7 @@ public class ShardCommandManagerTest { { BlurObject args = new BlurObject(); args.put("table", "test"); - ArgumentOverlay argumentOverlay = new ArgumentOverlay(args); + ArgumentOverlay argumentOverlay = new ArgumentOverlay(args, new BlurObjectSerDe()); Response response = manager.execute(getTableContextFactory(), "test", argumentOverlay); Map<Shard, Object> shardResults = response.getShardResults(); for (Object o : shardResults.values()) { @@ -156,7 +156,7 @@ public class ShardCommandManagerTest { { BlurObject args = new BlurObject(); args.put("table", "test"); - ArgumentOverlay argumentOverlay = new ArgumentOverlay(args); + ArgumentOverlay argumentOverlay = new ArgumentOverlay(args, new BlurObjectSerDe()); Response response = manager.execute(getTableContextFactory(), "test", argumentOverlay); Map<Shard, Object> shardResults = response.getShardResults(); for (Object o : shardResults.values()) { @@ -189,7 +189,7 @@ public class ShardCommandManagerTest { args.put("table", "test"); args.put("seconds", 5); - ArgumentOverlay argumentOverlay = new ArgumentOverlay(args); + ArgumentOverlay argumentOverlay = new ArgumentOverlay(args, new BlurObjectSerDe()); long start = System.nanoTime(); while (true) { @@ -216,7 +216,7 @@ public class ShardCommandManagerTest { BlurObject args = new BlurObject(); args.put("table", "test"); args.put("seconds", 1); - ArgumentOverlay argumentOverlay = new ArgumentOverlay(args); + ArgumentOverlay argumentOverlay = new ArgumentOverlay(args, new BlurObjectSerDe()); TableContextFactory tableContextFactory = getTableContextFactory(); try { _manager.execute(tableContextFactory, "error", argumentOverlay); @@ -237,7 +237,7 @@ public class ShardCommandManagerTest { args.put("table", "test"); args.put("seconds", 5); - ArgumentOverlay argumentOverlay = new ArgumentOverlay(args); + ArgumentOverlay argumentOverlay = new ArgumentOverlay(args, new BlurObjectSerDe()); try { TableContextFactory tableContextFactory = getTableContextFactory();
