Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 e1c242b85 -> b6cdb2cce
An initial implementation of a SolrJ SolrServer for Blur Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b6cdb2cc Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b6cdb2cc Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b6cdb2cc Branch: refs/heads/apache-blur-0.2 Commit: b6cdb2cce4ada876b92393e62b5174ae8e5b2a52 Parents: e1c242b Author: twilliams <[email protected]> Authored: Fri Feb 28 22:43:13 2014 -0500 Committer: twilliams <[email protected]> Committed: Fri Feb 28 22:43:13 2014 -0500 ---------------------------------------------------------------------- whiteboard/slur/README.txt | 39 ++++ whiteboard/slur/pom.xml | 64 +++++ .../org/apache/blur/slur/RowMutationHelper.java | 114 +++++++++ .../apache/blur/slur/SolrLookingBlurServer.java | 231 +++++++++++++++++++ .../apache/blur/slur/RowMutationHelperTest.java | 116 ++++++++++ .../blur/slur/SolrLookingBlurServerTest.java | 197 ++++++++++++++++ 6 files changed, 761 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/README.txt ---------------------------------------------------------------------- diff --git a/whiteboard/slur/README.txt b/whiteboard/slur/README.txt new file mode 100644 index 0000000..8b93c1d --- /dev/null +++ b/whiteboard/slur/README.txt @@ -0,0 +1,39 @@ +== Overview == +This project explores using SolrJ to work with a Blur installation. + +== Mismatches == +o) Commits - Blur commits on an update; Solr expects and explicit - rather + than buffering, I've chosen to go ahead and follow Blur's model. +o) Row/SolrInputDocument - The parent document in Solr seems to be a "real" + document, with fields. So far, you can either have subdocuments or fields + on the main document, but not both. +o) Optimize - Solr offers waitFlush, waitSearcher and Blur just offers maxSegments. + +== Usage == + + = Create a server = + SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table); + + = Add a document = + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "1"); + doc.addField("fam.value", "123"); + + server.add(doc); + + = Or add in batches = + List<SolrInputDocument> docs = Lists.newArrayList(); + + for (int i = 0; i < 100; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", i); + doc.addField("fam.value", "12" + i); + docs.add(doc); + } + + server.add(docs); + + + == Notes == + **caveate is that I don't have experience with SolrJ, so this may very + well be dangerous. http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/pom.xml ---------------------------------------------------------------------- diff --git a/whiteboard/slur/pom.xml b/whiteboard/slur/pom.xml new file mode 100644 index 0000000..3a03daa --- /dev/null +++ b/whiteboard/slur/pom.xml @@ -0,0 +1,64 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.blur</groupId> + <artifactId>Slur</artifactId> + <version>0.0.1-SNAPSHOT</version> + <name>Slur</name> + <description>A SolrJ implementation for Blur.</description> + <dependencies> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>4.6.1</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-core</artifactId> + <version>0.2.2-incubating-SNAPSHOT</version> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-util</artifactId> + <version>0.2.2-incubating-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-thrift</artifactId> + <version>0.2.2-incubating-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava-collections</artifactId> + <version>r03</version> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-core</artifactId> + <version>0.2.2-incubating-SNAPSHOT</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-util</artifactId> + <version>0.2.2-incubating-SNAPSHOT</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>1.2.1</version> + <scope>test</scope> + </dependency> + + + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java ---------------------------------------------------------------------- diff --git a/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java b/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java new file mode 100644 index 0000000..80549e6 --- /dev/null +++ b/whiteboard/slur/src/main/java/org/apache/blur/slur/RowMutationHelper.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.slur; + +import java.util.Collection; +import java.util.List; + +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RecordMutationType; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; + +import com.google.common.collect.Lists; + +public class RowMutationHelper { + + public static List<RowMutation> from(Collection<SolrInputDocument> docs, String table) { + List<RowMutation> mutations = Lists.newArrayList(); + for(SolrInputDocument d: docs) { + System.out.println("DOC: " + d.getFieldValue("id")); + mutations.add(from(d, table)); + } + return mutations; + } + + public static RowMutation from(SolrInputDocument doc, String table) { + validate(doc); + + RowMutation mutate = new RowMutation(); + mutate.setRowId(extractRowId(doc)); + mutate.setTable(table); + List<RecordMutation> recordMutations = Lists.newArrayList(); + if (doc.hasChildDocuments()) { + + } else { + recordMutations.add(createRecordMutation(doc, extractRowId(doc))); + } + mutate.setRecordMutations(recordMutations); + return mutate; + } + + private static String extractRowId(SolrInputDocument doc) { + Object id = doc.getFieldValue("id"); + if (id == null) { + id = doc.getFieldValue("rowid"); + } + if (id == null) { + throw new IllegalArgumentException("Document must either have id or rowid field."); + } + return id.toString(); + } + + private static RecordMutation createRecordMutation(SolrInputDocument doc, String id) { + RecordMutation recordMutation = new RecordMutation(); + // TODO: what's solr default behavior? + recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD); + Record record = new Record(); + record.setFamily(findFamily(doc)); + record.setRecordId(id); + + for (String fieldName : doc.getFieldNames()) { + if (!fieldName.contains(".")) { + continue; + } + SolrInputField field = doc.getField(fieldName); + String rawColumnName = fieldName.substring(fieldName.indexOf(".") + 1, fieldName.length()); + + if (field.getValueCount() > 1) { + for (Object fieldVal : field.getValues()) { + record.addToColumns(new Column(rawColumnName, fieldVal.toString())); + } + } else { + record.addToColumns(new Column(rawColumnName, field.getFirstValue().toString())); + } + } + recordMutation.setRecord(record); + return recordMutation; + } + + private static String findFamily(SolrInputDocument doc) { + for (String name : doc.getFieldNames()) { + if (name.contains(".")) { + return name.substring(0, name.indexOf(".")); + } + } + throw new IllegalArgumentException("Unable to determine column family from document"); + } + + private static void validate(SolrInputDocument doc) { + if ((doc.getFieldNames().size() > 1) && (doc.hasChildDocuments())) { + + } + + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java ---------------------------------------------------------------------- diff --git a/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java b/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java new file mode 100644 index 0000000..db2244d --- /dev/null +++ b/whiteboard/slur/src/main/java/org/apache/blur/slur/SolrLookingBlurServer.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.slur; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.BlurClientManager; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.StreamingResponseCallback; +import org.apache.solr.client.solrj.beans.DocumentObjectBinder; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrPingResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; + +import com.google.common.collect.Lists; + +public class SolrLookingBlurServer extends SolrServer { + private static final long serialVersionUID = -7141309168500300896L; + private static final int DEFAULT_OPTIMIZE_MAX_SEGMENTS = 3; + private String connectionString; + private String tableName; + + public SolrLookingBlurServer(String controllerConnectionStr, String table) { + connectionString = controllerConnectionStr; + tableName = table; + } + + @Override + public NamedList<Object> request(SolrRequest arg0) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public void shutdown() { + // Nothing to do + } + + @Override + public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, + IOException { + return add(docs); + } + + @Override + public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException { + UpdateResponse response = new UpdateResponse(); + + try { + long start = System.currentTimeMillis(); + if (docs.size() == 1) { + client().mutate(RowMutationHelper.from(docs.iterator().next(), tableName)); + } else { + client().mutateBatch(RowMutationHelper.from(docs, tableName)); + } + response.setElapsedTime((System.currentTimeMillis() - start)); + } catch (Exception e) { + throw new SolrServerException(e); + } + + return response; + } + + @Override + public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException { + return add(doc); + } + + @Override + public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException { + return add(Lists.newArrayList(new SolrInputDocument[] { doc })); + } + + @Override + public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse addBean(Object obj) throws IOException, SolrServerException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse addBeans(Collection<?> arg0, int arg1) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse commit() throws SolrServerException, IOException { + return new UpdateResponse(); + } + + @Override + public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, + IOException { + return new UpdateResponse(); + } + + @Override + public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + return new UpdateResponse(); + } + + @Override + public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse deleteById(String id) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public DocumentObjectBinder getBinder() { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse optimize() throws SolrServerException, IOException { + return optimize(true, true, DEFAULT_OPTIMIZE_MAX_SEGMENTS); + } + + @Override + public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, + IOException { + long start = System.currentTimeMillis(); + try { + client().optimize(tableName, maxSegments); + } catch (Exception e) { + throw new SolrServerException(e); + } + UpdateResponse response = new UpdateResponse(); + response.setElapsedTime((System.currentTimeMillis() - start)); + return response; + } + + @Override + public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException { + return optimize(waitFlush, waitSearcher, DEFAULT_OPTIMIZE_MAX_SEGMENTS); + } + + @Override + public SolrPingResponse ping() throws SolrServerException, IOException { + SolrPingResponse response = new SolrPingResponse(); + long start = System.currentTimeMillis(); + try { + client().ping(); + } catch (TException e) { + throw new SolrServerException(e); + } + response.setElapsedTime((System.currentTimeMillis()-start)); + return response; + } + + @Override + public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public QueryResponse query(SolrParams params) throws SolrServerException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback callback) + throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + @Override + public UpdateResponse rollback() throws SolrServerException, IOException { + throw new RuntimeException("Not Implemented."); + } + + private Iface client() { + return BlurClient.getClient(connectionString); + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java ---------------------------------------------------------------------- diff --git a/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java b/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java new file mode 100644 index 0000000..88a8b30 --- /dev/null +++ b/whiteboard/slur/src/test/java/org/apache/blur/slur/RowMutationHelperTest.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.slur; + +import static org.junit.Assert.*; + +import java.util.List; + +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.solr.common.SolrInputDocument; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class RowMutationHelperTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void basicOneForOneConversion() { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("fam.key", "value"); + doc.addField("id", "123"); + + RowMutation mutate = RowMutationHelper.from(doc, "foo"); + + assertEquals("Should get our rowid back.", "123", mutate.getRowId()); + assertEquals("Should get a single record.", 1, mutate.getRecordMutationsSize()); + assertEquals("Should get a simple value back.", "value", getRecordValue("key", mutate)); + assertEquals("Should properly figure our family.", "fam", getFirstRecord(mutate).getFamily()); + assertEquals("Tablename should be set", "foo", mutate.getTable()); + } + + @Test + public void multivalueFieldsShouldTranslate() { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "123"); + doc.addField("fam.key", "value1"); + doc.addField("fam.key", "value2"); + + RowMutation mutate = RowMutationHelper.from(doc, "foo"); + + List<Object> vals = getRecordValues("key", mutate); + assertEquals("Should get both values back.", 2, vals.size()); + + assertEquals("value1", vals.get(0).toString()); + assertEquals("value2", vals.get(1).toString()); + } + + @Test(expected = IllegalArgumentException.class) + public void docWithChildrenCantItselfHaveFieldValues() { + SolrInputDocument parent = new SolrInputDocument(); + parent.addField("fam.key1", "123"); + SolrInputDocument child = new SolrInputDocument(); + parent.addChildDocument(child); + + RowMutationHelper.from(parent, "foo"); + } + + @Test(expected = IllegalArgumentException.class) + public void docsMustUseTheNormalBlurFamilyColumnFormat() { + SolrInputDocument parent = new SolrInputDocument(); + parent.addField("columnWithoutFamily", "123"); + SolrInputDocument child = new SolrInputDocument(); + parent.addChildDocument(child); + + RowMutationHelper.from(parent, "foo"); + } + + private Object getRecordValue(String field, RowMutation mutate) { + return getRecordValues(field, mutate).get(0); + } + + private List<Object> getRecordValues(String field, RowMutation mutate) { + List<Object> vals = Lists.newArrayList(); + Record rec = getFirstRecord(mutate); + + for (Column col : rec.getColumns()) { + + if (col.getName().equals(field)) { + vals.add(col.getValue()); + } + } + return vals; + } + + private Record getFirstRecord(RowMutation mutate) { + return mutate.getRecordMutations().get(0).getRecord(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b6cdb2cc/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java ---------------------------------------------------------------------- diff --git a/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java b/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java new file mode 100644 index 0000000..401cdd7 --- /dev/null +++ b/whiteboard/slur/src/test/java/org/apache/blur/slur/SolrLookingBlurServerTest.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.slur; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.blur.MiniCluster; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.thrift.generated.TableStats; +import org.apache.blur.utils.GCWatcher; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Lists; + +public class SolrLookingBlurServerTest { + private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest")); + private static MiniCluster miniCluster; + private static String connectionStr; + + @BeforeClass + public static void startCluster() throws IOException { + GCWatcher.init(0.60); + LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); + File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); + testDirectory.mkdirs(); + + Path directory = new Path(testDirectory.getPath()); + FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); + FsAction userAction = dirPermissions.getUserAction(); + FsAction groupAction = dirPermissions.getGroupAction(); + FsAction otherAction = dirPermissions.getOtherAction(); + + StringBuilder builder = new StringBuilder(); + builder.append(userAction.ordinal()); + builder.append(groupAction.ordinal()); + builder.append(otherAction.ordinal()); + String dirPermissionNum = builder.toString(); + System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); + testDirectory.delete(); + miniCluster = new MiniCluster(); + miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true); + connectionStr = miniCluster.getControllerConnectionStr(); + + } + + @AfterClass + public static void shutdownCluster() { + miniCluster.shutdownBlurCluster(); + } + + @Test + public void addedDocumentShouldShowUpInBlur() throws SolrServerException, IOException, BlurException, TException { + String table = "addedDocumentShouldShowUpInBlur"; + createTable(table); + SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table); + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "1"); + doc.addField("fam.value", "123"); + + server.add(doc); + + TableStats stats = client().tableStats(table); + + assertEquals("We should have one record.", 1, stats.recordCount); + assertEquals("We should have one row.", 1, stats.rowCount); + + assertTotalResults(table, "fam.value:123", 1l); + + removeTable(table); + } + + @Test + public void docShouldBeDiscoverableWithMultiValuedFields() throws SolrServerException, IOException, BlurException, + TException { + String table = "docShouldBeDiscoverableWithMultiValuedFields"; + createTable(table); + SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table); + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "1"); + doc.addField("fam.value", "123"); + doc.addField("fam.value", "124"); + + server.add(doc); + + assertTotalResults(table, "fam.value:123", 1l); + assertTotalResults(table, "fam.value:124", 1l); + assertTotalResults(table, "fam.value:justincase", 0l); + + removeTable(table); + } + + @Test + public void documentsShouldBeAbleToBeIndexedInBatch() throws SolrServerException, IOException, BlurException, + TException { + String table = "multipleDocumentsShouldBeIndexed"; + createTable(table); + + SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table); + + List<SolrInputDocument> docs = Lists.newArrayList(); + + for (int i = 0; i < 100; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", i); + doc.addField("fam.value", "12" + i); + docs.add(doc); + } + + server.add(docs); + + assertTotalResults(table, "fam.value:123", 1l); + assertTotalResults(table, "fam.value:124", 1l); + assertTotalResults(table, "rowid:1", 1l); + assertTotalResults(table, "rowid:2", 1l); + assertTotalResults(table, "fam.value:1299", 1l); + assertTotalResults(table, "fam.value:justincase", 0l); + + removeTable(table); + } + + private void assertTotalResults(String table, String q, long expected) throws BlurException, TException { + BlurQuery bquery = new BlurQuery(); + Query query = new Query(); + query.setQuery(q); + bquery.setQuery(query); + BlurResults results = client().query(table, bquery); + + assertEquals("Should find our row.", expected, results.getTotalResults()); + } + + private void removeTable(String table) { + try { + client().disableTable(table); + } catch (Exception e) { + + } + try { + client().removeTable(table, true); + } catch (Exception e) { + + } + } + + private static void createTable(String tableName) throws BlurException, TException, IOException { + Blur.Iface client = client(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(tableName); + tableDescriptor.setShardCount(5); + tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName); + client.createTable(tableDescriptor); + List<String> tableList = client.tableList(); + assertTrue(tableList.contains(tableName)); + } + + private static Iface client() { + return BlurClient.getClient(connectionStr); + } + +}
