Remove old solr module and Rename solr5 module into solr module
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/3b2f282d Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/3b2f282d Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/3b2f282d Branch: refs/heads/master Commit: 3b2f282dd68e6e3bae74c43fdd0b42039a9193d1 Parents: c506f44 Author: madhawa-gunasekara <[email protected]> Authored: Tue May 30 01:18:37 2017 +0530 Committer: madhawa-gunasekara <[email protected]> Committed: Sun Jun 4 20:16:48 2017 +0530 ---------------------------------------------------------------------- gora-solr-5/pom.xml | 320 ---- .../java/org/apache/gora/solr/package-info.java | 20 - .../org/apache/gora/solr/query/SolrQuery.java | 80 - .../org/apache/gora/solr/query/SolrResult.java | 121 -- .../apache/gora/solr/query/package-info.java | 21 - .../org/apache/gora/solr/store/SolrMapping.java | 58 - .../org/apache/gora/solr/store/SolrStore.java | 841 ----------- .../apache/gora/solr/store/package-info.java | 20 - gora-solr-5/src/test/conf/gora-solr-mapping.xml | 40 - gora-solr-5/src/test/conf/gora.properties | 21 - gora-solr-5/src/test/conf/log4j.properties | 35 - .../Employee/conf/elevate.xml | 42 - .../Employee/conf/lang/stopwords_en.txt | 54 - .../Employee/conf/managed-schema | 33 - .../Employee/conf/protwords.txt | 21 - .../Employee/conf/solrconfig.xml | 1411 ------------------ .../Employee/conf/stopwords.txt | 14 - .../Employee/conf/synonyms.txt | 29 - .../WebPage/conf/elevate.xml | 42 - .../WebPage/conf/lang/stopwords_en.txt | 54 - .../WebPage/conf/managed-schema | 34 - .../WebPage/conf/protwords.txt | 21 - .../WebPage/conf/solrconfig.xml | 1411 ------------------ .../WebPage/conf/stopwords.txt | 14 - .../WebPage/conf/synonyms.txt | 29 - .../collection1/conf/elevate.xml | 42 - .../collection1/conf/managed-schema | 26 - .../collection1/conf/solrconfig.xml | 1403 ----------------- .../collection1/core.properties | 18 - .../src/test/conf/solr-managed-schema/solr.xml | 53 - .../src/test/conf/solr-managed-schema/zoo.cfg | 35 - .../solr/Employee/conf/lang/stopwords_en.txt | 54 - .../test/conf/solr/Employee/conf/protwords.txt | 21 - .../src/test/conf/solr/Employee/conf/schema.xml | 45 - .../test/conf/solr/Employee/conf/solrconfig.xml | 107 -- .../test/conf/solr/Employee/conf/stopwords.txt | 14 - .../test/conf/solr/Employee/conf/synonyms.txt | 29 - .../solr/WebPage/conf/lang/stopwords_en.txt | 54 - .../test/conf/solr/WebPage/conf/protwords.txt | 21 - .../src/test/conf/solr/WebPage/conf/schema.xml | 46 - .../test/conf/solr/WebPage/conf/solrconfig.xml | 107 -- .../test/conf/solr/WebPage/conf/stopwords.txt | 14 - .../test/conf/solr/WebPage/conf/synonyms.txt | 29 - .../test/conf/solr/collection1/conf/schema.xml | 32 - .../conf/solr/collection1/conf/solrconfig.xml | 113 -- .../test/conf/solr/collection1/core.properties | 18 - gora-solr-5/src/test/conf/solr/solr.xml | 69 - gora-solr-5/src/test/conf/solr/zoo.cfg | 34 - .../solr/GoraSolrManagedSchemaTestDriver.java | 89 -- .../apache/gora/solr/GoraSolrTestDriver.java | 92 -- .../java/org/apache/gora/solr/package-info.java | 21 - .../solr/store/TestSolrManagedSchemaStore.java | 31 - .../apache/gora/solr/store/package-info.java | 21 - gora-solr/pom.xml | 90 +- .../java/org/apache/gora/solr/package-info.java | 20 + .../org/apache/gora/solr/query/SolrQuery.java | 22 +- .../org/apache/gora/solr/query/SolrResult.java | 22 +- .../apache/gora/solr/query/package-info.java | 21 + .../org/apache/gora/solr/store/SolrMapping.java | 3 + .../org/apache/gora/solr/store/SolrStore.java | 117 +- .../apache/gora/solr/store/package-info.java | 20 + .../Employee/conf/elevate.xml | 42 + .../Employee/conf/lang/stopwords_en.txt | 54 + .../Employee/conf/managed-schema | 33 + .../Employee/conf/protwords.txt | 21 + .../Employee/conf/solrconfig.xml | 1411 ++++++++++++++++++ .../Employee/conf/synonyms.txt | 29 + .../WebPage/conf/elevate.xml | 42 + .../WebPage/conf/lang/stopwords_en.txt | 54 + .../WebPage/conf/managed-schema | 34 + .../WebPage/conf/protwords.txt | 21 + .../WebPage/conf/solrconfig.xml | 1411 ++++++++++++++++++ .../WebPage/conf/synonyms.txt | 29 + .../collection1/conf/elevate.xml | 42 + .../collection1/conf/managed-schema | 26 + .../collection1/conf/solrconfig.xml | 1403 +++++++++++++++++ .../collection1/core.properties | 18 + .../src/test/conf/solr-managed-schema/solr.xml | 53 + .../src/test/conf/solr-managed-schema/zoo.cfg | 35 + .../test/conf/solr/Employee/conf/solrconfig.xml | 2 +- .../test/conf/solr/Employee/conf/stopwords.txt | 14 - .../src/test/conf/solr/Employee/core.properties | 1 - .../test/conf/solr/WebPage/conf/solrconfig.xml | 2 +- .../test/conf/solr/WebPage/conf/stopwords.txt | 14 - .../src/test/conf/solr/WebPage/core.properties | 1 - .../conf/solr/collection1/conf/solrconfig.xml | 2 +- .../test/conf/solr/collection1/core.properties | 17 + gora-solr/src/test/conf/solr/solr.xml | 30 +- gora-solr/src/test/conf/solr/zoo.cfg | 17 + .../solr/GoraSolrManagedSchemaTestDriver.java | 89 ++ .../apache/gora/solr/GoraSolrTestDriver.java | 20 +- .../java/org/apache/gora/solr/package-info.java | 21 + .../solr/store/TestSolrManagedSchemaStore.java | 31 + .../apache/gora/solr/store/TestSolrStore.java | 76 +- .../apache/gora/solr/store/package-info.java | 21 + pom.xml | 9 +- 96 files changed, 5318 insertions(+), 7546 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/pom.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/pom.xml b/gora-solr-5/pom.xml deleted file mode 100644 index 816d04a..0000000 --- a/gora-solr-5/pom.xml +++ /dev/null @@ -1,320 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.gora</groupId> - <artifactId>gora</artifactId> - <version>0.8-SNAPSHOT</version> - <relativePath>../</relativePath> - </parent> - <artifactId>gora-solr-5</artifactId> - <packaging>bundle</packaging> - - <name>Apache Gora :: Solr5</name> - - <properties> - <osgi.import>*</osgi.import> - <osgi.export>org.apache.gora.solr*;version="${project.version}";-noimport:=true</osgi.export> - <jetty.version>9.3.14.v20161028</jetty.version> - </properties> - - <build> - <directory>target</directory> - <outputDirectory>target/classes</outputDirectory> - <finalName>${project.artifactId}-${project.version}</finalName> - <testOutputDirectory>target/test-classes</testOutputDirectory> - <testSourceDirectory>src/test/java</testSourceDirectory> - <sourceDirectory>src/main/java</sourceDirectory> - <testResources> - <testResource> - <directory>${project.basedir}/src/test/conf/</directory> - <includes> - <include>**</include> - </includes> - </testResource> - </testResources> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>${build-helper-maven-plugin.version}</version> - <executions> - <execution> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>src/examples/java</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - <repositories> - <repository> - <id>maven-restlet</id> - <name>Public online Restlet repository</name> - <url>http://maven.restlet.org</url> - </repository> - </repositories> - - <dependencies> - <!-- Gora Internal Dependencies --> - <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-core</artifactId> - <exclusions> - <exclusion> - <artifactId>servlet-api</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <artifactId>servlet-api-2.5</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <artifactId>jetty-util</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <artifactId>jetty</artifactId> - <groupId>org.mortbay.jetty</groupId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.gora</groupId> - <artifactId>gora-core</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - - <!-- Logging Dependencies --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <exclusions> - <exclusion> - <groupId>javax.jms</groupId> - <artifactId>jms</artifactId> - </exclusion> - </exclusions> - </dependency> - - <!-- Solr Dependencies --> - <dependency> - <groupId>org.apache.solr</groupId> - <artifactId>solr-core</artifactId> - <version>${lucene-solr-5.version}</version> - <exclusions> - <exclusion> - <groupId>org.eclipse.jetty.orbit</groupId> - <artifactId>javax.servlet</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.solr</groupId> - <artifactId>solr-solrj</artifactId> - <version>${solr-solrj-5.version}</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tika</groupId> - <artifactId>tika-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tika</groupId> - <artifactId>tika-parsers</artifactId> - <exclusions> - <exclusion> - <groupId>com.adobe.xmp</groupId> - <artifactId>xmpcore</artifactId> - </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.geronimo.specs</groupId> - <artifactId>geronimo-stax-api_1.0_spec</artifactId> - </exclusion> - <exclusion> - <groupId>org.gagravarr</groupId> - <artifactId>vorbis-java-core</artifactId> - </exclusion> - <exclusion> - <groupId>asm</groupId> - <artifactId>asm</artifactId> - </exclusion> - <exclusion> - <groupId>org.aspectj</groupId> - <artifactId>aspectjrt</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.eclipse.jetty.orbit</groupId> - <artifactId>javax.servlet</artifactId> - </exclusion> - </exclusions> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-webapp</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-deploy</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-continuation</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-jmx</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-rewrite</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-security</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlets</artifactId> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-xml</artifactId> - <scope>runtime</scope> - </dependency> - - <!-- ADDED TO AVOID PROBLEMS WITH JAVAX --> - <dependency> - <groupId>javax</groupId> - <artifactId>javaee-api</artifactId> - <version>7.0</version> - </dependency> - - <!-- Testing Dependencies --> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> -<!-- <dependency> --> -<!-- <groupId>org.apache.solr</groupId> --> -<!-- <artifactId>solr-test-framework</artifactId> --> -<!-- <version>${lucene-solr-5.version}</version> --> -<!-- <exclusions> --> -<!-- <exclusion> --> -<!-- <groupId>org.eclipse.jetty</groupId> --> -<!-- <artifactId>jetty-servlet</artifactId> --> -<!-- </exclusion> --> -<!-- </exclusions> --> -<!-- <scope>test</scope> --> -<!-- </dependency> --> -<!-- <dependency> --> -<!-- <groupId>org.apache.lucene</groupId> --> -<!-- <artifactId>lucene-test-framework</artifactId> --> -<!-- <version>${lucene-solr-5.version}</version> --> -<!-- <scope>test</scope> --> -<!-- </dependency> --> - </dependencies> - - <url>http://gora.apache.org</url> - <description>The Apache Gora open source framework provides an in-memory data model and - persistence for big data. Gora supports persisting to column stores, key value stores, - document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce - support.</description> - <inceptionYear>2010</inceptionYear> - <organization> - <name>The Apache Software Foundation</name> - <url>http://apache.org</url> - </organization> - <issueManagement> - <system>JIRA</system> - <url>https://issues.apache.org/jira/browse/GORA</url> - </issueManagement> - <ciManagement> - <system>Jenkins</system> - <url>https://builds.apache.org/job/Gora-trunk/</url> - </ciManagement> -</project> http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/package-info.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/package-info.java b/gora-solr-5/src/main/java/org/apache/gora/solr/package-info.java deleted file mode 100644 index e5531c1..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This package contains Solr datastore related all classes. - */ -package org.apache.gora.solr; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java deleted file mode 100644 index 425f5c4..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrQuery.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gora.solr.query; - -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.query.Query; -import org.apache.gora.query.impl.QueryBase; -import org.apache.gora.solr.store.SolrMapping; -import org.apache.gora.solr.store.SolrStore; -import org.apache.gora.store.DataStore; - -/** - * Solr specific implementation of the {@link Query} interface. - */ -public class SolrQuery<K, T extends PersistentBase> extends QueryBase<K, T> { - SolrStore<K, T> store; - - /** - * Constructor for the query - */ - public SolrQuery() { - super(null); - store = null; - } - - /** - * Constructor for the query - * - * @param dataStore Data store used - * - */ - public SolrQuery(DataStore<K, T> dataStore) { - super(dataStore); - store = (SolrStore<K, T>)dataStore; - } - - /** - * Create a solr query - * - * @return the solr query string - */ - public String toSolrQuery() { - SolrMapping mapping = store.getMapping(); - String fld = mapping.getPrimaryKey(); - String q; - if (getKey() != null) { - q = fld + ":" + SolrStore.escapeQueryKey(getKey().toString()); - } else { - q = fld + ":["; - if (getStartKey() != null) { - q += SolrStore.escapeQueryKey(getStartKey().toString()); - } else { - q += "*"; - } - q += " TO "; - if (getEndKey() != null) { - q += SolrStore.escapeQueryKey(getEndKey().toString()); - } else { - q += "*"; - } - q += "]"; - } - return q; - } -} http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java deleted file mode 100644 index d3ef88a..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/query/SolrResult.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gora.solr.query; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; - -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.query.Query; -import org.apache.gora.query.impl.PartitionQueryImpl; -import org.apache.gora.query.impl.ResultBase; -import org.apache.gora.solr.store.SolrStore; -import org.apache.gora.store.DataStore; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; - -/** - * SolrResult specific implementation of the {@link org.apache.gora.query.Result} - * interface. - */ -public class SolrResult<K, T extends PersistentBase> extends ResultBase<K, T> { - - SolrDocumentList list = null; - SolrStore<K, T> store; - String[] fields; - int pos = 0; - - /** - * Constructor for the result set - * - * @param dataStore Data store used - * @param query Query used - * @param server A client that talks directly to a Solr server - * @param resultsSize The number of rows to be returned - */ - public SolrResult(DataStore<K, T> dataStore, Query<K, T> query, - SolrClient server, int resultsSize) throws IOException { - super(dataStore, query); - store = (SolrStore<K, T>)dataStore; - ModifiableSolrParams params = new ModifiableSolrParams(); - if (query instanceof PartitionQueryImpl) { - query = ((PartitionQueryImpl<K, T>)query).getBaseQuery(); - } - String q = ((SolrQuery<K, T>)query).toSolrQuery(); - params.set(CommonParams.Q, q); - fields = query.getFields(); - if (fields == null) { - params.set(CommonParams.FL, "*"); - } else { - HashSet<String> uniqFields = new HashSet<>(Arrays.asList(fields)); - String keyFld = ((SolrStore<K, T>)dataStore).getMapping().getPrimaryKey(); - uniqFields.add(keyFld); // return also primary key - StringBuilder sb = new StringBuilder(); - for (String f : uniqFields) { - if (sb.length() > 0) sb.append(','); - sb.append(f); - } - params.set(CommonParams.FL, sb.toString()); - } - params.set(CommonParams.ROWS, resultsSize); - try { - QueryResponse rsp = server.query(params); - list = rsp.getResults(); - } catch (SolrServerException e) { - throw new IOException(e); - } - } - - /** - * Gets the next item - */ - @SuppressWarnings("unchecked") - @Override - protected boolean nextInner() throws IOException { - if (list == null || pos >= list.size()) { - return false; - } - SolrDocument doc = list.get(pos++); - key = (K) doc.get(store.getMapping().getPrimaryKey()); - persistent = store.newInstance(doc, fields); - return true; - } - - @Override - public void close() throws IOException { - if (list != null) list.clear(); - } - - /** - * Gets the items reading progress - */ - @Override - public float getProgress() throws IOException { - if (list != null && list.size() > 0) { - return (float)pos / (float)list.size(); - } else { - return 0; - } - } -} http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/query/package-info.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/query/package-info.java b/gora-solr-5/src/main/java/org/apache/gora/solr/query/package-info.java deleted file mode 100644 index 92f6bf5..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/query/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This package contains all the Solr store query representation class as well as Result set representing class - * when query is executed over the Solr dataStore. - */ -package org.apache.gora.solr.query; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java deleted file mode 100644 index 635b426..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrMapping.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gora.solr.store; - -import java.util.HashMap; - -/** - * Mapping definitions for Solr. - */ -public class SolrMapping { - HashMap<String,String> mapping; - String coreName; - String primaryKey; - - public SolrMapping() { - mapping = new HashMap<>(); - } - - public void addField(String goraField, String solrField) { - mapping.put(goraField, solrField); - } - - public void setPrimaryKey(String solrKey) { - primaryKey = solrKey; - } - - public void setCoreName(String coreName) { - this.coreName = coreName; - } - - public String getCoreName() { - return coreName; - } - - public String getPrimaryKey() { - return primaryKey; - } - - public String getSolrField(String goraField) { - return mapping.get(goraField); - } - -} http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java deleted file mode 100644 index ac44a06..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/store/SolrStore.java +++ /dev/null @@ -1,841 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.gora.solr.store; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.avro.Schema; -import org.apache.avro.Schema.Field; -import org.apache.avro.Schema.Type; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.avro.util.Utf8; -import org.apache.gora.persistency.Persistent; -import org.apache.gora.persistency.impl.PersistentBase; -import org.apache.gora.query.PartitionQuery; -import org.apache.gora.query.Query; -import org.apache.gora.query.Result; -import org.apache.gora.query.impl.PartitionQueryImpl; -import org.apache.gora.solr.query.SolrQuery; -import org.apache.gora.solr.query.SolrResult; -import org.apache.gora.store.DataStoreFactory; -import org.apache.gora.store.impl.DataStoreBase; -import org.apache.gora.util.AvroUtils; -import org.apache.gora.util.IOUtils; -import org.apache.hadoop.util.StringUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.LBHttpSolrClient; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.jdom.Document; -import org.jdom.Element; -import org.jdom.input.SAXBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of a Solr data store to be used by gora. - * - * @param <K> class to be used for the key - * @param <T> class to be persisted within the store - */ -public class SolrStore<K, T extends PersistentBase> extends DataStoreBase<K, T> { - - private static final Logger LOG = LoggerFactory.getLogger(SolrStore.class); - - /** The default file name value to be used for obtaining the Solr object field mapping's */ - protected static final String DEFAULT_MAPPING_FILE = "gora-solr-mapping.xml"; - - /** The URL of the Solr server - defined in <code>gora.properties</code> */ - protected static final String SOLR_URL_PROPERTY = "solr.url"; - - /** The <code>solrconfig.xml</code> file to be used - defined in <code>gora.properties</code>*/ - protected static final String SOLR_CONFIG_PROPERTY = "solr.config"; - - /** The <code>schema.xml</code> file to be used - defined in <code>gora.properties</code>*/ - protected static final String SOLR_SCHEMA_PROPERTY = "solr.schema"; - - /** A batch size unit (ArrayList) of SolrDocument's to be used for writing to Solr. - * Should be defined in <code>gora.properties</code>. - * A default value of 100 is used if this value is absent. This value must be of type <b>Integer</b>. - */ - protected static final String SOLR_BATCH_SIZE_PROPERTY = "solr.batch_size"; - - /** The solrj implementation to use. This has a default value of <i>http</i> for HttpSolrClient. - * Available options include <b>http</b>, <b>cloud</b>, <b>concurrent</b> and <b>loadbalance</b>. - * Defined in <code>gora.properties</code> - * This value must be of type <b>String</b>. - */ - protected static final String SOLR_SOLRJSERVER_IMPL = "solr.solrjserver"; - - /** Whether to use secured Solr client or not. - * Available options include <b>true</b>, and <b>false</b>. - * Defined in <code>gora.properties</code> - * This value must be of type <b>boolean</b>. - */ - protected static final String SOLR_SERVER_USER_AUTH = "solr.solrjserver.user_auth"; - - /** Solr client username. - * Solr client user authentication should be enabled for this property. - * Defined in <code>gora.properties</code> - * This value must be of type <b>String</b>. - */ - protected static final String SOLR_SERVER_USERNAME = "solr.solrjserver.username"; - - /** Solr client password. - * Solr client user authentication should be enabled for this property. - * Defined in <code>gora.properties</code> - * This value must be of type <b>String</b>. - */ - protected static final String SOLR_SERVER_PASSWORD = "solr.solrjserver.password"; - - /** A batch commit unit for SolrDocument's used when making (commit) calls to Solr. - * Should be defined in <code>gora.properties</code>. - * A default value of 1000 is used if this value is absent. This value must be of type <b>Integer</b>. - */ - protected static final String SOLR_COMMIT_WITHIN_PROPERTY = "solr.commit_within"; - - /** The maximum number of result to return when we make a call to - * {@link org.apache.gora.solr.store.SolrStore#execute(Query)}. This should be - * defined in <code>gora.properties</code>. This value must be of type <b>Integer</b>. - */ - protected static final String SOLR_RESULTS_SIZE_PROPERTY = "solr.results_size"; - - /** The default batch size (ArrayList) of SolrDocuments to be used in the event of an absent - * value for <code>solr.batchSize</code>. - * Set to 100 by default. - */ - protected static final int DEFAULT_BATCH_SIZE = 100; - - /** The default commit size of SolrDocuments to be used in the event of an absent - * value for <code>solr.commitSize</code>. - * Set to 1000 by default. - */ - protected static final int DEFAULT_COMMIT_WITHIN = 1000; - - /** The default results size of SolrDocuments to be used in the event of an absent - * value for <code>solr.resultsSize</code>. - * Set to 100 by default. - */ - protected static final int DEFAULT_RESULTS_SIZE = 100; - - private SolrMapping mapping; - - private String SolrClientUrl, solrConfig, solrSchema, solrJServerImpl; - - private SolrClient server, adminServer; - - private boolean serverUserAuth; - - private String serverUsername; - - private String serverPassword; - - private ArrayList<SolrInputDocument> batch; - - private int batchSize = DEFAULT_BATCH_SIZE; - - private int commitWithin = DEFAULT_COMMIT_WITHIN; - - private int resultsSize = DEFAULT_RESULTS_SIZE; - - /** - * Default schema index with value "0" used when AVRO Union data types are - * stored - */ - public static final int DEFAULT_UNION_SCHEMA = 0; - - /* - * Create a threadlocal map for the datum readers and writers, because they - * are not thread safe, at least not before Avro 1.4.0 (See AVRO-650). When - * they are thread safe, it is possible to maintain a single reader and writer - * pair for every schema, instead of one for every thread. - */ - - public static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>(); - - public static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>(); - - /** - * Initialize the data store by reading the credentials, setting the client's properties up and - * reading the mapping file. Initialize is called when then the call to - * {@link org.apache.gora.store.DataStoreFactory#createDataStore} is made. - * - * @param keyClass - * @param persistentClass - * @param properties - */ - @Override - public void initialize(Class<K> keyClass, Class<T> persistentClass, - Properties properties) { - super.initialize(keyClass, persistentClass, properties); - try { - String mappingFile = DataStoreFactory.getMappingFile(properties, this, - DEFAULT_MAPPING_FILE); - mapping = readMapping(mappingFile); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - - SolrClientUrl = DataStoreFactory.findProperty(properties, this, - SOLR_URL_PROPERTY, null); - solrConfig = DataStoreFactory.findProperty(properties, this, - SOLR_CONFIG_PROPERTY, null); - solrSchema = DataStoreFactory.findProperty(properties, this, - SOLR_SCHEMA_PROPERTY, null); - solrJServerImpl = DataStoreFactory.findProperty(properties, this, - SOLR_SOLRJSERVER_IMPL, "http"); - serverUserAuth = DataStoreFactory.findBooleanProperty(properties, this, - SOLR_SERVER_USER_AUTH, "false"); - if (serverUserAuth) { - serverUsername = DataStoreFactory.findProperty(properties, this, - SOLR_SERVER_USERNAME, null); - serverPassword = DataStoreFactory.findProperty(properties, this, - SOLR_SERVER_PASSWORD, null); - } - LOG.info("Using Solr server at " + SolrClientUrl); - String solrJServerType = ((solrJServerImpl == null || solrJServerImpl.equals(""))?"http":solrJServerImpl); - // HttpSolrClient - denoted by "http" in properties - if (solrJServerType.toLowerCase(Locale.getDefault()).equals("http")) { - LOG.info("Using HttpSolrClient Solrj implementation."); - this.adminServer = new HttpSolrClient(SolrClientUrl); - this.server = new HttpSolrClient( SolrClientUrl + "/" + mapping.getCoreName() ); - if (serverUserAuth) { - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((HttpSolrClient) adminServer).getHttpClient(), - serverUsername, serverPassword); - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((HttpSolrClient) server).getHttpClient(), - serverUsername, serverPassword); - } - // CloudSolrClient - denoted by "cloud" in properties - } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("cloud")) { - LOG.info("Using CloudSolrClient Solrj implementation."); - this.adminServer = new CloudSolrClient(SolrClientUrl); - this.server = new CloudSolrClient( SolrClientUrl + "/" + mapping.getCoreName() ); - if (serverUserAuth) { - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((CloudSolrClient) adminServer).getLbClient().getHttpClient(), - serverUsername, serverPassword); - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((CloudSolrClient) server).getLbClient().getHttpClient(), - serverUsername, serverPassword); - } - } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("concurrent")) { - LOG.info("Using ConcurrentUpdateSolrClient Solrj implementation."); - this.adminServer = new ConcurrentUpdateSolrClient(SolrClientUrl, 1000, 10); - this.server = new ConcurrentUpdateSolrClient( SolrClientUrl + "/" + mapping.getCoreName(), 1000, 10); - // LBHttpSolrClient - denoted by "loadbalance" in properties - } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("loadbalance")) { - LOG.info("Using LBHttpSolrClient Solrj implementation."); - String[] solrUrlElements = StringUtils.split(SolrClientUrl); - try { - this.adminServer = new LBHttpSolrClient(solrUrlElements); - } catch (MalformedURLException e) { - LOG.error(e.getMessage()); - throw new RuntimeException(e); - } - try { - this.server = new LBHttpSolrClient( solrUrlElements + "/" + mapping.getCoreName() ); - } catch (MalformedURLException e) { - LOG.error(e.getMessage()); - throw new RuntimeException(e); - } - if (serverUserAuth) { - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((LBHttpSolrClient) adminServer).getHttpClient(), - serverUsername, serverPassword); - HttpClientUtil.setBasicAuth( - (DefaultHttpClient) ((LBHttpSolrClient) server).getHttpClient(), - serverUsername, serverPassword); - } - } - if (autoCreateSchema) { - createSchema(); - } - String batchSizeString = DataStoreFactory.findProperty(properties, this, - SOLR_BATCH_SIZE_PROPERTY, null); - if (batchSizeString != null) { - try { - batchSize = Integer.parseInt(batchSizeString); - } catch (NumberFormatException nfe) { - LOG.warn("Invalid batch size '{}', using default {}", batchSizeString, DEFAULT_BATCH_SIZE); - } - } - batch = new ArrayList<>(batchSize); - String commitWithinString = DataStoreFactory.findProperty(properties, this, - SOLR_COMMIT_WITHIN_PROPERTY, null); - if (commitWithinString != null) { - try { - commitWithin = Integer.parseInt(commitWithinString); - } catch (NumberFormatException nfe) { - LOG.warn("Invalid commit within '{}' , using default {}", commitWithinString, DEFAULT_COMMIT_WITHIN); - } - } - String resultsSizeString = DataStoreFactory.findProperty(properties, this, - SOLR_RESULTS_SIZE_PROPERTY, null); - if (resultsSizeString != null) { - try { - resultsSize = Integer.parseInt(resultsSizeString); - } catch (NumberFormatException nfe) { - LOG.warn("Invalid results size '{}' , using default {}", resultsSizeString, DEFAULT_RESULTS_SIZE); - } - } - } - - @SuppressWarnings("unchecked") - private SolrMapping readMapping(String filename) throws IOException { - SolrMapping map = new SolrMapping(); - try { - SAXBuilder builder = new SAXBuilder(); - Document doc = builder.build(getClass().getClassLoader() - .getResourceAsStream(filename)); - - List<Element> classes = doc.getRootElement().getChildren("class"); - - for (Element classElement : classes) { - if (classElement.getAttributeValue("keyClass").equals( - keyClass.getCanonicalName()) - && classElement.getAttributeValue("name").equals( - persistentClass.getCanonicalName())) { - - String tableName = getSchemaName( - classElement.getAttributeValue("table"), persistentClass); - map.setCoreName(tableName); - - Element primaryKeyEl = classElement.getChild("primarykey"); - map.setPrimaryKey(primaryKeyEl.getAttributeValue("column")); - - List<Element> fields = classElement.getChildren("field"); - - for (Element field : fields) { - String fieldName = field.getAttributeValue("name"); - String columnName = field.getAttributeValue("column"); - map.addField(fieldName, columnName); - } - break; - } - LOG.warn("Check that 'keyClass' and 'name' parameters in gora-solr-mapping.xml " - + "match with intended values. A mapping mismatch has been found therefore " - + "no mapping has been initialized for class mapping at position " - + " {} in mapping file.", classes.indexOf(classElement)); - } - } catch (Exception ex) { - throw new IOException(ex); - } - - return map; - } - - public SolrMapping getMapping() { - return mapping; - } - - @Override - public String getSchemaName() { - return mapping.getCoreName(); - } - - @Override - public void createSchema() { - try { - if (!schemaExists()) - CoreAdminRequest.createCore(mapping.getCoreName(), - mapping.getCoreName(), adminServer, solrConfig, solrSchema); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - - @Override - /** Default implementation deletes and recreates the schema*/ - public void truncateSchema() { - try { - server.deleteByQuery("*:*"); - server.commit(); - } catch (Exception e) { - // ignore? - LOG.error(e.getMessage(), e); - } - } - - @Override - public void deleteSchema() { - // XXX should this be only in truncateSchema ??? - try { - server.deleteByQuery("*:*"); - server.commit(); - } catch (Exception e) { - // ignore? - // LOG.error(e.getMessage(), e); - } - try { - CoreAdminRequest.unloadCore(mapping.getCoreName(), adminServer); - } catch (Exception e) { - if (e.getMessage().contains("No such core")) { - return; // it's ok, the core is not there - } else { - LOG.error(e.getMessage(), e); - } - } - } - - @Override - public boolean schemaExists() { - boolean exists = false; - try { - CoreAdminResponse rsp = CoreAdminRequest.getStatus(mapping.getCoreName(), - adminServer); - exists = rsp.getUptime(mapping.getCoreName()) != null; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - return exists; - } - - private static final String toDelimitedString(String[] arr, String sep) { - if (arr == null || arr.length == 0) { - return ""; - } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < arr.length; i++) { - if (i > 0) - sb.append(sep); - sb.append(arr[i]); - } - return sb.toString(); - } - - public static String escapeQueryKey(String key) { - if (key == null) { - return null; - } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < key.length(); i++) { - char c = key.charAt(i); - switch (c) { - case ':': - case '*': - sb.append("\\").append(c); - break; - default: - sb.append(c); - } - } - return sb.toString(); - } - - @Override - public T get(K key, String[] fields) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CommonParams.QT, "/get"); - params.set(CommonParams.FL, toDelimitedString(fields, ",")); - params.set("id", key.toString()); - try { - QueryResponse rsp = server.query(params); - Object o = rsp.getResponse().get("doc"); - if (o == null) { - return null; - } - return newInstance((SolrDocument) o, fields); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - public T newInstance(SolrDocument doc, String[] fields) throws IOException { - T persistent = newPersistent(); - if (fields == null) { - fields = fieldMap.keySet().toArray(new String[fieldMap.size()]); - } - String pk = mapping.getPrimaryKey(); - for (String f : fields) { - Field field = fieldMap.get(f); - Schema fieldSchema = field.schema(); - String sf = null; - if (pk.equals(f)) { - sf = f; - } else { - sf = mapping.getSolrField(f); - } - Object sv = doc.get(sf); - if (sv == null) { - continue; - } - - Object v = deserializeFieldValue(field, fieldSchema, sv, persistent); - persistent.put(field.pos(), v); - persistent.setDirty(field.pos()); - - } - persistent.clearDirty(); - return persistent; - } - - @SuppressWarnings("rawtypes") - private SpecificDatumReader getDatumReader(Schema fieldSchema) { - SpecificDatumReader<?> reader = readerMap.get(fieldSchema); - if (reader == null) { - reader = new SpecificDatumReader(fieldSchema);// ignore dirty bits - SpecificDatumReader localReader = null; - if ((localReader = readerMap.putIfAbsent(fieldSchema, reader)) != null) { - reader = localReader; - } - } - return reader; - } - - @SuppressWarnings("rawtypes") - private SpecificDatumWriter getDatumWriter(Schema fieldSchema) { - SpecificDatumWriter writer = writerMap.get(fieldSchema); - if (writer == null) { - writer = new SpecificDatumWriter(fieldSchema);// ignore dirty bits - writerMap.put(fieldSchema, writer); - } - - return writer; - } - - @SuppressWarnings("unchecked") - private Object deserializeFieldValue(Field field, Schema fieldSchema, - Object solrValue, T persistent) throws IOException { - Object fieldValue = null; - switch (fieldSchema.getType()) { - case MAP: - case ARRAY: - case RECORD: - @SuppressWarnings("rawtypes") - SpecificDatumReader reader = getDatumReader(fieldSchema); - fieldValue = IOUtils.deserialize((byte[]) solrValue, reader, - persistent.get(field.pos())); - break; - case ENUM: - fieldValue = AvroUtils.getEnumValue(fieldSchema, (String) solrValue); - break; - case FIXED: - throw new IOException("???"); - // break; - case BYTES: - fieldValue = ByteBuffer.wrap((byte[]) solrValue); - break; - case STRING: - fieldValue = new Utf8(solrValue.toString()); - break; - case UNION: - if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) { - // schema [type0, type1] - Type type0 = fieldSchema.getTypes().get(0).getType(); - Type type1 = fieldSchema.getTypes().get(1).getType(); - - // Check if types are different and there's a "null", like - // ["null","type"] or ["type","null"] - if (!type0.equals(type1)) { - if (type0.equals(Schema.Type.NULL)) - fieldSchema = fieldSchema.getTypes().get(1); - else - fieldSchema = fieldSchema.getTypes().get(0); - } else { - fieldSchema = fieldSchema.getTypes().get(0); - } - fieldValue = deserializeFieldValue(field, fieldSchema, solrValue, - persistent); - } else { - @SuppressWarnings("rawtypes") - SpecificDatumReader unionReader = getDatumReader(fieldSchema); - fieldValue = IOUtils.deserialize((byte[]) solrValue, unionReader, - persistent.get(field.pos())); - break; - } - break; - default: - fieldValue = solrValue; - } - return fieldValue; - } - - @Override - public void put(K key, T persistent) { - Schema schema = persistent.getSchema(); - if (!persistent.isDirty()) { - // nothing to do - return; - } - SolrInputDocument doc = new SolrInputDocument(); - // add primary key - doc.addField(mapping.getPrimaryKey(), key); - // populate the doc - List<Field> fields = schema.getFields(); - for (Field field : fields) { - String sf = mapping.getSolrField(field.name()); - // Solr will append values to fields in a SolrInputDocument, even the key - // mapping won't find the primary - if (sf == null) { - continue; - } - Schema fieldSchema = field.schema(); - Object v = persistent.get(field.pos()); - if (v == null) { - continue; - } - v = serializeFieldValue(fieldSchema, v); - doc.addField(sf, v); - - } - LOG.info("Putting DOCUMENT: " + doc); - batch.add(doc); - if (batch.size() >= batchSize) { - try { - add(batch, commitWithin); - batch.clear(); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - } - - @SuppressWarnings("unchecked") - private Object serializeFieldValue(Schema fieldSchema, Object fieldValue) { - switch (fieldSchema.getType()) { - case MAP: - case ARRAY: - case RECORD: - byte[] data = null; - try { - @SuppressWarnings("rawtypes") - SpecificDatumWriter writer = getDatumWriter(fieldSchema); - data = IOUtils.serialize(writer, fieldValue); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - fieldValue = data; - break; - case BYTES: - fieldValue = ((ByteBuffer) fieldValue).array(); - break; - case ENUM: - case STRING: - fieldValue = fieldValue.toString(); - break; - case UNION: - // If field's schema is null and one type, we do undertake serialization. - // All other types are serialized. - if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) { - int schemaPos = getUnionSchema(fieldValue, fieldSchema); - Schema unionSchema = fieldSchema.getTypes().get(schemaPos); - fieldValue = serializeFieldValue(unionSchema, fieldValue); - } else { - byte[] serilazeData = null; - try { - @SuppressWarnings("rawtypes") - SpecificDatumWriter writer = getDatumWriter(fieldSchema); - serilazeData = IOUtils.serialize(writer, fieldValue); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - fieldValue = serilazeData; - } - break; - default: - // LOG.error("Unknown field type: " + fieldSchema.getType()); - break; - } - return fieldValue; - } - - private boolean isNullable(Schema unionSchema) { - for (Schema innerSchema : unionSchema.getTypes()) { - if (innerSchema.getType().equals(Schema.Type.NULL)) { - return true; - } - } - return false; - } - - /** - * Given an object and the object schema this function obtains, from within - * the UNION schema, the position of the type used. If no data type can be - * inferred then we return a default value of position 0. - * - * @param pValue - * @param pUnionSchema - * @return the unionSchemaPosition. - */ - private int getUnionSchema(Object pValue, Schema pUnionSchema) { - int unionSchemaPos = 0; - for (Schema currentSchema : pUnionSchema.getTypes()) { - Type schemaType = currentSchema.getType(); - if (pValue instanceof Utf8 && schemaType.equals(Type.STRING)) - return unionSchemaPos; - else if (pValue instanceof ByteBuffer && schemaType.equals(Type.BYTES)) - return unionSchemaPos; - else if (pValue instanceof Integer && schemaType.equals(Type.INT)) - return unionSchemaPos; - else if (pValue instanceof Long && schemaType.equals(Type.LONG)) - return unionSchemaPos; - else if (pValue instanceof Double && schemaType.equals(Type.DOUBLE)) - return unionSchemaPos; - else if (pValue instanceof Float && schemaType.equals(Type.FLOAT)) - return unionSchemaPos; - else if (pValue instanceof Boolean && schemaType.equals(Type.BOOLEAN)) - return unionSchemaPos; - else if (pValue instanceof Map && schemaType.equals(Type.MAP)) - return unionSchemaPos; - else if (pValue instanceof List && schemaType.equals(Type.ARRAY)) - return unionSchemaPos; - else if (pValue instanceof Persistent && schemaType.equals(Type.RECORD)) - return unionSchemaPos; - unionSchemaPos++; - } - // if we weren't able to determine which data type it is, then we return the - // default - return DEFAULT_UNION_SCHEMA; - } - - @Override - public boolean delete(K key) { - String keyField = mapping.getPrimaryKey(); - try { - UpdateResponse rsp = server.deleteByQuery(keyField + ":" - + escapeQueryKey(key.toString())); - server.commit(); - LOG.info(rsp.toString()); - return true; - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - return false; - } - - @Override - public long deleteByQuery(Query<K, T> query) { - UpdateResponse rsp; - try { - /* - In this If block we check whether, user needs to delete full document or some fields in the document. We can't delete fields in a document by using solr deleteByQuery method. - therefore what we have done here is setting the particular fields values into null. - */ - if (query.getFields() != null && query.getFields().length < mapping.mapping.size() && !(Arrays.asList(query.getFields()).contains(mapping.getPrimaryKey()))) { - Result<K, T> result = query.execute(); - Map<String, String> partialUpdateNull = new HashMap<>(); - partialUpdateNull.put("set", null); - while (result.next()) { - SolrInputDocument inputDoc = new SolrInputDocument(); - inputDoc.setField(mapping.getPrimaryKey(), result.getKey()); - for (String field : query.getFields()) { - inputDoc.setField(field, partialUpdateNull); - } - batch.add(inputDoc); - } - if (commitWithin == 0) { - rsp = server.add(batch); - server.commit(false, true, true); - batch.clear(); - LOG.info(rsp.toString()); - } else { - rsp = server.add(batch, commitWithin); - batch.clear(); - LOG.info(rsp.toString()); - } - } else { - SolrQuery<K, T> solrQuery = (SolrQuery<K, T>) query; - String q = solrQuery.toSolrQuery(); - rsp = server.deleteByQuery(q); - server.commit(); - LOG.info(rsp.toString()); - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - return 0; - } - - @Override - public Result<K, T> execute(Query<K, T> query) { - try { - return new SolrResult<>(this, query, server, resultsSize); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - @Override - public Query<K, T> newQuery() { - return new SolrQuery<>(this); - } - - @Override - public List<PartitionQuery<K, T>> getPartitions(Query<K, T> query) - throws IOException { - // TODO: implement this using Hadoop DB support - - ArrayList<PartitionQuery<K, T>> partitions = new ArrayList<>(); - PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query); - pqi.setConf(getConf()); - partitions.add(pqi); - - return partitions; - } - - @Override - public void flush() { - try { - if (batch.size() > 0) { - add(batch, commitWithin); - batch.clear(); - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - } - - @Override - public void close() { - flush(); - } - - private void add(ArrayList<SolrInputDocument> batch, int commitWithin) - throws SolrServerException, IOException { - if (commitWithin == 0) { - server.add(batch); - server.commit(false, true, true); - } else { - server.add(batch, commitWithin); - } - } -} http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/main/java/org/apache/gora/solr/store/package-info.java ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/main/java/org/apache/gora/solr/store/package-info.java b/gora-solr-5/src/main/java/org/apache/gora/solr/store/package-info.java deleted file mode 100644 index 7ae4275..0000000 --- a/gora-solr-5/src/main/java/org/apache/gora/solr/store/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * This package contains all the Solr store related classes. - */ -package org.apache.gora.solr.store; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/gora-solr-mapping.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/gora-solr-mapping.xml b/gora-solr-5/src/test/conf/gora-solr-mapping.xml deleted file mode 100644 index 78ca39b..0000000 --- a/gora-solr-5/src/test/conf/gora-solr-mapping.xml +++ /dev/null @@ -1,40 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<gora-otd> - <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee"> - <primarykey column="ssn"/> - <field name="name" column="name"/> - <field name="dateOfBirth" column="dateOfBirth"/> - <field name="salary" column="salary"/> - <field name="boss" column="boss"/> - <field name="webpage" column="webpage"/> - </class> - - <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage"> - <primarykey column="url"/> - <field name="content" column="content"/> - <field name="parsedContent" column="parsedContent"/> - <field name="outlinks" column="outlinks"/> - <field name="headers" column="headers"/> - <field name="metadata" column="metadata"/> - <field name="byteData" column="byteData"/> - <field name="stringData" column="stringData"/> - </class> -</gora-otd> - http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/gora.properties ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/gora.properties b/gora-solr-5/src/test/conf/gora.properties deleted file mode 100644 index 27403e2..0000000 --- a/gora-solr-5/src/test/conf/gora.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -gora.solrstore.solr.url=http://localhost:9876/solr -gora.datastore.solr.commit_within=0 -gora.solrstore.solr.solrjserver=http -gora.solrstore.solr.solrjserver.user_auth=false -#gora.solrstore.solr.solrjserver.username=solr-user -#gora.solrstore.solr.solrjserver.password=solr-password http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/log4j.properties b/gora-solr-5/src/test/conf/log4j.properties deleted file mode 100644 index 3094e6b..0000000 --- a/gora-solr-5/src/test/conf/log4j.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set root logger level to error -log4j.rootLogger=DEBUG, Console - -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.hadoop=WARN - -###### Console appender definition ####### - -# All outputs currently set to be a ConsoleAppender. -log4j.appender.Console=org.apache.log4j.ConsoleAppender -log4j.appender.Console.layout=org.apache.log4j.PatternLayout - -log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %x %-5p [%c{3}] [%t] %m%n - -###### File appender definition ####### -log4j.appender.File=org.apache.log4j.DailyRollingFileAppender -log4j.appender.File.Append=true -log4j.appender.File.DatePattern='.'yyyy-MM-dd -log4j.appender.File.layout=org.apache.log4j.PatternLayout -log4j.appender.File.layout.ConversionPattern=%d{ABSOLUTE} %-5p [%c] %m%n http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/elevate.xml ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/elevate.xml b/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/elevate.xml deleted file mode 100644 index ed6a886..0000000 --- a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/elevate.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- If this file is found in the config directory, it will only be - loaded once at startup. If it is found in Solr's data - directory, it will be re-loaded every commit. - - See http://wiki.apache.org/solr/QueryElevationComponent for more info - ---> -<elevate> - <!-- Query elevation examples - <query text="foo bar"> - <doc id="1" /> - <doc id="2" /> - <doc id="3" /> - </query> - -for use with techproducts example - - <query text="ipod"> - <doc id="MA147LL/A" /> put the actual ipod at the top - <doc id="IW-02" exclude="true" /> exclude this cable - </query> ---> - -</elevate> http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/lang/stopwords_en.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/lang/stopwords_en.txt b/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/lang/stopwords_en.txt deleted file mode 100644 index 224230c..0000000 --- a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/lang/stopwords_en.txt +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# a couple of test stopwords to test that the words are really being -# configured from this file: -stopworda -stopwordb - -# Standard english stop words taken from Lucene's StopAnalyzer -a -an -and -are -as -at -be -but -by -for -if -in -into -is -it -no -not -of -on -or -such -that -the -their -then -there -these -they -this -to -was -will -with http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/managed-schema ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/managed-schema b/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/managed-schema deleted file mode 100644 index 83c296f..0000000 --- a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/managed-schema +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<!-- Solr managed schema - automatically generated - DO NOT EDIT --> -<schema name="test example Employee" version="1.5"> - <uniqueKey>ssn</uniqueKey> - <fieldType name="binary" class="solr.BinaryField"/> - <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/> - <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/> - <fieldType name="string" class="solr.StrField" sortMissingLast="true"/> - <field name="_version_" type="long" indexed="true" stored="true"/> - <field name="boss" type="binary" stored="true"/> - <field name="dateOfBirth" type="long" stored="true"/> - <field name="name" type="string" indexed="true" stored="true"/> - <field name="salary" type="int" stored="true"/> - <field name="ssn" type="string" multiValued="false" indexed="true" required="true" stored="true"/> - <field name="webpage" type="binary" stored="true"/> -</schema> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/3b2f282d/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/protwords.txt ---------------------------------------------------------------------- diff --git a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/protwords.txt b/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/protwords.txt deleted file mode 100644 index 5a32e50..0000000 --- a/gora-solr-5/src/test/conf/solr-managed-schema/Employee/conf/protwords.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky -
