Author: ivol37 at gmail.com
Date: Wed Jan 26 17:02:20 2011
New Revision: 712
Log:
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrApi.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApiImpl.java
- copied, changed from r711,
/sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
Removed:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
Added:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrApi.java
==============================================================================
--- (empty file)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrApi.java
Wed Jan 26 17:02:20 2011
@@ -0,0 +1,38 @@
+/*
+ Copyright (C) 2010 Amdatu.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.amdatu.searchandindex.solr;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+import javax.xml.stream.XMLStreamException;
+
+import org.amdatu.core.tenant.Tenant;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+
+public interface SolrApi {
+ void updateIndex(Tenant tenant, String index, List<SolrInputDocument>
solrDocuments) throws XMLStreamException, SolrServerException, IOException;
+
+ QueryResponse queryIndex(Tenant tenant, String index, String query) throws
SolrServerException;
+
+ Collection<String> getIndices(Tenant tenant) throws SolrServerException;
+
+ String getSchema(Tenant tenant, String index) throws IOException;
+}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrRestService.java
Wed Jan 26 17:02:20 2011
@@ -1,3 +1,19 @@
+/*
+ Copyright (C) 2010 Amdatu.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
package org.amdatu.searchandindex.solr;
public interface SolrRestService {
Copied:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApiImpl.java
(from r711,
/sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java)
==============================================================================
---
/sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApi.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrApiImpl.java
Wed Jan 26 17:02:20 2011
@@ -21,15 +21,20 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
+import java.util.Collection;
+import java.util.List;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLStreamException;
import org.amdatu.core.config.templates.ConfigTemplateManager;
import org.amdatu.core.tenant.Tenant;
+import org.amdatu.searchandindex.solr.SolrApi;
+import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
@@ -44,10 +49,11 @@
import org.osgi.service.log.LogService;
import org.xml.sax.SAXException;
-public class SolrApi {
+ at SuppressWarnings("restriction")
+public class SolrApiImpl implements SolrApi {
// Statics
- private static final String DEFAULT_SCHEMA = "nutch_schema.xml";
- private static final String DEFAULT_SOLRCONFIG = "nutch_solrconfig.xml";
+ private static final String DEFAULT_SCHEMA = "conf/default_schema.xml";
+ private static final String DEFAULT_SOLRCONFIG =
"conf/default_solrconfig.xml";
// Service dependencies injected by the dependency manager
private volatile LogService m_logService;
@@ -58,7 +64,7 @@
private File m_workDir;
private CoreContainer m_coreContainer;
- public SolrApi(File workDir, CoreContainer coreContainer) {
+ public SolrApiImpl(File workDir, CoreContainer coreContainer) {
m_workDir = workDir;
m_coreContainer = coreContainer;
}
@@ -83,16 +89,47 @@
catch (SAXException e) {
m_logService.log(LogService.LOG_ERROR, "Could not create Solr
core configuration for '" + coreName + "'", e);
}
- } else {
- m_logService.log(LogService.LOG_INFO, "Solr index '" + index + "'
for tenant '" + tenant.getId() + "' not created since it already exists");
}
}
public void updateIndex(Tenant tenant, String index, InputStream is)
throws XMLStreamException, SolrServerException, IOException {
String coreName = tenant.getId() + "_" + index;
+ m_logService.log(LogService.LOG_DEBUG, "Starting update for index '" +
coreName + "'");
SolrServer server = new EmbeddedSolrServer(m_coreContainer, coreName);
- SolrInputDocument solrDocument = SolrUtil.fromXMLStream(is);
- server.add(solrDocument);
+ List<SolrInputDocument> solrDocuments = SolrUtil.fromXMLStream(is);
+ for (SolrInputDocument solrDocument : solrDocuments) {
+ server.add(solrDocument);
+ }
+ server.commit();
+ m_logService.log(LogService.LOG_DEBUG, "Index '" + coreName + "'
updated");
+ }
+
+ public void updateIndex(Tenant tenant, String index,
List<SolrInputDocument> solrDocuments) throws XMLStreamException,
SolrServerException, IOException {
+ String coreName = tenant.getId() + "_" + index;
+ SolrServer server = new EmbeddedSolrServer(m_coreContainer, coreName);
+ for (SolrInputDocument solrDocument : solrDocuments) {
+ server.add(solrDocument);
+ }
+ server.commit();
+ m_logService.log(LogService.LOG_DEBUG, "Index '" + coreName + "'
updated");
+ }
+
+ public Collection<String> getIndices(Tenant tenant) throws
SolrServerException {
+ return m_coreContainer.getCoreNames();
+ }
+
+ public QueryResponse queryIndex(Tenant tenant, String index, String query)
throws SolrServerException {
+ String coreName = tenant.getId() + "_" + index;
+ SolrServer server = new EmbeddedSolrServer(m_coreContainer, coreName);
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setQuery(query);
+ return server.query(solrQuery);
+ }
+
+ public String getSchema(Tenant tenant, String index) throws IOException {
+ File schema = new File(m_workDir, tenant.getId() + "/" + index +
"/conf/schema.xml");
+ String contents = SolrUtil.loadFromFile(schema);
+ return contents;
}
private void createCore(String coreName, String coreDirName) throws
IOException, ParserConfigurationException, SAXException {
@@ -110,7 +147,7 @@
coreConfDir.delete();
}
coreConfDir.mkdirs();
- File coresConfigFile = new File(m_workDir,
SolrDaemonServiceImpl.SOLR_CONFIG_XML);
+ File coresConfigFile = new File(m_workDir,
SolrDaemonServiceImpl.SOLR);
Bundle bundle = m_bundleContext.getBundle();
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDaemonServiceImpl.java
Wed Jan 26 17:02:20 2011
@@ -23,6 +23,7 @@
import org.amdatu.core.config.templates.ConfigTemplateManager;
import org.amdatu.core.tenant.TenantManagementService;
+import org.amdatu.searchandindex.solr.SolrApi;
import org.amdatu.searchandindex.solr.SolrRestService;
import org.amdatu.searchandindex.solr.SolrService;
import org.amdatu.web.httpcontext.HttpContextServiceFactory;
@@ -44,7 +45,7 @@
// Statics
private static final String DEFAULT_SOLAR_CONFIG =
"default_solrconfig.xml";
private static final String DEFAULT_SCHEMA = "default_schema.xml";
- private static final String SOLR = "solr.xml";
+ static final String SOLR = "solr.xml";
private static final String CONFIG_DIR = "conf";
// Services injected by the Felix dependency manager
@@ -69,7 +70,7 @@
m_workDir.mkdirs();
try {
// Write the solr.xml file to the solr root directory
- copyConfig("conf/" + SOLR, solrFile);
+ copyConfig(SOLR, solrFile);
// Solr uses this system property to find its storage
location.
System.setProperty("solr.solr.home",
solrFile.getParentFile().getAbsolutePath());
@@ -79,11 +80,14 @@
mainConfigDir.mkdirs();
// Write default schema and solr config to /conf
- copyConfig("conf/" + DEFAULT_SOLAR_CONFIG, new
File(mainConfigDir, "solrconfig.xml"));
- copyConfig("conf/" + DEFAULT_SCHEMA, new
File(mainConfigDir, "schema.xml"));
+ copyConfig(DEFAULT_SOLAR_CONFIG, new File(mainConfigDir,
"solrconfig.xml"));
+ copyConfig(DEFAULT_SCHEMA, new File(mainConfigDir,
"schema.xml"));
} catch (IOException e) {
m_logService.log(LogService.LOG_ERROR, "Could not replace
configuration entries in storage-conf.xml", e);
}
+ } else {
+ // Solr uses this system property to find its storage location.
+ System.setProperty("solr.solr.home",
solrFile.getParentFile().getAbsolutePath());
}
} catch (Exception e) {
m_logService.log(LogService.LOG_ERROR, "Could not create context
component", e);
@@ -96,7 +100,7 @@
startSolrDaemon();
// Register the Solr API service
- SolrApi solrApi = registerSolrApi();
+ SolrApiImpl solrApi = registerSolrApi();
// Register the Solr dispatch filter
registerRestAPI(solrApi);
@@ -157,17 +161,18 @@
}
// Register the Solr API
- private SolrApi registerSolrApi() {
- SolrApi solrApiService = new SolrApi(m_workDir, m_coreContainer);
+ private SolrApiImpl registerSolrApi() {
+ SolrApiImpl solrApiService = new SolrApiImpl(m_workDir,
m_coreContainer);
m_dependencyManager.add(m_dependencyManager.createComponent()
.setImplementation(solrApiService)
+ .setInterface(SolrApi.class.getName(), null)
.add(createServiceDependency(ConfigTemplateManager.class))
.add(createServiceDependency(LogService.class)));
return solrApiService;
}
// Register the REST service
- private void registerRestAPI(SolrApi solrApi) {
+ private void registerRestAPI(SolrApiImpl solrApi) {
SolrRestServiceImpl restService = new SolrRestServiceImpl(m_workDir,
solrApi);
m_dependencyManager.add(m_dependencyManager.createComponent()
.setImplementation(restService)
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrRestServiceImpl.java
Wed Jan 26 17:02:20 2011
@@ -16,15 +16,10 @@
*/
package org.amdatu.searchandindex.solr.impl;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.StringWriter;
-import java.io.Writer;
import java.net.URL;
+import java.util.Collection;
import java.util.Dictionary;
import java.util.Hashtable;
@@ -35,10 +30,10 @@
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.xml.stream.XMLStreamException;
import org.amdatu.core.tenant.Tenant;
import org.amdatu.core.tenant.TenantException;
@@ -49,7 +44,8 @@
import org.amdatu.web.httpcontext.ResourceProvider;
import org.apache.felix.dm.Component;
import org.apache.felix.dm.DependencyManager;
-import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
import org.osgi.framework.BundleContext;
import org.osgi.framework.Constants;
import org.osgi.service.log.LogService;
@@ -68,12 +64,12 @@
private volatile BundleContext m_bundleContext;
private volatile DependencyManager m_dependencyManager;
private volatile TenantManagementService m_tenantService;
- private volatile SolrApi m_solrApi;
+ private volatile SolrApiImpl m_solrApi;
// Private members
private File m_workDir;
- public SolrRestServiceImpl(File workDir, SolrApi solrApi) {
+ public SolrRestServiceImpl(File workDir, SolrApiImpl solrApi) {
m_workDir = workDir;
m_solrApi = solrApi;
}
@@ -115,73 +111,107 @@
/**
* This method can be used to check the availability of the Index Service.
- * GET /rest/index
+ * GET /rest/index/status
* @return The text "Index service online"
*/
@GET
+ @Path("{status}")
@Produces( { MediaType.TEXT_PLAIN })
public String status() {
return "Index service online";
}
+ /**
+ * This method can be used to add a new index or update it.
+ * PUT /rest/index/[indexname]
+ * @return The text "Index service online"
+ */
@PUT
@Path("{index}")
@Produces({MediaType.APPLICATION_XML})
public Response createIndex(@PathParam("index") final String index,
@Context final HttpServletRequest request, InputStream is) {
- // Resolve tenant from hostname
- String serverName = request.getServerName();
try {
- for (Tenant tenant : m_tenantService.getTenants()) {
- if (serverName.equals(tenant.getProperties().get("hostname")))
{
- // This is the tenant for which the index should be created
- m_solrApi.createIndex(tenant, index);
- if (is.available() > 0) {
- // When more then 0 bytes are available, a document is
send along with the PUT request
- // that should be added to this index
- m_solrApi.updateIndex(tenant, index, is);
- }
- }
+ // Resolve tenant from hostname
+ Tenant tenant = resolveTenant(request);
+ // This is the tenant for which the index should be created
+ m_solrApi.createIndex(tenant, index);
+ if (is.available() > 0) {
+ // When more then 0 bytes are available, a document is send
along with the PUT request
+ // that should be added to this index
+ m_solrApi.updateIndex(tenant, index, is);
}
- }
- catch (TenantException e) {
- // TODO Auto-generated catch block
+ } catch (Exception e) {
e.printStackTrace();
}
- catch (IOException e) {
- // TODO Auto-generated catch block
+ return Response.status(Response.Status.OK).build();
+ }
+
+ /**
+ * Perform Solr query on the specified index.
+ * GET /rest/index/[indexname]/?q=[Solr query]
+ */
+ @GET
+ @Path("{index}")
+ @Produces( { MediaType.TEXT_PLAIN })
+ public String queryIndex(@PathParam("index") final String index,
@QueryParam("q") final String q,
+ @Context final HttpServletRequest request) {
+ try {
+ Tenant tenant = resolveTenant(request);
+ QueryResponse response = m_solrApi.queryIndex(tenant, index, q);
+ SolrDocumentList solrDocuments = response.getResults();
+ return "<found>"+solrDocuments.getNumFound()+"</found>";
+ } catch (Exception e) {
e.printStackTrace();
}
- catch (XMLStreamException e) {
- // TODO Auto-generated catch block
+ return "NOPE";
+
+ }
+
+ /**
+ * Returns a list of all available incdices for this tenant.
+ * GET /rest/index
+ */
+ @GET
+ @Produces( { MediaType.TEXT_PLAIN })
+ public String getIndices(@PathParam("index") final String index, @Context
final HttpServletRequest request) {
+ try {
+ Tenant tenant = resolveTenant(request);
+ Collection<String> indices = m_solrApi.getIndices(tenant);
+ return "<found>" + indices + "</found>";
+ } catch (Exception e) {
e.printStackTrace();
}
- catch (SolrServerException e) {
- // TODO Auto-generated catch block
+ return "NOPE";
+ }
+
+ /**
+ * Returns a list of all available incdices for this tenant.
+ * GET /rest/index/[indexname]/schema
+ */
+ @GET
+ @Path("{index}/schema")
+ @Produces( { MediaType.TEXT_PLAIN })
+ public String getSchema(@PathParam("index") final String index, @Context
final HttpServletRequest request) {
+ try {
+ Tenant tenant = resolveTenant(request);
+ String schema = m_solrApi.getSchema(tenant, index);
+ return schema;
+ } catch (Exception e) {
e.printStackTrace();
}
- return Response.status(Response.Status.OK).build();
+ return "NOPE";
}
-
- public String convertStreamToString(InputStream is)
- throws IOException {
- if (is != null) {
- Writer writer = new StringWriter();
- char[] buffer = new char[1024];
- try {
- Reader reader = new BufferedReader(
- new InputStreamReader(is, "UTF-8"));
- int n;
- while ((n = reader.read(buffer)) != -1) {
- writer.write(buffer, 0, n);
- }
- } finally {
- is.close();
+ private Tenant resolveTenant(HttpServletRequest request) throws
TenantException {
+ // Resolve tenant from hostname
+ String serverName = request.getServerName();
+ for (Tenant tenant : m_tenantService.getTenants()) {
+ if (serverName.equals(tenant.getProperties().get("hostname"))) {
+ // This is the tenant for which the index should be created
+ return tenant;
}
- return writer.toString();
- } else {
- return "";
}
- }
+ return null;
+ }
}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrTest.java
Wed Jan 26 17:02:20 2011
@@ -2,10 +2,15 @@
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.stream.XMLStreamException;
import org.amdatu.core.tenant.TenantEntity;
import org.amdatu.core.tenant.TenantException;
import org.amdatu.core.tenant.TenantManagementService;
+import org.amdatu.searchandindex.solr.SolrApi;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
@@ -16,6 +21,7 @@
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrInputDocument;
/**
@@ -31,7 +37,10 @@
*/
public class SolrTest {
private volatile TenantManagementService m_tenantService;
+ private volatile SolrApi m_solrApi;
+
private static int ID = 0;
+ private static int SIZE = 1;
public void start() {
try {
@@ -62,6 +71,7 @@
}
class TestThread extends Thread {
+
public void run() {
try {
String baseUrl1 = "http://localhost:2237/rest/index";;
@@ -77,17 +87,31 @@
invokePUT(baseUrl1 + "/course");
invokePUT(baseUrl2 + "/course");
- // Index a document
- SolrInputDocument document = new SolrInputDocument();
- document.addField( "id", "id1", 1.0f );
- document.addField( "name", "doc1", 1.0f );
- document.addField( "price", 10 );
+ // Now do the same but without REST
+ TenantEntity tenant1 =
m_tenantService.getTenantById("school_1");
+ TenantEntity tenant2 =
m_tenantService.getTenantById("school_1");
+ List<SolrInputDocument> docs1 = new
ArrayList<SolrInputDocument>();
+ List<SolrInputDocument> docs2 = new
ArrayList<SolrInputDocument>();
+ for (int i=0; i<SIZE; i++) {
+ docs1.add(generateSolrDocument());
+ docs2.add(generateSolrDocument());
+ }
+ long before = System.currentTimeMillis();
+ m_solrApi.updateIndex(tenant1, "course", docs1);
+ m_solrApi.updateIndex(tenant2, "course", docs2);
+ long diff = System.currentTimeMillis() - before;
+ System.out.println("Added " + 2*SIZE + " documents to Solr
index using SPI in " + diff + " ms");
// Add 10.000 Solr documents to the indices
- for (int i=0; i<10000; i++) {
- streamPUT(baseUrl1 + "/course",
SolrUtil.toXMLStream(generateSolrDocument()));
- streamPUT(baseUrl2 + "/course",
SolrUtil.toXMLStream(generateSolrDocument()));
- }
+ before = System.currentTimeMillis();
+ InputStream xml1 = SolrUtil.toXMLStream(docs1);
+ InputStream xml2 = SolrUtil.toXMLStream(docs2);
+ long beforerest = System.currentTimeMillis();
+ streamPUT(baseUrl1 + "/course", xml1);
+ streamPUT(baseUrl2 + "/course", xml2);
+ diff = System.currentTimeMillis() - before;
+ long diff2 = System.currentTimeMillis() - beforerest;
+ System.out.println("Added " + 2*SIZE + " documents to Solr
index using REST in " + diff + " ms / " + diff2 + " ms");
}
catch (InterruptedException e) {
// TODO Auto-generated catch block
@@ -97,14 +121,26 @@
// TODO Auto-generated catch block
e.printStackTrace();
}
+ catch (TenantException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (XMLStreamException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ catch (SolrServerException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
}
}
private SolrInputDocument generateSolrDocument() {
SolrInputDocument document = new SolrInputDocument();
- document.addField( "id", ID++, 0.5f);
- document.addField( "name", "Solr document " + ID, 1.0f );
- document.addField( "price", 100*Math.random(), 0.5f);
+ document.addField("id", ID++, 0.5f);
+ document.addField("name_s", "Solr document " + ID, 1.0f );
+ document.addField("price_d", 100*Math.random(), 0.5f);
return document;
}
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrUtil.java
Wed Jan 26 17:02:20 2011
@@ -1,8 +1,18 @@
package org.amdatu.searchandindex.solr.impl;
+import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
@@ -15,6 +25,16 @@
import org.apache.solr.common.util.StrUtils;
public class SolrUtil {
+ public static InputStream toXMLStream(List<SolrInputDocument> documents)
throws UnsupportedEncodingException {
+ StringBuffer xml = new StringBuffer("<docs>");
+ for (SolrInputDocument doc : documents) {
+ xml.append(ClientUtils.toXML(doc));
+ }
+ xml.append("</docs>");
+ InputStream is = new
ByteArrayInputStream(xml.toString().getBytes("UTF-8"));
+ return is;
+ }
+
public static InputStream toXMLStream(SolrInputDocument document) throws
UnsupportedEncodingException {
String xml = ClientUtils.toXML(document);
InputStream is = new ByteArrayInputStream(xml.getBytes("UTF-8"));
@@ -22,7 +42,7 @@
}
@SuppressWarnings("restriction")
- public static SolrInputDocument fromXMLStream(InputStream is) throws
UnsupportedEncodingException, XMLStreamException {
+ public static List<SolrInputDocument> fromXMLStream(InputStream is) throws
UnsupportedEncodingException, XMLStreamException {
// Save the current context classloader first
final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
try {
@@ -32,18 +52,18 @@
XMLInputFactory factory = XMLInputFactory.newInstance();
XMLStreamReader parser = factory.createXMLStreamReader(is);
- SolrInputDocument doc = null;
+ List<SolrInputDocument> documents = new
ArrayList<SolrInputDocument>();
while (true) {
int event = parser.next();
switch (event) {
case XMLStreamConstants.END_DOCUMENT:
parser.close();
- return doc;
+ return documents;
case XMLStreamConstants.START_ELEMENT:
String currTag = parser.getLocalName();
if ("doc".equals(currTag)) {
- doc = readDoc(parser);
+ documents.add(readDoc(parser));
}
break;
}
@@ -124,6 +144,32 @@
}
}
+ public static String loadFromFile(File file) throws IOException {
+ FileInputStream fis = new FileInputStream(file);
+ String contents = convertStreamToString(fis);
+ return contents;
+ }
+
+ public static String convertStreamToString(InputStream is) throws
IOException {
+ if (is != null) {
+ Writer writer = new StringWriter();
+ char[] buffer = new char[1024];
+ try {
+ Reader reader = new BufferedReader(
+ new InputStreamReader(is, "UTF-8"));
+ int n;
+ while ((n = reader.read(buffer)) != -1) {
+ writer.write(buffer, 0, n);
+ }
+ } finally {
+ is.close();
+ }
+ return writer.toString();
+ } else {
+ return "";
+ }
+ }
+
public static void executeInContextClassLoader(ContextExecutor executor) {
// Save the current context classloader first
final ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
Modified:
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
==============================================================================
---
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
(original)
+++
sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java
Wed Jan 26 17:02:20 2011
@@ -18,6 +18,7 @@
import org.amdatu.core.config.templates.ConfigTemplateManager;
import org.amdatu.core.tenant.TenantManagementService;
+import org.amdatu.searchandindex.solr.SolrApi;
import org.amdatu.searchandindex.solr.SolrRestService;
import org.amdatu.searchandindex.solr.SolrService;
import org.amdatu.searchandindex.solr.impl.SolrDaemonServiceImpl;
@@ -44,6 +45,7 @@
manager.add(createComponent()
.setImplementation(SolrTest.class)
.add(createServiceDependency().setService(SolrRestService.class).setRequired(true))
+
.add(createServiceDependency().setService(SolrApi.class).setRequired(true))
.add(createServiceDependency().setService(TenantManagementService.class).setRequired(true)));
// Create and register the Solr service.
