Author: kwright
Date: Wed Jan 23 17:32:45 2013
New Revision: 1437592
URL: http://svn.apache.org/viewvc?rev=1437592&view=rev
Log:
Solr tester, except for history check for failures
Added:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
(with props)
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
(with props)
Modified:
manifoldcf/trunk/build.xml
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
Modified: manifoldcf/trunk/build.xml
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1437592&r1=1437591&r2=1437592&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Wed Jan 23 17:32:45 2013
@@ -2255,6 +2255,10 @@
<ant dir="tests/jdbc" target="run-UI-derby"/>
</target>
+ <target name="run-solr-tests-derby"
depends="build-tests-framework,build-tests-filesystem-connector,build-tests-solr-connector,calculate-solr-tests-condition"
if="solr-tests.include">
+ <ant dir="tests/solr" target="run-derby"/>
+ </target>
+
<target name="run-solr-UI-tests-derby"
depends="build-tests-framework,build-tests-solr-connector,build-tests-filesystem-connector,calculate-solr-tests-condition"
if="solr-tests.include">
<ant dir="tests/solr" target="run-UI-derby"/>
</target>
Added:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java?rev=1437592&view=auto
==============================================================================
---
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
(added)
+++
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
Wed Jan 23 17:32:45 2013
@@ -0,0 +1,90 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.solr_tests;
+
+import java.io.*;
+import java.util.*;
+
+/** Helper class for local file manipulation */
+public class FileHelper
+{
+
+ protected static void createDirectory(File f)
+ throws Exception
+ {
+ if (f.mkdirs() == false)
+ throw new Exception("Failed to create directory "+f.toString());
+ }
+
+ protected static void removeDirectory(File f)
+ throws Exception
+ {
+ File[] files = f.listFiles();
+ if (files != null)
+ {
+ int i = 0;
+ while (i < files.length)
+ {
+ File subfile = files[i++];
+ if (subfile.isDirectory())
+ removeDirectory(subfile);
+ else
+ subfile.delete();
+ }
+ }
+ f.delete();
+ }
+
+ protected static void createFile(File f, String contents)
+ throws Exception
+ {
+ OutputStream os = new FileOutputStream(f);
+ try
+ {
+ Writer w = new OutputStreamWriter(os,"utf-8");
+ try
+ {
+ w.write(contents);
+ }
+ finally
+ {
+ w.flush();
+ }
+ }
+ finally
+ {
+ os.close();
+ }
+ }
+
+ protected static void removeFile(File f)
+ throws Exception
+ {
+ if (f.delete() == false)
+ throw new Exception("Failed to delete file "+f.toString());
+ }
+
+ protected static void changeFile(File f, String newContents)
+ throws Exception
+ {
+ removeFile(f);
+ createFile(f,newContents);
+ }
+
+}
Propchange:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/FileHelper.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java?rev=1437592&r1=1437591&r2=1437592&view=diff
==============================================================================
---
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
(original)
+++
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/MockSolrService.java
Wed Jan 23 17:32:45 2013
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.manifoldcf.rss_tests;
+package org.apache.manifoldcf.solr_tests;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.server.Server;
Added:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
URL:
http://svn.apache.org/viewvc/manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java?rev=1437592&view=auto
==============================================================================
---
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
(added)
+++
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
Wed Jan 23 17:32:45 2013
@@ -0,0 +1,176 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.solr_tests;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import org.apache.manifoldcf.crawler.interfaces.*;
+import org.apache.manifoldcf.crawler.system.ManifoldCF;
+
+import java.io.*;
+import java.util.*;
+
+import org.apache.manifoldcf.agents.output.solr.SolrConfig;
+
+/** This is a 100 document crawl */
+public class SolrTester
+{
+ protected org.apache.manifoldcf.crawler.tests.ManifoldCFInstance instance;
+
+ public SolrTester(org.apache.manifoldcf.crawler.tests.ManifoldCFInstance
instance)
+ {
+ this.instance = instance;
+ }
+
+ public void setupTestArea()
+ throws Exception
+ {
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ FileHelper.createDirectory(f);
+ // Create the test data files.
+ String baseFileName = "testdata/";
+ int i0 = 0;
+ while (i0 < 10)
+ {
+ String fileName0 = baseFileName + "/dir-" + i0;
+ FileHelper.createDirectory(new File(fileName0));
+ int i1 = 0;
+ while (i1 < 10)
+ {
+ String fileName1 = fileName0 + "/dir-" + i1;
+ FileHelper.createDirectory(new File(fileName1));
+ int i2 = 0;
+ while (i2 < 10)
+ {
+ String fileName2 = fileName1 + "/file-"+i2;
+ FileHelper.createFile(new File(fileName2),"Test file
"+i0+":"+i1+":"+i2);
+ i2++;
+ }
+ i1++;
+ }
+ i0++;
+ }
+ System.err.println("Done generating files");
+ }
+
+ public void teardownTestArea()
+ throws Exception
+ {
+ System.err.println("Removing generated files");
+ File f = new File("testdata");
+ FileHelper.removeDirectory(f);
+ }
+
+ public void executeTest()
+ throws Exception
+ {
+ // Hey, we were able to install the file system connector etc.
+ // Now, create a local test job and run it.
+ IThreadContext tc = ThreadContextFactory.make();
+
+ // Create a basic file system connection, and save it.
+ IRepositoryConnectionManager mgr =
RepositoryConnectionManagerFactory.make(tc);
+ IRepositoryConnection conn = mgr.create();
+ conn.setName("File Connection");
+ conn.setDescription("File Connection");
+
conn.setClassName("org.apache.manifoldcf.crawler.connectors.filesystem.FileConnector");
+ conn.setMaxConnections(100);
+ // Now, save
+ mgr.save(conn);
+
+ // Create a basic null output connection, and save it.
+ IOutputConnectionManager outputMgr =
OutputConnectionManagerFactory.make(tc);
+ IOutputConnection outputConn = outputMgr.create();
+ outputConn.setName("Solr Connection");
+ outputConn.setDescription("Solr Connection");
+
outputConn.setClassName("org.apache.manifoldcf.agents.output.solr.SolrConnector");
+ outputConn.setMaxConnections(10);
+ // Set the connection parameters
+ ConfigParams configParams = outputConn.getConfigParams();
+
configParams.setParameter(SolrConfig.PARAM_PROTOCOL,SolrConfig.PROTOCOL_TYPE_HTTP);
+ configParams.setParameter(SolrConfig.PARAM_SERVER,"localhost");
+ configParams.setParameter(SolrConfig.PARAM_PORT,"8188");
+ configParams.setParameter(SolrConfig.PARAM_WEBAPPNAME,"solr");
+ configParams.setParameter(SolrConfig.PARAM_UPDATEPATH,"/update/extract");
+ configParams.setParameter(SolrConfig.PARAM_REMOVEPATH,"/update");
+ configParams.setParameter(SolrConfig.PARAM_STATUSPATH,"/admin/ping");
+ configParams.setParameter(SolrConfig.PARAM_IDFIELD,"id");
+ // Now, save
+ outputMgr.save(outputConn);
+
+ // Create a job.
+ IJobManager jobManager = JobManagerFactory.make(tc);
+ IJobDescription job = jobManager.createJob();
+ job.setDescription("Test Job");
+ job.setConnectionName("File Connection");
+ job.setOutputConnectionName("Solr Connection");
+ job.setType(job.TYPE_SPECIFIED);
+ job.setStartMethod(job.START_DISABLE);
+ job.setHopcountMode(job.HOPCOUNT_NEVERDELETE);
+
+ // Now, set up the document specification.
+ DocumentSpecification ds = job.getSpecification();
+ // Crawl everything underneath the 'testdata' area
+ File testDataFile = new File("testdata").getCanonicalFile();
+ if (!testDataFile.exists())
+ throw new ManifoldCFException("Test data area not found! Looking in
"+testDataFile.toString());
+ if (!testDataFile.isDirectory())
+ throw new ManifoldCFException("Test data area not a directory! Looking
in "+testDataFile.toString());
+ SpecificationNode sn = new SpecificationNode("startpoint");
+ sn.setAttribute("path",testDataFile.toString());
+ SpecificationNode n = new SpecificationNode("include");
+ n.setAttribute("type","file");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ n = new SpecificationNode("include");
+ n.setAttribute("type","directory");
+ n.setAttribute("match","*");
+ sn.addChild(sn.getChildCount(),n);
+ ds.addChild(ds.getChildCount(),sn);
+
+ // Set up the output specification.
+ OutputSpecification os = job.getOutputSpecification();
+ // Solr output specification is not needed
+
+ // Save the job.
+ jobManager.save(job);
+
+ // Now, start the job, and wait until it completes.
+ long startTime = System.currentTimeMillis();
+ jobManager.manualStart(job.getID());
+ instance.waitJobInactiveNative(jobManager,job.getID(),18000L);
+ System.err.println("Crawl required "+new
Long(System.currentTimeMillis()-startTime).toString()+" milliseconds");
+
+ // Look in the connection history for anything other than an OK
+ // MHL
+
+ // Check to be sure we actually processed the right number of documents.
+ JobStatus status = jobManager.getStatus(job.getID());
+ if (status.getDocumentsProcessed() != 111)
+ throw new ManifoldCFException("Wrong number of documents processed -
expected 111, saw "+new Long(status.getDocumentsProcessed()).toString());
+
+ // Now, delete the job.
+ jobManager.deleteJob(job.getID());
+ instance.waitJobDeletedNative(jobManager,job.getID(),18000L);
+
+ // Cleanup is automatic by the base class, so we can feel free to leave
jobs and connections lying around.
+ }
+
+}
Propchange:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/trunk/tests/solr/src/test/java/org/apache/manifoldcf/solr_tests/SolrTester.java
------------------------------------------------------------------------------
svn:keywords = Id
