Author: rwesten
Date: Mon Apr 11 12:25:07 2011
New Revision: 1091050
URL: http://svn.apache.org/viewvc?rev=1091050&view=rev
Log:
STANBOL-141: The last missing things to support for Installing SolrIndexes
based on Archives.
Added support for Index Data References (Index Archives that just contain a
link to the actual data). The reference to the index data is resolved by the
DataFileProvider Service. The SolrYard now keeps track of "uninitialised
indexes" - this are indexes that where installed, but could not be initialised
because the required data where not available via the DataFileProvider service.
- Updated the SolrDirectoryManager interface to deal with "uninitialized"
indexes.
- Added a parameter to the SolrYardConfig that allows to specify if the Solr
Index is allowed to be initialised with the default schema (and without data).
- The DefaultSolrDirectoryManager now keeps a persistent list of
"uninitialised" indexes. This means that index data references (property files)
are stored within the data directory of the SolrYard bundle so that they can be
resent to the DataFileProvider service when the initialisation of the index is
requested the next time.
This functionality requires an OSGI environment. In case the SolrYard is
running outside an OSGI Environment such methods will throw an
IllegalStateException with an according message.
Note that directly including the index data in the solr archive is still
supported and still the preferred way to go for small or empty indexes. Empty
indexes can be used to use a special Solr Schema/Configuration for the
initialisation of an yard.
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallTask.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallerConstants.java
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/SolrIndexInstaller.java
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManager.java
Mon Apr 11 12:25:07 2011
@@ -19,6 +19,7 @@ package org.apache.stanbol.entityhub.yar
import java.io.File;
import java.io.IOException;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.solr.client.solrj.SolrServer;
@@ -80,7 +81,9 @@ public interface SolrDirectoryManager {
String DEFAULT_SOLR_DATA_DIR = "indexes";
/**
- * Checks if a solrIndex with the parsed name is managed or not.
+ * Checks if a solrIndex with the parsed name is managed or not. Note that
+ * an Index might be managed, but not yet be initialised. To check if an
+ * index is managed and can be used use {@link #isInitialisedIndex(String)
* @param solrIndexName the name of the Solr index to check
* @return <code>true</code> only if a Solr index with the parsed name is
* already present within the manages Solr directory.
@@ -90,12 +93,24 @@ public interface SolrDirectoryManager {
* @throws IllegalArgumentException In case <code>null</code> or an empty
* string is parsed as solrIndexName
*/
- boolean isManagedIndex(String solrIndexName) throws IllegalStateException;
+ boolean isManagedIndex(String solrIndexName) throws
IllegalStateException,IllegalArgumentException;
+ /**
+ * Checks if the managed index is also initialised and ready to be used.<p>
+ * Indexes are managed as soon as they are announced to the
+ * SolrDirectoryManager. However when using the
+ * {@link #createSolrDirectory(String, String, Properties)} it can not be
+ * assured that the archive with the actual data is already available.<p>
+ * @param indexName the name of the index
+ * @return
+ * @throws IllegalStateException
+ * @throws IllegalArgumentException
+ */
+ boolean isInitialisedIndex(String indexName) throws
IllegalStateException,IllegalArgumentException;
/**
* Getter for all the indexes currently available in the managed solr
directory.
* The key is the name of the index and the value is the File pointing to
the
- * directory.
+ * directory. For uninitialised indexes the value will be
<code>null</code>.
* @return map containing all the currently available indexes
* @throws IllegalStateException In case the managed Solr directory can not
* be obtained (usually indicates that this component is currently
@@ -108,19 +123,23 @@ public interface SolrDirectoryManager {
* ensure that returned directories are valid Solr indices (or Solr
Cores)<p>
* Directories returned by this method are typically used as second
parameter
* of {@link SolrServerProvider#getSolrServer(Type, String, String...)} to
- * create an {@link SolrServer} instance.
+ * create an {@link SolrServer} instance.<p>
+ * This method may trigger the initialisation of the SolrIndex if not
already
+ * done.<p>
+ * This method needs to wait until the initialisation of the index i
+ * completed (even in multi threaded environments) <p>
* @param solrPathOrUri the name of the requested solr index. If no index
* with that name does exist a new one will be initialised base on the
* default core configuration part of this bundle.
- * @param If <code>true</code> the Solr directory is created and
initialised
- * with the default configuration if it does not already exist.
+ * @param allowDefaultInit If <code>true</code> the Solr Index can be
initialised
+ * with the default configuration if not already present.
* @return the directory (instanceDir) of the index or <code>null</code> if
- * <code>false</code> was parsed as create and the index does not already
- * exist.
+ * <code>false</code> was parsed as allowDefaultInit and the data for the
index
+ * are not yet available.
* @throws IllegalArgumentException if the parsed solrIndexName is
* <code>null</code> or empty
*/
- File getSolrDirectory(final String solrIndexName,boolean create) throws
IllegalArgumentException;
+ File getSolrIndexDirectory(final String solrIndexName,boolean
allowDefaultInit) throws IllegalArgumentException;
/**
* Creates a new Solr Index based on the data in the provided {@link
ArchiveInputStream}
* @param solrIndexName the name of the index to create
@@ -137,13 +156,13 @@ public interface SolrDirectoryManager {
* {@link DataFileProvider} service
* @param solrIndexName The name of the solrIndex to create
* @param indexPath the name of the dataFile looked up via the {@link
DataFileProvider}
- * @param comments The comments describing the data on how to get the index
+ * @param propergies Additional properties describing the index
* @return the directory (instanceDir) of the index or null if the index
* data could not be found
* @throws IllegalArgumentException
* @throws IOException
*/
- File createSolrDirectory(final String solrIndexName, String
indexPath,Map<String,String> comments) throws IllegalArgumentException,
IOException;
+ File createSolrDirectory(final String solrIndexName, String
indexPath,Properties propergies) throws IllegalArgumentException, IOException;
/**
* Getter for the managed Solr Directory.
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/ConfigUtils.java
Mon Apr 11 12:25:07 2011
@@ -17,7 +17,9 @@
package org.apache.stanbol.entityhub.yard.solr.impl;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -40,9 +42,12 @@ import org.apache.commons.compress.compr
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import
org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider;
import
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants;
import org.osgi.framework.Bundle;
+import org.osgi.service.component.ComponentContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,8 +77,13 @@ public final class ConfigUtils {
cfm.put("zip", "zip");
cfm.put("jar", "zip");
cfm.put("ref", "properties"); //reference
+ cfm.put("properties", "properties"); //also accept properties as
references
SUPPORTED_SOLR_ARCHIVE_FORMAT = Collections.unmodifiableMap(cfm);
}
+ /**
+ * Use <indexName>.solrindex[.<archiveType>] as file name
+ */
+ public static final String SOLR_INDEX_ARCHIVE_EXTENSION = "solrindex";
public static ArchiveInputStream getArchiveInputStream(String
solrArchiveName,InputStream is) throws IOException {
String archiveFormat;
@@ -513,4 +523,103 @@ public final class ConfigUtils {
}
return file;
}
+
+ /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
+ * Methods for storing and loading configurations for uninitialised indexes
+ * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
+ */
+
+ /**
+ * The path to the directory used to store properties of uninitialised
+ * referenced sites.<p>
+ * Such sites are one that are created by using
+ * {@link #createSolrDirectory(String, String, Map)} but the
+ * {@link DataFileProvider} does not yet provide the necessary data to
+ * initialise the index.<p>
+ * This directory will store properties files with the indexName as name,
+ * properties as extension and the properties as value
+ */
+ private static final String UNINITIALISED_INDEX_DIRECTORY =
"config/uninitialised-index";
+ /**
+ * Saves the configuration of an uninitialised index
+ * @param context the context used to get the data storage
+ * @param indexName the name of the uninitialised index
+ * @param properties the properties of the uninitialised index
+ * @throws IOException on any error while saving the configuration
+ */
+ public static void saveUninitialisedIndexConfig(ComponentContext
context,String indexName,java.util.Properties properties) throws IOException {
+ File uninstalledConfigDir =
getUninitialisedSiteDirectory(context,true);
+ File config = new
File(uninstalledConfigDir,indexName+'.'+ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION+".ref");
+ FileOutputStream out = null;
+ if(properties == null){ //if no config is provided
+ properties = new java.util.Properties();//save an empty one
+ }
+ try {
+ out = new FileOutputStream(config);
+ properties.store(out,null);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ /**
+ * Returns the directory used to store the configurations of uninitialised
+ * Solr Indexes
+ * @param init if <code>true</code> the directory is created if needed
+ * @return the directory
+ */
+ public static File getUninitialisedSiteDirectory(ComponentContext
context,boolean init) {
+ if(context == null){
+ throw new IllegalStateException("This functionality only works
when running within an OSGI Environment");
+ }
+ File uninstalledConfigDir =
context.getBundleContext().getDataFile(UNINITIALISED_INDEX_DIRECTORY);
+ if(!uninstalledConfigDir.exists()){
+ if(init){
+ if(!uninstalledConfigDir.mkdirs()){
+ throw new IllegalStateException("Unable to create
Directory "+
+ UNINITIALISED_INDEX_DIRECTORY+"for storing information
of uninitialised Solr Indexes");
+ }
+ }
+ } else if(!uninstalledConfigDir.isDirectory()){
+ throw new IllegalStateException("The directory
"+UNINITIALISED_INDEX_DIRECTORY+
+ "for storing uninitialised Solr Indexes Information exists" +
+ "but is not a directory!");
+ } //else -> it exists and is a dir -> nothing todo
+ return uninstalledConfigDir;
+ }
+ /**
+ * Loads the configurations of uninitialised Solr Indexes
+ * @return the map with the index name as key and the properties as values
+ * @throws IOException on any error while loading the configurations
+ */
+ public static Map<String,java.util.Properties>
loadUninitialisedIndexConfigs(ComponentContext context) throws IOException {
+ File uninstalledConfigDir =
getUninitialisedSiteDirectory(context,false);
+ Map<String,java.util.Properties> configs = new
HashMap<String,java.util.Properties>();
+ if(uninstalledConfigDir.exists()){
+ for(String file : uninstalledConfigDir.list(new
SuffixFileFilter(ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION+".ref"))){
+ String indexName = file.substring(0,file.indexOf('.'));
+ java.util.Properties props = new java.util.Properties();
+ InputStream is = null;
+ try {
+ is = new FileInputStream(new
File(uninstalledConfigDir,file));
+ props.load(is);
+ configs.put(indexName, props);
+ } finally {
+ IOUtils.closeQuietly(is);
+ }
+ }
+ }
+ return configs;
+ }
+ /**
+ * Removes the configuration for the index with the parsed name form the
+ * list if uninitialised indexes
+ * @param indexName the name of the index
+ * @return if the file was deleted.
+ */
+ public static boolean removeUninitialisedIndexConfig(ComponentContext
context,String indexName){
+ File configFile = new
File(getUninitialisedSiteDirectory(context,false),
+ indexName+'.'+ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION+".ref");
+ return configFile.delete();
+ }
}
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/DefaultSolrDirectoryManager.java
Mon Apr 11 12:25:07 2011
@@ -16,6 +16,8 @@
*/
package org.apache.stanbol.entityhub.yard.solr.impl;
+import static org.apache.stanbol.entityhub.yard.solr.impl.ConfigUtils.*;
+
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -23,6 +25,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.Map.Entry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
@@ -60,6 +63,13 @@ public class DefaultSolrDirectoryManager
private final Logger log =
LoggerFactory.getLogger(DefaultSolrDirectoryManager.class);
/**
+ * This key is used to store the file name of the archive supposed to
provide
+ * the data for the uninitialised index within the configuration the
configuration
+ */
+ private static final String UNINITIALISED_INDEX_ARCHIVE_NAME_KEY =
"Uninitialised-Index-Archive-Name";
+
+
+ /**
* The dataFileProvider used to lookup index data
*/
@Reference
@@ -95,6 +105,17 @@ public class DefaultSolrDirectoryManager
* as an initialisation completed this set is notified.
*/
private Set<String> initCores = new HashSet<String>();
+ /**
+ * Holds the list of cores that where installed by using
+ * {@link #createSolrDirectory(String, String, java.util.Properties)} but
the
+ * {@link DataFileProvider} could not yet provide the necessary data for
the
+ * initialisation.<p>
+ * The list of uninitialised cores is stored within the data folder of the
+ * bundle under {@link #UNINITIALISED_SITE_DIRECTORY_NAME} and loaded at
+ * activation.
+ *
+ */
+ private Map<String,java.util.Properties> uninitialisedCores = new
HashMap<String,java.util.Properties>();
public DefaultSolrDirectoryManager() {
}
@@ -109,8 +130,12 @@ public class DefaultSolrDirectoryManager
if(solrIndexName.isEmpty()){
throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
}
- //also here we need to initialise the SolrDirectory if not already done
- return new
File(lookupManagedSolrDir(componentContext),solrIndexName).exists();
+ //first check if the directory for the parsed index exists
+ boolean exists = new
File(lookupManagedSolrDir(componentContext),solrIndexName).exists();
+ return !exists ? //if no directory exists
+ //check also if an uninitialised index was requested
+ uninitialisedCores.containsKey(solrIndexName):
+ true;
}
/* (non-Javadoc)
* @see
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#getManagedIndices()
@@ -123,29 +148,110 @@ public class DefaultSolrDirectoryManager
//TODO: validate that this is actually a SolrCore!
indexes.put(indexName, new File(solrDir,indexName));
}
+ //we need also add the uninitialised indexes (with a null as value)
+ for(String indexName: uninitialisedCores.keySet()){
+ if(!indexes.containsKey(indexName)){
+ indexes.put(indexName, null);
+ }
+ }
return indexes;
}
+ public boolean isInitialisedIndex(String solrIndexName){
+ if(solrIndexName == null){
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be NULL");
+ }
+ if(solrIndexName.isEmpty()){
+ throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
+ }
+ if(initCores.contains(solrIndexName)){ //if it is currently initialised
+ return false; //return false
+ } else { //check if the dir is there
+ return new
File(lookupManagedSolrDir(componentContext),solrIndexName).exists();
+ }
+ }
/* (non-Javadoc)
* @see
org.apache.stanbol.entityhub.yard.solr.impl.ManagedSolrDirectory#getSolrDirectory(java.lang.String)
*/
- public final File getSolrDirectory(final String solrIndexName,boolean
create) throws IllegalArgumentException {
- return initSolrDirectory(solrIndexName,null,create,componentContext);
+ public final File getSolrIndexDirectory(final String solrIndexName,boolean
allowDefaultInit) throws IllegalArgumentException {
+ return
initSolrDirectory(solrIndexName,null,allowDefaultInit,componentContext);
}
public final File createSolrIndex(final String solrIndexName,
ArchiveInputStream ais){
- return initSolrDirectory(solrIndexName,ais,true,componentContext);
+ return initSolrDirectory(solrIndexName,ais,false,componentContext);
}
@Override
- public final File createSolrDirectory(String solrIndexName, String
indexPath, Map<String,String> comments) throws
IllegalArgumentException,IOException {
- ComponentContext context = componentContext;
+ public final File createSolrDirectory(String solrIndexName, String
indexArchiveRef, java.util.Properties properties) throws
IllegalArgumentException,IOException {
if(componentContext == null){
throw new IllegalStateException("Creating an Index by using the
DataFileProvider does only work when running within an OSGI");
}
- //TODO add the comments how to download the index!
- InputStream is =
dataFileProvider.getInputStream(context.getBundleContext().getBundle().getSymbolicName(),
indexPath, comments);
- if(is == null){
- throw new IllegalStateException("SolrServer arvive "+indexPath+" is
currently not available via the "+DataFileProvider.class.getSimpleName()+"
service");
- }
- return
createSolrIndex(solrIndexName,ConfigUtils.getArchiveInputStream(indexPath, is));
+ //now add the index to the list of uninitialised
+ addUninitialisedIndex(solrIndexName,indexArchiveRef, properties);
+ return initSolrDirectory(solrIndexName, null, false, componentContext);
+ }
+
+ /**
+ * Tries to get the {@link ArchiveInputStream} for the index from the
+ * {@link DataFileProvider}.
+ * @param context the context used to perform the operations
+ * @param solrIndexName the name of the index to initialise
+ * @param properties the properties for this index. Must contain the
+ * {@link #UNINITIALISED_INDEX_ARCHIVE_NAME_KEY}.
+ * @return The {@link ArchiveInputStream} or <code>null</code> if the
+ * data are still not available
+ * @throws IOException on any IO related error while initialising the index
+ * @throws IllegalStateException if the parsed configuration does not
provide
+ * a value for {@link #UNINITIALISED_INDEX_ARCHIVE_NAME_KEY}.
+ */
+ private ArchiveInputStream lookupIndexArchive(ComponentContext context,
String solrIndexName,java.util.Properties properties) throws IOException,
IllegalStateException {
+ String archiveName =
properties.getProperty(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY);
+ if(archiveName == null){
+ throw new IllegalStateException("Found uninitialised index config
that does not contain the required "+
+ UNINITIALISED_INDEX_ARCHIVE_NAME_KEY+" property!");
+ }
+ //we need to copy the properties to a map
+ Map<String,String> propMap;
+ if(properties == null){
+ properties = new java.util.Properties(); //create an empty
properties file
+ propMap = null;
+ } else {
+ propMap = new HashMap<String,String>();
+ for(Entry<Object,Object> entry : properties.entrySet()){
+ propMap.put(entry.getKey().toString(),
entry.getValue()!=null?entry.getValue().toString():null);
+ }
+ }
+ propMap.remove(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY);//do not parse
this internal property
+ InputStream is =
dataFileProvider.getInputStream(context.getBundleContext().getBundle().getSymbolicName(),
archiveName, propMap);
+ return is == null?null:ConfigUtils.getArchiveInputStream(archiveName,
is);
+ }
+ private void addUninitialisedIndex(String indexName,String
sourceFileName,java.util.Properties config) throws IOException {
+ ComponentContext context = componentContext;
+ if(context == null){
+ throw new IllegalStateException("This feature is only available
when running within an OSGI environment");
+ }
+ if(config == null){
+ config = new java.util.Properties();
+ }
+ config.setProperty(UNINITIALISED_INDEX_ARCHIVE_NAME_KEY,
sourceFileName);
+ synchronized (uninitialisedCores) {
+ if(uninitialisedCores.put(indexName, config) != null){
+ removeUninitialisedIndexConfig(context,indexName); //remove
the old version
+ }
+ saveUninitialisedIndexConfig(context,indexName, config); //save
the new version
+ }
+ }
+ private void removeUninitialisedIndex(String indexName){
+ ComponentContext context = componentContext;
+ synchronized (uninitialisedCores) {
+ if(uninitialisedCores.remove(indexName) != null){
+ if(context == null){
+ //check only for the context if we need actually to remove
+ //an entry, because this method is also called outside an
+ //OSGI environment (but will never remove something from
+ //uninitialisedCores)
+ throw new IllegalStateException("This feature is only
available when running within an OSGI environment");
+ }
+ removeUninitialisedIndexConfig(context,indexName); //remove
the old version
+ }
+ }
}
/**
* Internally used to get/init the Solr directory of a SolrCore or the root
@@ -154,23 +260,24 @@ public class DefaultSolrDirectoryManager
* the root solr directory
* @param ais The Input stream of the Archive to load the index from or
* <code>null</code> to load the default core configuration.
- * @param create If <code>true</code> a new core is initialised if not
already
- * present. Make sure this is set to <code>true</code> if parsing an
InputStream.
- * Otherwise the index will not be created from the parsed stream!
+ * @param allowDefaultInitialisation If <code>true</code> a new core is
+ * initialised with the default configuration (empty index with the default
+ * Solr schema and configuration). If <code>false</code> the core is only
+ * created if a valid configuration is parsed.
* @param context A reference to the component context or
<code>null</code> if
* running outside an OSGI container. This is needed to avoid that
* {@link #deactivate(ComponentContext)} sets the context to
<code>null</code>
* during this method does its initialisation work.
* @return the Solr directory or <code>null</code> if the requested index
* could not be created (e.g. because of <code>false</code> was parsed as
- * create) orin case this component is deactivated
+ * create) or in case this component is deactivated
* @throws IllegalStateException in case this method is called when this
* component is running within an OSGI environment and it is deactivated or
* the initialisation for the parsed index failed.
* @throws IllegalArgumentException if the parsed solrIndexName is
<code>null</code> or
* empty.
*/
- private final File initSolrDirectory(final String
solrIndexName,ArchiveInputStream ais,boolean create,ComponentContext context)
throws IllegalStateException {
+ private final File initSolrDirectory(final String
solrIndexName,ArchiveInputStream ais,boolean
allowDefaultInitialisation,ComponentContext context) throws
IllegalStateException {
if(solrIndexName == null){
throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be NULL");
}
@@ -178,35 +285,64 @@ public class DefaultSolrDirectoryManager
throw new IllegalArgumentException("The parsed name of the Solr
index MUST NOT be empty");
}
File managedCoreContainerDirectory = lookupManagedSolrDir(context);
- if(solrIndexName == null){
- return managedCoreContainerDirectory;
+ if(solrIndexName == null){ //if the indexName is null
+ return managedCoreContainerDirectory; //return the root directory
}
File coreDir = new File(managedCoreContainerDirectory,solrIndexName);
if(!coreDir.exists()){
- if(!create){ //we are not allowed to create it
- return null; //return null
- }
+ //first add the index to the list of currently init cores
synchronized (initCores) {
- log.info(" > start initializing SolrIndex "+solrIndexName);
+ log.debug(" > start initializing SolrIndex {}"+solrIndexName);
initCores.add(solrIndexName);
}
+ //second check if the Index is an uninitialised one and if this is
the case
+ //try to get the ArchiveInputStream form the DataFileProvider
+ java.util.Properties uninitialisedProperties;
+ synchronized (uninitialisedCores) {
+ uninitialisedProperties =
uninitialisedCores.get(solrIndexName);
+ if(uninitialisedProperties != null){
+ //NOTE: this may override an parsed ArchiveInputStream
+ // -> this is an error by the implementation of this class
+ // so throw an Exception to detect such errors early!
+ if(ais != null){
+ throw new IllegalStateException("The parsed
ArchiveInputStream is not null for an uninitialised Index. " +
+ "Please report this error the the
stanbol-def mailing list!");
+ }
+ try {
+ ais = lookupIndexArchive(context, solrIndexName,
uninitialisedProperties);
+ } catch (Exception e) {
+ log.warn("The Index Archive for index {} not available
(see \"Stanbol Data File Provider\" Tab of the Apache Webconsole for
details).", solrIndexName);
+ }
+ }
+ }
+ //third do the actual initialisation work
try {
if(ais != null){
ConfigUtils.copyCore(ais, coreDir, solrIndexName, false);
- } else if(context != null){ //load via bundle
-
ConfigUtils.copyCore(context.getBundleContext().getBundle(),
- coreDir, null, false);
- } else { //load from jar
- ConfigUtils.copyCore((Class<?>)null, coreDir, null, false);
+ //try to remove from uninitialised
+ removeUninitialisedIndex(solrIndexName);
+ } else if(allowDefaultInitialisation){
+ //TODO: Refactor so that the lookup via Bundle and/or jar
+ // file works via an internal implementation of an
+ // FileDataProvider
+ if(context != null){ //load via bundle
+
ConfigUtils.copyCore(context.getBundleContext().getBundle(),
+ coreDir, null, false);
+ } else { //load from jar
+ ConfigUtils.copyCore((Class<?>)null, coreDir, null,
false);
+ }
}
} catch (Exception e) {
throw new IllegalStateException(
String.format("Unable to copy default configuration for
Solr Index %s to the configured path %s"
,solrIndexName==null?"":solrIndexName,managedCoreContainerDirectory.getAbsoluteFile()),e);
} finally {
+ //regardless what happened remove the index from the currently
init
+ //indexes and notify all other waiting for the initialisation
synchronized (initCores) {
+ //initialisation done
initCores.remove(solrIndexName);
- log.info(" ... finished inizializaiton of SolrIndex
"+solrIndexName);
+ log.debug(" ... notify after trying to init SolrIndex
{}"+solrIndexName);
//notify that the initialisation completed or failed
initCores.notifyAll();
}
@@ -324,13 +460,22 @@ public class DefaultSolrDirectoryManager
substitution.append(value.substring(prevAt, value.length()));
return substitution.toString();
}
+
@Activate
- protected void activate(ComponentContext context) {
+ protected void activate(ComponentContext context) throws IOException {
componentContext = context;
withinOSGI = true;
+ Map<String,java.util.Properties> uninitIndexes;
+ synchronized (uninitialisedCores) {
+
uninitialisedCores.putAll(loadUninitialisedIndexConfigs(componentContext));
+ }
}
@Deactivate
protected void deactivate(ComponentContext context) {
+ synchronized (uninitialisedCores) {
+ uninitialisedCores.clear();
+ }
componentContext = null;
+
}
}
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/SolrYard.java
Mon Apr 11 12:25:07 2011
@@ -344,7 +344,7 @@ public class SolrYard extends AbstractYa
File indexDirectory =
ConfigUtils.toFile(config.getSolrServerLocation());
if(!indexDirectory.isAbsolute()){ //relative paths
// need to be resolved based on the internally managed Solr
directory
- indexDirectory = solrDirectoryManager.getSolrDirectory(
+ indexDirectory = solrDirectoryManager.getSolrIndexDirectory(
indexDirectory.toString(),config.isDefaultInitialisation());
if(indexDirectory == null){
throw new
ConfigurationException(SolrYard.SOLR_SERVER_LOCATION,
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallTask.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallTask.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallTask.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallTask.java
Mon Apr 11 12:25:07 2011
@@ -16,22 +16,17 @@
*/
package org.apache.stanbol.entityhub.yard.solr.impl.install;
-import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.*;
+import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.PROPERTY_ARCHIVE_FORMAT;
+import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.PROPERTY_INDEX_ARCHIVE;
+import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.PROPERTY_INDEX_NAME;
import java.io.File;
-import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
-import java.util.Map.Entry;
-import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.archivers.ArchiveInputStream;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
-import
org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.sling.installer.api.tasks.InstallTask;
import org.apache.sling.installer.api.tasks.InstallationContext;
@@ -72,23 +67,19 @@ public class IndexInstallTask extends In
try {
is = getResource().getInputStream();
if("properties".equals(archiveFormat)){
- Map<String,String> properties = new
HashMap<String,String>();
InputStreamReader reader = new
InputStreamReader(is,"UTF-8");
+ Properties props = new Properties();
try {
- Properties props = new Properties();
props.load(reader);
- for(Entry<Object,Object> config : props.entrySet()){
-
properties.put(config.getKey().toString(),config.getValue()!=
null?config.getValue().toString():null);
- }
} finally {
IOUtils.closeQuietly(reader);
}
- String indexPath = properties.get(PROPERTY_INDEX_ARCHIVE);
+ String indexPath =
props.getProperty(PROPERTY_INDEX_ARCHIVE);
if(indexPath == null){
- indexPath =
indexName+'.'+IndexInstallerConstants.SOLR_INDEX_ARCHIVE_EXTENSION;
+ indexPath =
indexName+'.'+ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION;
log.info("Property \""+PROPERTY_INDEX_ARCHIVE+"\" not
present within the SolrIndex references file. Will use the default name
\""+indexPath+"\"");
}
-
solrDirectoryManager.createSolrDirectory(indexName,indexPath,properties);
+
solrDirectoryManager.createSolrDirectory(indexName,indexPath,props);
setFinishedState(ResourceState.INSTALLED);
} else {
ArchiveInputStream ais = null;
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallerConstants.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallerConstants.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallerConstants.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/IndexInstallerConstants.java
Mon Apr 11 12:25:07 2011
@@ -29,15 +29,16 @@ public final class IndexInstallerConstan
private IndexInstallerConstants(){ /* do not create instances*/ }
/**
- * Use <indexName>.solrindex[.<archiveType>] as file name
- */
- public static final String SOLR_INDEX_ARCHIVE_EXTENSION = "solrindex";
- /**
* The schema used for transformed resources.
*/
public static final String SOLR_INDEX_ARCHIVE_RESOURCE_TYPE =
"solrarchive";
private static final String PROPERTY_PREFIX =
"org.apache.stanbol.yard.solr.installer.";
+ /**
+ * The key used to configure the name of the Index-Archive
+ * The default name is \"<indexName>.solrarchive\".
+ */
+ public static final String PROPERTY_INDEX_ARCHIVE = "Index-Archive";
/**
* The key used for the name of the index
@@ -48,9 +49,4 @@ public final class IndexInstallerConstan
*/
public static final String PROPERTY_ARCHIVE_FORMAT =
PROPERTY_PREFIX+"archive.format";
- /**
- * The key used to configure the name of the Index-Archive
- * The default name is \"<indexName>.solrarchive\".
- */
- public static final String PROPERTY_INDEX_ARCHIVE = "Index-Archive";
}
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/SolrIndexInstaller.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/SolrIndexInstaller.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/SolrIndexInstaller.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/main/java/org/apache/stanbol/entityhub/yard/solr/impl/install/SolrIndexInstaller.java
Mon Apr 11 12:25:07 2011
@@ -16,11 +16,11 @@
*/
package org.apache.stanbol.entityhub.yard.solr.impl.install;
+import static
org.apache.stanbol.entityhub.yard.solr.impl.ConfigUtils.SOLR_INDEX_ARCHIVE_EXTENSION;
+import static
org.apache.stanbol.entityhub.yard.solr.impl.ConfigUtils.SUPPORTED_SOLR_ARCHIVE_FORMAT;
import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.PROPERTY_ARCHIVE_FORMAT;
import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.PROPERTY_INDEX_NAME;
-import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.SOLR_INDEX_ARCHIVE_EXTENSION;
import static
org.apache.stanbol.entityhub.yard.solr.impl.install.IndexInstallerConstants.SOLR_INDEX_ARCHIVE_RESOURCE_TYPE;
-import static
org.apache.stanbol.entityhub.yard.solr.impl.ConfigUtils.SUPPORTED_SOLR_ARCHIVE_FORMAT;
import java.util.HashMap;
import java.util.Map;
Modified:
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java
URL:
http://svn.apache.org/viewvc/incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java?rev=1091050&r1=1091049&r2=1091050&view=diff
==============================================================================
---
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java
(original)
+++
incubator/stanbol/trunk/entityhub/yard/solr/src/test/java/org/apache/stanbol/entityhub/yard/solr/SolrDirectoryManagerTest.java
Mon Apr 11 12:25:07 2011
@@ -71,12 +71,12 @@ public class SolrDirectoryManagerTest {
@Test(expected=IllegalArgumentException.class)
public void testNullIndexName(){
- solrDirectoryManager.getSolrDirectory(null,true);
+ solrDirectoryManager.getSolrIndexDirectory(null,true);
}
@Test(expected=IllegalArgumentException.class)
public void testEmptyIndexName(){
- solrDirectoryManager.getSolrDirectory("",true);
+ solrDirectoryManager.getSolrIndexDirectory("",true);
}
@Test
@@ -105,7 +105,7 @@ public class SolrDirectoryManagerTest {
//this is actually tested already by the initialisation of the
//SolrYardTest ...
String indexName =
"testIndexInitialisation_"+System.currentTimeMillis();
- File indexDir = solrDirectoryManager.getSolrDirectory(indexName,true);
+ File indexDir =
solrDirectoryManager.getSolrIndexDirectory(indexName,true);
assertEquals(new File(expectedManagedDirectory,indexName), indexDir);
assertTrue(indexDir.isDirectory());
}
