- Improve Geonames startup to make it more resilient to missing mappings - Add sanity checks for endpoints before adding them - Add database copying (Geocity and Geonames) to test script.
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/603346ad Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/603346ad Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/603346ad Branch: refs/heads/master Commit: 603346ad3fc50756edbeeaa19e6204e7b82815fe Parents: e6ec7a0 Author: Serge Huber <[email protected]> Authored: Mon Dec 19 16:36:41 2016 +0100 Committer: Serge Huber <[email protected]> Committed: Mon Dec 19 16:36:41 2016 +0100 ---------------------------------------------------------------------- .gitignore | 3 + buildAndRunNoTests.sh | 8 ++ .../geonames/services/GeonamesServiceImpl.java | 85 ++++++++++++-------- .../ElasticSearchPersistenceServiceImpl.java | 25 +++--- .../persistence/spi/PersistenceService.java | 7 +- 5 files changed, 85 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index efa4748..bf90830 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ target .DS_Store /performance-tests/.cache maven-metadata-local.xml +GeoLite2-City.mmdb +allCountries.zip +rest/.miredot-offline.json http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/buildAndRunNoTests.sh ---------------------------------------------------------------------- diff --git a/buildAndRunNoTests.sh b/buildAndRunNoTests.sh index c8f7641..fa53832 100755 --- a/buildAndRunNoTests.sh +++ b/buildAndRunNoTests.sh @@ -27,6 +27,14 @@ mvn clean install -P \!integration-tests,\!performance-tests,rat pushd package/target echo Uncompressing Unomi package... tar zxvf unomi-$UNOMI_VERSION.tar.gz +if [ -f "../../GeoLite2-City.mmdb" ]; then + echo Installing GeoLite2 City database... + cp ../../GeoLite2-City.mmdb unomi-$UNOMI_VERSION/etc +fi +if [ -f "../../allCountries.zip" ]; then + echo Installing Geonames countries database... + cp ../../allCountries.zip unomi-$UNOMI_VERSION/etc +fi cd unomi-$UNOMI_VERSION/bin echo Starting Unomi... ./karaf debug http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java ---------------------------------------------------------------------- diff --git a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java index e89a630..ebe53a0 100644 --- a/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java +++ b/extensions/geonames/services/src/main/java/org/apache/unomi/geonames/services/GeonamesServiceImpl.java @@ -86,46 +86,67 @@ public class GeonamesServiceImpl implements GeonamesService { } final File f = new File(pathToGeonamesDatabase); if (f.exists()) { - Timer t = new Timer(); + final Timer t = new Timer(); t.schedule(new TimerTask() { @Override public void run() { - try { - ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(f)); - ZipEntry zipEntry = zipInputStream.getNextEntry(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, "UTF-8")); - - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - String line; - logger.info("Starting to import geonames database ..."); - while ((line = reader.readLine()) != null) { - String[] values = line.split("\t"); - - if (FEATURES_CLASSES.contains(values[6])) { - GeonameEntry geonameEntry = new GeonameEntry(values[0], values[1], values[2], - StringUtils.isEmpty(values[4]) ? null : Double.parseDouble(values[4]), - StringUtils.isEmpty(values[5]) ? null : Double.parseDouble(values[5]), - values[6], values[7], values[8], - Arrays.asList(values[9].split(",")), - values[10], values[11], values[12], values[13], - StringUtils.isEmpty(values[14]) ? null : Integer.parseInt(values[14]), - StringUtils.isEmpty(values[15]) ? null : Integer.parseInt(values[15]), - values[16], values[17], - sdf.parse(values[18])); - - persistenceService.save(geonameEntry); - } - } - logger.info("Geonames database imported"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } + importGeoNameDatabase(f, t); } }, 5000); } } + private void importGeoNameDatabase(final File f, final Timer t) { + Map<String,Map<String,Object>> typeMappings = persistenceService.getPropertiesMapping(GeonameEntry.ITEM_TYPE); + if (typeMappings == null) { + logger.warn("Type mappings for type {} are not yet installed, delaying import until they are ready!", GeonameEntry.ITEM_TYPE); + t.schedule(new TimerTask() { + @Override + public void run() { + importGeoNameDatabase(f, t); + } + }, 5000); + return; + } + try { + + ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(f)); + ZipEntry zipEntry = zipInputStream.getNextEntry(); // used to advance to the first entry in the ZipInputStream + BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, "UTF-8")); + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String line; + logger.info("Starting to import geonames database from file {}...", f); + long lineCount = 0; + long importStartTime = System.currentTimeMillis(); + while ((line = reader.readLine()) != null) { + String[] values = line.split("\t"); + + if (FEATURES_CLASSES.contains(values[6])) { + GeonameEntry geonameEntry = new GeonameEntry(values[0], values[1], values[2], + StringUtils.isEmpty(values[4]) ? null : Double.parseDouble(values[4]), + StringUtils.isEmpty(values[5]) ? null : Double.parseDouble(values[5]), + values[6], values[7], values[8], + Arrays.asList(values[9].split(",")), + values[10], values[11], values[12], values[13], + StringUtils.isEmpty(values[14]) ? null : Integer.parseInt(values[14]), + StringUtils.isEmpty(values[15]) ? null : Integer.parseInt(values[15]), + values[16], values[17], + sdf.parse(values[18])); + + persistenceService.save(geonameEntry); + } + lineCount++; + if (lineCount % 1000 == 0) { + logger.info("{} lines imported from file {}", lineCount, f); + } + } + logger.info("{} lines from Geonames database file {} imported in {}ms", lineCount, f, System.currentTimeMillis()-importStartTime); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + public List<GeonameEntry> getHierarchy(String itemId) { return getHierarchy(persistenceService.load(itemId, GeonameEntry.class)); } http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java ---------------------------------------------------------------------- diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index f1912bd..5e806d1 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -33,7 +33,10 @@ import org.apache.unomi.api.query.DateRange; import org.apache.unomi.api.query.IpRange; import org.apache.unomi.api.query.NumericRange; import org.apache.unomi.api.services.ClusterService; -import org.apache.unomi.persistence.elasticsearch.conditions.*; +import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; +import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; @@ -1516,15 +1519,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, ClusterNode clusterNode = new ClusterNode(); clusterNode.setHostName(karafCellarNode.getHost()); String publicEndpoint = publicNodeEndpoints.get(karafCellarNode.getId()); - String[] publicEndpointParts = publicEndpoint.split(":"); - clusterNode.setHostAddress(publicEndpointParts[0]); - clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1])); + if (publicEndpoint != null) { + String[] publicEndpointParts = publicEndpoint.split(":"); + clusterNode.setHostAddress(publicEndpointParts[0]); + clusterNode.setPublicPort(Integer.parseInt(publicEndpointParts[1])); + } String secureEndpoint = secureNodeEndpoints.get(karafCellarNode.getId()); - String[] secureEndpointParts = secureEndpoint.split(":"); - clusterNode.setSecureHostAddress(secureEndpointParts[0]); - clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1])); - clusterNode.setMaster(false); - clusterNode.setData(false); + if (secureEndpoint != null) { + String[] secureEndpointParts = secureEndpoint.split(":"); + clusterNode.setSecureHostAddress(secureEndpointParts[0]); + clusterNode.setSecurePort(Integer.parseInt(secureEndpointParts[1])); + clusterNode.setMaster(false); + clusterNode.setData(false); + } try { // now let's connect to remote JMX service to retrieve information from the runtime and operating system MX beans JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+karafCellarNode.getHost() + ":"+karafJMXPort+"/karaf-root"); http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/603346ad/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java ---------------------------------------------------------------------- diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java index 5501645..e191829 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java @@ -163,9 +163,12 @@ public interface PersistenceService { boolean removeQuery(String queryName); /** - * TODO + * Retrieve the type mappings for a given itemType. This method queries the persistence service implementation + * to retrieve any type mappings it may have for the specified itemType. * - * @param itemType + * This method may not return any results if the implementation doesn't support property type mappings + * + * @param itemType the itemType we want to retrieve the mappings for * @return */ Map<String, Map<String, Object>> getPropertiesMapping(String itemType);
