------------------------------------------------------------ revno: 9162 committer: Lars Helge Øverland <[email protected]> branch nick: dhis2 timestamp: Mon 2012-12-03 22:04:39 +0100 message: Initial work on new data analytics engine added: dhis-2/dhis-services/dhis-service-analytics/ dhis-2/dhis-services/dhis-service-analytics/pom.xml dhis-2/dhis-services/dhis-service-analytics/src/ dhis-2/dhis-services/dhis-service-analytics/src/main/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/resources/ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml modified: dhis-2/dhis-services/pom.xml dhis-2/pom.xml
-- lp:dhis2 https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk Your team DHIS 2 developers is subscribed to branch lp:dhis2. To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== added directory 'dhis-2/dhis-services/dhis-service-analytics' === added file 'dhis-2/dhis-services/dhis-service-analytics/pom.xml' --- dhis-2/dhis-services/dhis-service-analytics/pom.xml 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/pom.xml 2012-12-03 21:04:39 +0000 @@ -0,0 +1,65 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-services</artifactId> + <version>2.11-SNAPSHOT</version> + </parent> + + <artifactId>dhis-service-analytics</artifactId> + <packaging>jar</packaging> + <name>DHIS Analytics</name> + + <dependencies> + + <!-- DHIS --> + + <dependency> + <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-api</artifactId> + </dependency> + <dependency> + <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-service-core</artifactId> + </dependency> + <dependency> + <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-service-administration</artifactId> + </dependency> + <dependency> + <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-support-system</artifactId> + </dependency> + + <!-- Other --> + + <dependency> + <groupId>jep</groupId> + <artifactId>jep</artifactId> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>org.amplecode</groupId> + <artifactId>quick</artifactId> + </dependency> + <dependency> + <groupId>com.whalin</groupId> + <artifactId>Memcached-Java-Client</artifactId> + <version>3.0.1</version> + </dependency> + + </dependencies> + <properties> + <rootDir>../../</rootDir> + </properties> +</project> === added directory 'dhis-2/dhis-services/dhis-service-analytics/src' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsManager.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,40 @@ +package org.hisp.dhis.analytics; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Future; + +import org.hisp.dhis.aggregation.AggregatedDataValue; + +public interface AnalyticsManager +{ + Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds, + Collection<String> periodIds, Collection<Integer> organisationUnitIds ); +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsService.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,39 @@ +package org.hisp.dhis.analytics; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; +import java.util.List; + +import org.hisp.dhis.aggregation.AggregatedDataValue; + +public interface AnalyticsService +{ + List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds, + Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception; +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,54 @@ +package org.hisp.dhis.analytics; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.List; +import java.util.concurrent.Future; + +public interface AnalyticsTableManager +{ + void dropTable(); + + void createTable(); + + void populateTable(); + + Future<?> createIndexesAsync( List<String> columns ); + + /** + * Returns a list of string arrays in where the first index holds the database + * column name, the second index holds the database column type and the third + * column holds a table alias. + */ + List<String[]> getDimensionColumns(); + + /** + * Returns a list of database column names. + */ + List<String> getDimensionColumnNames(); +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,35 @@ +package org.hisp.dhis.analytics; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.concurrent.Future; + +public interface AnalyticsTableService +{ + Future<?> update(); +} === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,73 @@ +package org.hisp.dhis.analytics.data; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Future; + +import org.hisp.dhis.aggregation.AggregatedDataValue; +import org.hisp.dhis.analytics.AnalyticsManager; +import org.hisp.dhis.analytics.AnalyticsService; +import org.hisp.dhis.system.util.PaginatedList; +import org.hisp.dhis.system.util.Timer; +import org.springframework.beans.factory.annotation.Autowired; + +public class DefaultAnalyticsService + implements AnalyticsService +{ + @Autowired + private AnalyticsManager analyticsManager; + + public List<AggregatedDataValue> getAggregatedDataValueTotals( Collection<Integer> dataElementIds, + Collection<String> periodIds, Collection<Integer> organisationUnitIds ) throws Exception + { + Timer t = new Timer().start(); + + List<List<Integer>> dePages = new PaginatedList<Integer>( dataElementIds ).setNumberOfPages( 4 ).getPages(); + + List<Future<List<AggregatedDataValue>>> futures = new ArrayList<Future<List<AggregatedDataValue>>>(); + + List<AggregatedDataValue> values = new ArrayList<AggregatedDataValue>(); + + for ( List<Integer> dePage : dePages ) + { + futures.add( analyticsManager.getAggregatedDataValueTotals( values, dePage, periodIds, organisationUnitIds ) ); + } + + for ( Future<List<AggregatedDataValue>> future : futures ) + { + values.addAll( future.get() ); + } + + t.getTime( "Got aggregated values" ); + + return values; + } +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/JdbcAnalyticsManager.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,102 @@ +package org.hisp.dhis.analytics.data; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.system.util.TextUtils.getCommaDelimitedString; +import static org.hisp.dhis.system.util.TextUtils.getQuotedCommaDelimitedString; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.aggregation.AggregatedDataValue; +import org.hisp.dhis.analytics.AnalyticsManager; +import org.hisp.dhis.expression.ExpressionService; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.PeriodType; +import org.hisp.dhis.system.objectmapper.AggregatedDataValueRowMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.AsyncResult; + +/** + * This class is responsible for producing aggregated data values. It reads data + * from the analytics table. Organisation units provided as arguments must be on + * the same level in the hierarchy. + * + * @author Lars Helge Overland + */ +public class JdbcAnalyticsManager + implements AnalyticsManager +{ + private static final Log log = LogFactory.getLog( JdbcAnalyticsManager.class ); + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Autowired + private OrganisationUnitService organisationUnitService; + + @Autowired + private PeriodService periodService; + + @Autowired + private ExpressionService expressionService; + + // ------------------------------------------------------------------------- + // Implementation + // ------------------------------------------------------------------------- + + //TODO period aggregation for multiple period types + //TODO hierarchy aggregation for org units at multiple levels + //TODO indicator aggregation + + @Async + public Future<List<AggregatedDataValue>> getAggregatedDataValueTotals( Collection<AggregatedDataValue> values, Collection<Integer> dataElementIds, + Collection<String> periodIds, Collection<Integer> organisationUnitIds ) + { + int level = organisationUnitService.getLevelOfOrganisationUnit( organisationUnitIds.iterator().next() ); + String periodType = PeriodType.getPeriodTypeFromIsoString( periodIds.iterator().next() ).getName().toLowerCase(); + + final String sql = + "SELECT dataelementid, 0 as categoryoptioncomboid, periodid, idlevel" + level + " as organisationunitid, SUM(value) as value " + + "FROM analytics " + + "WHERE dataelementid IN ( " + getCommaDelimitedString( dataElementIds ) + " ) " + + "AND " + periodType + " IN ( " + getQuotedCommaDelimitedString( periodIds ) + " ) " + + "AND idlevel" + level + " IN ( " + getCommaDelimitedString( organisationUnitIds ) + " ) " + + "GROUP BY dataelementid, periodid, idlevel" + level; + + log.info( sql ); + + return new AsyncResult<List<AggregatedDataValue>>( jdbcTemplate.query( sql, new AggregatedDataValueRowMapper() ) ); + } +} === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,97 @@ +package org.hisp.dhis.analytics.table; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.analytics.AnalyticsTableManager; +import org.hisp.dhis.analytics.AnalyticsTableService; +import org.hisp.dhis.system.util.ConcurrentUtils; +import org.hisp.dhis.system.util.PaginatedList; +import org.hisp.dhis.system.util.SystemUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; + +public class DefaultAnalyticsTableService + implements AnalyticsTableService +{ + private static final Log log = LogFactory.getLog( DefaultAnalyticsTableService.class ); + + @Autowired + private AnalyticsTableManager tableManager; + + // ------------------------------------------------------------------------- + // Implementation + // ------------------------------------------------------------------------- + + //TODO generateOrganisationUnitStructures + //TODO generateOrganisationUnitGroupSetTable + //TODO generatePeriodStructure + + @Async + public Future<?> update() + { + log.info( "Starting update..." ); + + tableManager.dropTable(); + tableManager.createTable(); + tableManager.populateTable(); + + createIndexes(); + + log.info( "Update done" ); + + return null; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + private void createIndexes() + { + int pages = Math.max( ( SystemUtils.getCpuCores() - 1 ), 1 ); + + log.info( "No of pages: " + pages ); + + List<Future<?>> futures = new ArrayList<Future<?>>(); + + List<List<String>> columnPages = new PaginatedList<String>( tableManager.getDimensionColumnNames() ).setNumberOfPages( pages ).getPages(); + + for ( List<String> columnPage : columnPages ) + { + futures.add( tableManager.createIndexesAsync( columnPage ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + } +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2012-12-03 21:04:39 +0000 @@ -0,0 +1,244 @@ +package org.hisp.dhis.analytics.table; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.analytics.AnalyticsTableManager; +import org.hisp.dhis.organisationunit.OrganisationUnitGroupService; +import org.hisp.dhis.organisationunit.OrganisationUnitGroupSet; +import org.hisp.dhis.organisationunit.OrganisationUnitLevel; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.PeriodType; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Async; + +/** + * This class manages the analytics table. The analytics table is a denormalized + * table designed for analysis which contains raw data values. It has columns for + * each organisation unit group set and organisation unit level. Also, columns + * for dataelementid, periodid, organisationunitid, categoryoptioncomboid, value. + * + * The data records in this table are not aggregated. Typically, queries will + * aggregate in organisation unit hierarchy dimension, in the period/time dimension, + * and the category dimensions, as well as org unit group set dimensions. + * + * @author Lars Helge Overland + */ +public class JdbcAnalyticsTableManager + implements AnalyticsTableManager +{ + private static final Log log = LogFactory.getLog( JdbcAnalyticsTableManager.class ); + + public static final String PREFIX_ORGUNITGROUPSET = "ougs_"; + public static final String PREFIX_ORGUNITLEVEL = "idlevel"; + public static final String PREFIX_INDEX = "index_"; + public static final String TABLE_NAME = "analytics"; + + @Autowired + private OrganisationUnitService organisationUnitService; + + @Autowired + private OrganisationUnitGroupService organisationUnitGroupService; + + @Autowired + private JdbcTemplate jdbcTemplate; + + // ------------------------------------------------------------------------- + // Implementation + // ------------------------------------------------------------------------- + + //TODO all data types + //TODO create temp table then swap + //TODO shard on data year + //TODO average aggregation operator data, pre-aggregate in time dimension, not in org unit dimension + + public void dropTable() + { + final String sql = "drop table " + TABLE_NAME; + + executeSilently( sql ); + + log.info( "Dropped table: " + TABLE_NAME ); + } + + public void createTable() + { + String sql = "create table " + TABLE_NAME + " ("; + + for ( String[] col : getDimensionColumns() ) + { + sql += col[0] + " " + col[1] + ","; + } + + sql += "value double precision)"; + + log.info( "Create SQL: " + sql ); + + executeSilently( sql ); + + log.info( "Created table: " + TABLE_NAME ); + } + + @Async + public Future<?> createIndexesAsync( List<String> columns ) + { + for ( String column : columns ) + { + final String sql = "create index " + PREFIX_INDEX + + column + " on " + TABLE_NAME + " (" + column + ")"; + + executeSilently( sql ); + + log.info( "Created index on column: " + column ); + } + + log.info( "Indexes created" ); + + return null; + } + + public void populateTable() + { + String insert = "insert into analytics ("; + + for ( String[] col : getDimensionColumns() ) + { + insert += col[0] + ","; + } + + insert += "value) "; + + String select = "select "; + + for ( String[] col : getDimensionColumns() ) + { + select += col[2] + col[0] + ","; + } + + select = select.replace( "organisationunitid", "sourceid" ); // Legacy fix + + select += + "cast(dv.value as double precision) " + + "from datavalue dv " + + "left join _organisationunitgroupsetstructure ougs on dv.sourceid=ougs.organisationunitid " + + "left join _orgunitstructure ous on dv.sourceid=ous.organisationunitid " + + "left join _period_no_disaggregation_structure ps on dv.periodid=ps.periodid " + + "left join dataelement de on dv.dataelementid=de.dataelementid " + + "left join period pe on dv.periodid=pe.periodid " + + "where de.valuetype='int' and pe.startdate >= '2011-10-01'"; + + final String sql = insert + select; + + log.info( "Populate SQL: "+ sql ); + + jdbcTemplate.execute( sql ); + + log.info( "Populated analytics table" ); + } + + public List<String[]> getDimensionColumns() + { + List<String[]> columns = new ArrayList<String[]>(); + + Collection<OrganisationUnitGroupSet> orgUnitGroupSets = + organisationUnitGroupService.getCompulsoryOrganisationUnitGroupSets(); + + Collection<OrganisationUnitLevel> levels = + organisationUnitService.getOrganisationUnitLevels(); + + for ( OrganisationUnitGroupSet groupSet : orgUnitGroupSets ) + { + String[] col = { PREFIX_ORGUNITGROUPSET + groupSet.getUid(), "integer", "ougs." }; + columns.add( col ); + } + + for ( OrganisationUnitLevel level : levels ) + { + String[] col = { PREFIX_ORGUNITLEVEL + level.getLevel(), "integer", "ous." }; + columns.add( col ); + } + + for ( PeriodType periodType : PeriodType.getAvailablePeriodTypes().subList( 0, 7 ) ) + { + String[] col = { periodType.getName().toLowerCase(), "character varying(10)", "ps." }; + columns.add( col ); + } + + String[] de = { "dataelementid", "integer not null", "dv." }; + String[] pe = { "periodid", "integer not null", "dv." }; + String[] ou = { "organisationunitid", "integer not null", "dv." }; + String[] co = { "categoryoptioncomboid", "integer not null", "dv." }; + + columns.addAll( Arrays.asList( de, pe, ou, co ) ); + + return columns; + } + + public List<String> getDimensionColumnNames() + { + List<String[]> columns = getDimensionColumns(); + + List<String> columnNames = new ArrayList<String>(); + + for ( String[] column : columns ) + { + columnNames.add( column[0] ); + } + + return columnNames; + } + + // ------------------------------------------------------------------------- + // Supportive methods + // ------------------------------------------------------------------------- + + /** + * Executes a SQL statement. Ignores existing tables/indexes when attempting + * to create new. + */ + private void executeSilently( String sql ) + { + try + { + jdbcTemplate.execute( sql ); + } + catch ( BadSqlGrammarException ex ) + { + log.warn( ex.getMessage() ); + } + } +} === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF' === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2012-12-03 21:04:39 +0000 @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<beans xmlns="http://www.springframework.org/schema/beans"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:aop="http://www.springframework.org/schema/aop"; xmlns:sec="http://www.springframework.org/schema/security"; + xsi:schemaLocation=" +http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd +http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd +http://www.springframework.org/schema/security http://www.springframework.org/schema/security/spring-security-3.1.xsd";> + + <bean id="org.hisp.dhis.analytics.AnalyticsTableManager" class="org.hisp.dhis.analytics.table.JdbcAnalyticsTableManager" /> + + <bean id="org.hisp.dhis.analytics.AnalyticsTableService" class="org.hisp.dhis.analytics.table.DefaultAnalyticsTableService" /> + + <bean id="org.hisp.dhis.analytics.AnalyticsManager" class="org.hisp.dhis.analytics.data.JdbcAnalyticsManager" /> + + <bean id="org.hisp.dhis.analytics.AnalyticsService" class="org.hisp.dhis.analytics.data.DefaultAnalyticsService" /> + +</beans> === modified file 'dhis-2/dhis-services/pom.xml' --- dhis-2/dhis-services/pom.xml 2012-11-20 13:38:49 +0000 +++ dhis-2/dhis-services/pom.xml 2012-12-03 21:04:39 +0000 @@ -22,6 +22,7 @@ <module>dhis-service-reporting</module> <module>dhis-service-mapgeneration</module> <module>dhis-service-aggregationengine-default</module> + <module>dhis-service-analytics</module> <module>dhis-service-sms</module> <module>dhis-service-patient</module> <module>dhis-service-mobile</module> === modified file 'dhis-2/pom.xml' --- dhis-2/pom.xml 2012-11-29 16:21:29 +0000 +++ dhis-2/pom.xml 2012-12-03 21:04:39 +0000 @@ -285,6 +285,11 @@ </dependency> <dependency> <groupId>org.hisp.dhis</groupId> + <artifactId>dhis-service-analytics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.hisp.dhis</groupId> <artifactId>dhis-support-jdbc</artifactId> <version>${project.version}</version> </dependency>
_______________________________________________ Mailing list: https://launchpad.net/~dhis2-devs Post to : [email protected] Unsubscribe : https://launchpad.net/~dhis2-devs More help : https://help.launchpad.net/ListHelp
