------------------------------------------------------------ revno: 17139 committer: Lars Helge Overland <larshe...@gmail.com> branch nick: dhis2 timestamp: Thu 2014-10-16 16:51:51 +0200 message: Analytics. Added validation of analytics tables. modified: dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java
-- lp:dhis2 https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk Your team DHIS 2 developers is subscribed to branch lp:dhis2. To unsubscribe from this branch go to https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java 2014-10-07 12:16:23 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/util/Timer.java 2014-10-16 14:51:51 +0000 @@ -38,7 +38,7 @@ private long startTime; private boolean printDisabled; - + public Timer disablePrint() { this.printDisabled = true; === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultAnalyticsService.java 2014-10-16 14:51:51 +0000 @@ -577,8 +577,6 @@ Grid grid = getAggregatedDataValues( params ); Map<String, Double> valueMap = getAggregatedDataValueMapping( grid ); - - log.info( "Got aggregated values for table layout" ); return reportTable.getGrid( new ListGrid( grid.getMetaData() ), valueMap, false ); } @@ -743,11 +741,11 @@ int optimalQueries = MathUtils.getWithin( getProcessNo(), 1, MAX_QUERIES ); - Timer t = new Timer().start(); + Timer t = new Timer().start().disablePrint(); DataQueryGroups queryGroups = queryPlanner.planQuery( params, optimalQueries, tableName ); - log.debug( "Planned analytics query, got: " + queryGroups.getLargestGroupSize() + " for optimal: " + optimalQueries ); + t.getSplitTime( "Planned analytics query, got: " + queryGroups.getLargestGroupSize() + " for optimal: " + optimalQueries ); Map<String, Object> map = new HashMap<>(); === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-10-16 12:28:42 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/event/data/DefaultEventAnalyticsService.java 2014-10-16 14:51:51 +0000 @@ -48,8 +48,6 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.hisp.dhis.analytics.AnalyticsSecurityManager; import org.hisp.dhis.analytics.AnalyticsService; import org.hisp.dhis.analytics.SortOrder; @@ -85,9 +83,9 @@ import org.hisp.dhis.system.grid.ListGrid; import org.hisp.dhis.system.util.DateUtils; import org.hisp.dhis.system.util.ListUtils; -import org.hisp.dhis.util.Timer; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.hisp.dhis.trackedentity.TrackedEntityAttributeService; +import org.hisp.dhis.util.Timer; import org.springframework.beans.factory.annotation.Autowired; /** @@ -95,9 +93,7 @@ */ public class DefaultEventAnalyticsService implements EventAnalyticsService -{ - private static final Log log = LogFactory.getLog( DefaultEventAnalyticsService.class ); - +{ private static final String ITEM_EVENT = "psi"; private static final String ITEM_PROGRAM_STAGE = "ps"; private static final String ITEM_EXECUTION_DATE = "eventdate"; @@ -178,11 +174,11 @@ // Data // --------------------------------------------------------------------- - Timer t = new Timer().start(); + Timer t = new Timer().start().disablePrint(); List<EventQueryParams> queries = queryPlanner.planAggregateQuery( params ); - log.debug( "Planned event query, got partitions: " + params.getPartitions() ); + t.getSplitTime( "Planned event query, got partitions: " + params.getPartitions() ); for ( EventQueryParams query : queries ) { @@ -285,7 +281,7 @@ params = queryPlanner.planEventQuery( params ); - log.debug( "Planned event query, got partitions: " + params.getPartitions() ); + t.getSplitTime( "Planned event query, got partitions: " + params.getPartitions() ); int count = 0; === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java 2014-10-16 14:51:51 +0000 @@ -32,6 +32,7 @@ import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Future; @@ -50,6 +51,7 @@ import org.hisp.dhis.period.Cal; import org.hisp.dhis.period.Period; import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.system.util.ListUtils; import org.joda.time.DateTime; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.BadSqlGrammarException; @@ -328,4 +330,30 @@ log.debug( ex.getMessage() ); } } + + /** + * Checks whether the given list of dimensions are valid. + * @throws IllegalStateException if not valid. + */ + protected void validateDimensionColumns( List<String[]> dimensions ) + { + if ( dimensions == null || dimensions.isEmpty() ) + { + throw new IllegalStateException( "Analytics table dimensions are empty" ); + } + + List<String> columns = new ArrayList<>(); + + for ( String[] dimension : dimensions ) + { + columns.add( dimension[0] ); + } + + Set<String> duplicates = ListUtils.getDuplicates( columns ); + + if ( !duplicates.isEmpty() ) + { + throw new IllegalStateException( "Analytics table dimensions contain duplicates: " + duplicates ); + } + } } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java 2014-10-16 14:51:51 +0000 @@ -114,8 +114,12 @@ executeSilently( sqlDrop ); String sqlCreate = "create table " + tableName + " ("; - - for ( String[] col : getDimensionColumns( table ) ) + + List<String[]> columns = getDimensionColumns( table ); + + validateDimensionColumns( columns ); + + for ( String[] col : columns ) { sqlCreate += col[0] + " " + col[1] + ","; } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java 2014-10-16 14:51:51 +0000 @@ -77,8 +77,12 @@ executeSilently( sqlDrop ); String sqlCreate = "create table " + tableName + " ("; - - for ( String[] col : getDimensionColumns( table ) ) + + List<String[]> columns = getDimensionColumns( table ); + + validateDimensionColumns( columns ); + + for ( String[] col : columns ) { sqlCreate += col[0] + " " + col[1] + ","; } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTargetTableManager.java 2014-10-16 14:51:51 +0000 @@ -79,7 +79,11 @@ String sqlCreate = "create table " + tableName + " ("; - for ( String[] col : getDimensionColumns( table ) ) + List<String[]> columns = getDimensionColumns( table ); + + validateDimensionColumns( columns ); + + for ( String[] col : columns ) { sqlCreate += col[0] + " " + col[1] + ","; } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java 2014-10-16 14:51:51 +0000 @@ -125,7 +125,11 @@ String sqlCreate = "create table " + tableName + " ("; - for ( String[] col : getDimensionColumns( table ) ) + List<String[]> columns = getDimensionColumns( table ); + + validateDimensionColumns( columns ); + + for ( String[] col : columns ) { sqlCreate += col[0] + " " + col[1] + ","; } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 2014-10-16 06:17:19 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcOrgUnitTargetTableManager.java 2014-10-16 14:51:51 +0000 @@ -78,7 +78,11 @@ String sqlCreate = "create table " + tableName + " ("; - for ( String[] col : getDimensionColumns( table ) ) + List<String[]> columns = getDimensionColumns( table ); + + validateDimensionColumns( columns ); + + for ( String[] col : columns ) { sqlCreate += col[0] + " " + col[1] + ","; } === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java 2014-09-28 17:12:48 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ListUtils.java 2014-10-16 14:51:51 +0000 @@ -150,7 +150,7 @@ * @param comparator the comparator. * @return a set of duplicates from the given list. */ - public static <T> Collection<T> getDuplicates( List<T> list, Comparator<T> comparator ) + public static <T> Set<T> getDuplicates( List<T> list, Comparator<T> comparator ) { Set<T> duplicates = new HashSet<>(); @@ -170,6 +170,28 @@ return duplicates; } + + /** + * Returns the duplicates in the given list. + * + * @param list the list. + * @return a set of duplicates from the given list. + */ + public static <T> Set<T> getDuplicates( List<T> list ) + { + Set<T> duplicates = new HashSet<>(); + UniqueArrayList<T> uniqueElements = new UniqueArrayList<>(); + + for ( T element : list ) + { + if ( !uniqueElements.add( element ) ) + { + duplicates.add( element ); + } + } + + return duplicates; + } /** * Returns a Collection with the given items. === modified file 'dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java' --- dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java 2014-08-15 07:40:20 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/util/ListUtilsTest.java 2014-10-16 14:51:51 +0000 @@ -30,7 +30,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.junit.Test; @@ -58,4 +60,15 @@ assertTrue( list.contains( "e" ) ); assertTrue( list.contains( "g" ) ); } + + @Test + public void testGetDuplicates() + { + List<String> list = new ArrayList<>( Arrays.asList( "a", "b", "c", "c", "d", "e", "e", "e", "f" ) ); + Set<String> expected = new HashSet<>( Arrays.asList( "c", "e" ) ); + assertEquals( expected, ListUtils.getDuplicates( list ) ); + + list = new ArrayList<>( Arrays.asList( "a", "b", "c", "d", "e", "f", "g", "h" ) ); + assertEquals( 0, ListUtils.getDuplicates( list ).size() ); + } }
_______________________________________________ Mailing list: https://launchpad.net/~dhis2-devs Post to : dhis2-devs@lists.launchpad.net Unsubscribe : https://launchpad.net/~dhis2-devs More help : https://help.launchpad.net/ListHelp