------------------------------------------------------------
revno: 3417
committer: Lars Helge Overland <[email protected]>
branch nick: dhis2
timestamp: Sat 2011-04-16 18:18:58 +0200
message:
Crosstab step/aggregation in datamart process now runs agains an in-memory
HSQL database. This means that we are not prohibited by fixed restrictions on
number of columns in the crosstab table. Makes performance more scalable in
terms of available memory and less cpu/disk read-write bound. Cuts time with
40% on kenya nigtly export.
modified:
dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml
dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java
dhis-2/pom.xml
--
lp:dhis2
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk
Your team DHIS 2 developers is subscribed to branch lp:dhis2.
To unsubscribe from this branch go to
https://code.launchpad.net/~dhis2-devs-core/dhis2/trunk/+edit-subscription
=== modified file 'dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-04-14 09:05:18 +0000
+++ dhis-2/dhis-services/dhis-service-datamart-default/src/main/resources/META-INF/dhis/beans.xml 2011-04-16 16:18:58 +0000
@@ -70,6 +70,7 @@
class="org.hisp.dhis.datamart.DataMartInternalProcess"
scope="prototype">
<property name="statementManager" ref="statementManager"/>
+ <property name="inMemoryStatementManager" ref="inMemoryStatementManager"/>
<property name="dataMartService"
ref="org.hisp.dhis.datamart.DataMartService"/>
</bean>
@@ -78,12 +79,12 @@
<bean id="org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore"
class="org.hisp.dhis.datamart.crosstab.jdbc.JDBCCrossTabStore">
- <property name="statementManager" ref="statementManager"/>
+ <property name="statementManager" ref="inMemoryStatementManager"/>
</bean>
<bean id="org.hisp.dhis.datamart.crosstab.CrossTabService"
class="org.hisp.dhis.datamart.crosstab.DefaultCrossTabService">
- <property name="batchHandlerFactory" ref="batchHandlerFactory"/>
+ <property name="batchHandlerFactory" ref="inMemoryBatchHandlerFactory"/>
<property name="crossTabStore"
ref="org.hisp.dhis.datamart.crosstab.jdbc.CrossTabStore"/>
<property name="aggregatedDataValueService"
@@ -223,7 +224,7 @@
<aop:before pointcut="execution( * org.hisp.dhis.datamart.DataMartService.deleteDataMartExport(..) )" method="intercept"/>
</aop:aspect>
- <aop:aspect ref="statementInterceptor">
+ <aop:aspect ref="inMemoryStatementInterceptor">
<aop:around pointcut="execution( * org.hisp.dhis.datamart.DataMartService.export(..) )" method="intercept" />
</aop:aspect>
=== modified file 'dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml'
--- dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml 2011-04-15 14:38:27 +0000
+++ dhis-2/dhis-support/dhis-support-jdbc/src/main/resources/META-INF/dhis/beans.xml 2011-04-16 16:18:58 +0000
@@ -10,16 +10,37 @@
<property name="jdbcConfiguration" ref="jdbcConfiguration"/>
</bean>
+ <bean id="inMemoryStatementManager" class="org.amplecode.quick.statement.JdbcStatementManager">
+ <property name="inMemory" value="true"/>
+ </bean>
+
<!-- BatchHandlerFactory -->
<bean id="batchHandlerFactory" class="org.amplecode.quick.factory.DefaultBatchHandlerFactory">
<property name="jdbcConfiguration" ref="jdbcConfiguration"/>
</bean>
-
+
+ <bean id="inMemoryBatchHandlerFactory" class="org.amplecode.quick.factory.DefaultBatchHandlerFactory">
+ <property name="inMemory" value="true"/>
+ </bean>
+
<!-- Interceptor -->
<bean id="statementInterceptor" class="org.amplecode.quick.StatementInterceptor">
- <property name="statementManager" ref="statementManager"/>
+ <property name="statementManagers">
+ <list>
+ <ref bean="statementManager"/>
+ </list>
+ </property>
+ </bean>
+
+ <bean id="inMemoryStatementInterceptor" class="org.amplecode.quick.StatementInterceptor">
+ <property name="statementManagers">
+ <list>
+ <ref bean="statementManager"/>
+ <ref bean="inMemoryStatementManager"/>
+ </list>
+ </property>
</bean>
<!-- StatementDialect -->
=== modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java'
--- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java 2010-07-02 14:03:28 +0000
+++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/process/AbstractStatementInternalProcess.java 2011-04-16 16:18:58 +0000
@@ -64,6 +64,13 @@
this.statementManager = statementManager;
}
+ private StatementManager inMemoryStatementManager;
+
+ public void setInMemoryStatementManager( StatementManager inMemoryStatementManager )
+ {
+ this.inMemoryStatementManager = inMemoryStatementManager;
+ }
+
// -------------------------------------------------------------------------
// InternalProcess implementation
// -------------------------------------------------------------------------
@@ -82,6 +89,11 @@
statementManager.initialise();
+ if ( inMemoryStatementManager != null )
+ {
+ inMemoryStatementManager.initialise();
+ }
+
getState().setMessage( PROCESS_STARTED );
log.info( "Internal process started" );
@@ -106,6 +118,11 @@
finally
{
statementManager.destroy();
+
+ if ( inMemoryStatementManager != null )
+ {
+ inMemoryStatementManager.destroy();
+ }
}
}
=== modified file 'dhis-2/pom.xml'
--- dhis-2/pom.xml 2011-04-15 14:38:27 +0000
+++ dhis-2/pom.xml 2011-04-16 16:18:58 +0000
@@ -379,7 +379,7 @@
<dependency>
<groupId>org.amplecode</groupId>
<artifactId>quick</artifactId>
- <version>1.4</version>
+ <version>1.5</version>
</dependency>
<dependency>
<groupId>com.lowagie</groupId>
_______________________________________________
Mailing list: https://launchpad.net/~dhis2-devs
Post to : [email protected]
Unsubscribe : https://launchpad.net/~dhis2-devs
More help : https://help.launchpad.net/ListHelp
