Repository: cassandra Updated Branches: refs/heads/cassandra-2.2 70d0d5f2c -> 55a736acf refs/heads/trunk 8ad6dc9af -> e14ce11c0
Update and refactor ant test/test-compression to run the tests in parallel patch by Ariel Weisberg; reviewed by yukim for CASSANDRA-9583 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/55a736ac Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/55a736ac Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/55a736ac Branch: refs/heads/cassandra-2.2 Commit: 55a736acff1bfae779c6f01b2280ef50befca1a5 Parents: 70d0d5f Author: Ariel Weisberg <[email protected]> Authored: Thu Jun 18 14:10:37 2015 -0500 Committer: Yuki Morishita <[email protected]> Committed: Thu Jun 18 14:10:37 2015 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + build.xml | 237 ++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 148 insertions(+), 90 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/55a736ac/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b96a2b0..56f0dc0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ * Make sure we cancel non-compacting sstables from LifecycleTransaction (CASSANDRA-9566) * Fix deprecated repair JMX API (CASSANDRA-9570) * Add logback metrics (CASSANDRA-9378) + * Update and refactor ant test/test-compression to run the tests in parallel (CASSANDRA-9583) Merged from 2.1: * Fix bug in cardinality check when compacting (CASSANDRA-9580) * Fix memory leak in Ref due to ConcurrentLinkedQueue.remove() behaviour (CASSANDRA-9549) http://git-wip-us.apache.org/repos/asf/cassandra/blob/55a736ac/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index e768a394..82ec33e 100644 --- a/build.xml +++ b/build.xml @@ -1114,8 +1114,12 @@ </copy> </target> - <macrodef name="testmacro"> - <attribute name="suitename" /> + <!-- Defines how to run a set of tests. If you change the defaults for attributes + you should also update them in testmacro., + The two are split because the helper doesn't generate + a junit report or fail on errors, since this is called in parallel to run tests + when we choose to run tests in parallel --> + <macrodef name="testmacrohelper"> <attribute name="inputdir" /> <attribute name="timeout" default="${test.timeout}" /> <attribute name="forkmode" default="perTest"/> @@ -1133,7 +1137,6 @@ else=""> <istrue value="${usejacoco}"/> </condition> - <echo message="running @{suitename} tests"/> <mkdir dir="${build.test.dir}/cassandra"/> <mkdir dir="${build.test.dir}/output"/> <mkdir dir="${build.test.dir}/output/@{testtag}"/> @@ -1168,22 +1171,9 @@ <filelist dir="@{inputdir}" files="@{filelist}"/> </batchtest> </junit> - <junitreport todir="${build.test.dir}"> - <fileset dir="${build.test.dir}/output"> - <include name="**/TEST-*.xml"/> - </fileset> - <report format="frames" todir="${build.test.dir}/junitreport"/> - </junitreport> - <fail message="Some @{suitename} test(s) failed."> - <condition> - <and> - <isset property="testfailed"/> - <not> - <isset property="ant.test.failure.ignore"/> - </not> - </and> - </condition> - </fail> + <delete quiet="true" failonerror="false" dir="${build.test.dir}/cassandra/commitlog:@{poffset}"/> + <delete quiet="true" failonerror="false" dir="${build.test.dir}/cassandra/data:@{poffset}"/> + <delete quiet="true" failonerror="false" dir="${build.test.dir}/cassandra/saved_caches:@{poffset}"/> </sequential> </macrodef> @@ -1231,18 +1221,47 @@ </fileset> </target> - <target name="testlist"> - <testmacro suitename="${testlist.name}" inputdir="${test.unit.src}" filelist="${test.file.list}" poffset="${testlist.offset}" exclude="**/*.java" timeout="${test.timeout}"> - <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> - <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/> - <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> - <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/> - <jvmarg value="-Dcassandra.ring_delay_ms=1000"/> - <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> - <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/> - <jvmarg value="-Dcassandra.skip_sync=true" /> - </testmacro> - </target> + <!-- Will not generate a junit report or fail on error since it is called in parallel for test-compression + That is taken care of by testparallel --> + <macrodef name="testlist"> + <attribute name="test.file.list"/> + <attribute name="testlist.offset"/> + <sequential> + <testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}" poffset="@{testlist.offset}" exclude="**/*.java" timeout="${test.timeout}"> + <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> + <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/> + <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> + <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/> + <jvmarg value="-Dcassandra.ring_delay_ms=1000"/> + <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> + <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/> + <jvmarg value="-Dcassandra.skip_sync=true" /> + </testmacrohelper> + </sequential> + </macrodef> + + <!-- Will not generate a junit report or fail on error since it is called in parallel for test-compression + That is taken care of by testparallel --> + <macrodef name="testlist-compression"> + <attribute name="test.file.list" /> + <attribute name="testlist.offset" /> + <sequential> + <property name="compressed_yaml" value="${build.test.dir}/cassandra.compressed.yaml"/> + <testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}" poffset="@{testlist.offset}" + exclude="**/*.java" timeout="${test.timeout}" testtag="compression"> + <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> + <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/> + <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> + <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/> + <jvmarg value="-Dcassandra.test.compression=true"/> + <jvmarg value="-Dcassandra.ring_delay_ms=1000"/> + <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> + <jvmarg value="-Dcassandra.config=file:///${compressed_yaml}"/> + <jvmarg value="-Dcassandra.skip_sync=true" /> + <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/> + </testmacrohelper> + </sequential> + </macrodef> <!-- Run named ant task with jacoco, such as "ant jacoco-run -Dtaskname=pig-test" @@ -1278,21 +1297,11 @@ <fileset file="${test.conf}/cassandra.yaml"/> <fileset file="${test.conf}/commitlog_compression.yaml"/> </concat> - <echo>Compressed config: ${compressed_yaml}</echo> - <testmacro suitename="unit" inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}" testtag="compression"> - <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/> - <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/> - <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/> - <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/> - <jvmarg value="-Dcassandra.test.compression=true"/> - <jvmarg value="-Dcassandra.ring_delay_ms=1000"/> - <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> - <jvmarg value="-Dcassandra.config=file:///${compressed_yaml}"/> - <jvmarg value="-Dcassandra.skip_sync=true" /> - </testmacro> - <fileset dir="${test.unit.src}"> - <exclude name="**/pig/*.java" /> - </fileset> + <path id="all-test-classes-path"> + <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" /> + </path> + <property name="all-test-classes" refid="all-test-classes-path"/> + <testparallel testdelegate="testlist-compression" /> </target> <target name="msg-ser-gen-test" depends="build-test" description="Generates message serializations"> @@ -1538,60 +1547,108 @@ </create-javadoc> </target> - <!-- Split test classes into n buckets and run across processes --> - <target name="test" depends="build-test" description="Parallel Test Runner"> - <path id="all-test-classes-path"> - <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" /> - </path> - <property name="all-test-classes" refid="all-test-classes-path"/> - <script language="javascript"> <![CDATA[ - var Integer = java.lang.Integer; - sep = project.getProperty("path.separator"); - all = project.getProperty("all-test-classes").split(sep); - dir = project.getProperty("test.unit.src"); + <!-- Run tests not in parallel and reports errors and generates a junit report after --> + <macrodef name="testmacro"> + <attribute name="suitename" /> + <attribute name="inputdir" /> + <attribute name="timeout" default="${test.timeout}" /> + <attribute name="forkmode" default="perTest"/> + <element name="optjvmargs" implicit="true" optional="true" /> + <attribute name="filter" default="**/${test.name}.java"/> + <attribute name="exclude" default="" /> + <attribute name="filelist" default="" /> + <attribute name="poffset" default="0"/> + <attribute name="testtag" default=""/> - numRunners = parseInt(project.getProperty("test.runners")); - - buckets = new Array(numRunners); - for (i = 0; i < all.length; i++) { - bucketNum = i % numRunners; - if (buckets[bucketNum] == undefined) - buckets[bucketNum] = ""; - else - buckets[bucketNum] += ","; - - buckets[bucketNum] += all[i]; - } + <sequential> + <testmacrohelper suitename="@{suitename}" inputdir="@{inputdir}" timeout="@{timeout}" + forkmode="@{forkmode}" filter="@{filter}" + exclude="@{exclude}" filelist="@{filelist}" poffset="@{poffset}" + testtag="@{testtag}" > + <optjvmargs/> + </testmacrohelper> + <junitreport todir="${build.test.dir}"> + <fileset dir="${build.test.dir}/output"> + <include name="**/TEST-*.xml"/> + </fileset> + <report format="frames" todir="${build.test.dir}/junitreport"/> + </junitreport> + <fail message="Some test(s) failed."> + <condition> + <and> + <isset property="testfailed"/> + <not> + <isset property="ant.test.failure.ignore"/> + </not> + </and> + </condition> + </fail> + </sequential> + </macrodef> + <!-- Run tests in parallel and report errors after and generate a junit report --> + <macrodef name="testparallel"> + <attribute name="testdelegate"/> + <sequential> + <testparallelhelper testdelegate="@{testdelegate}"/> + <junitreport todir="${build.test.dir}"> + <fileset dir="${build.test.dir}/output"> + <include name="**/TEST-*.xml"/> + </fileset> + <report format="frames" todir="${build.test.dir}/junitreport"/> + </junitreport> + <fail message="Some test(s) failed."> + <condition> + <and> + <isset property="testfailed"/> + <not> + <isset property="ant.test.failure.ignore"/> + </not> + </and> + </condition> + </fail> + </sequential> + </macrodef> - var p = project.createTask('parallel'); - p.setThreadCount(numRunners); + <!-- Run multiple junit tasks in parallel, but don't track errors or generate a report after + If a test fails the testfailed property will be set. All the tests are run using te testdelegate + macro that is specified as an attribute and they will be run concurrently in this ant process --> + <scriptdef name="testparallelhelper" language="javascript"> + <attribute name="testdelegate"/> + <![CDATA[ + var Integer = java.lang.Integer; + sep = project.getProperty("path.separator"); + all = project.getProperty("all-test-classes").split(sep); + dir = project.getProperty("test.unit.src"); - for (i = 0; i < buckets.length; i++) { + numRunners = parseInt(project.getProperty("test.runners")); - if (buckets[i] == undefined) continue; + var p = project.createTask('parallel'); + p.setThreadCount(numRunners); - task = project.createTask( 'antcall' ); + for (i = 0; i < all.length; i++) { - task.setTarget("testlist"); - param = task.createParam(); - param.setName("test.file.list"); - param.setValue("" + buckets[i]); + if (all[i] == undefined) continue; - param = task.createParam(); - param.setName("testlist.name"); - param.setValue("test bucket "+i); + task = project.createTask( attributes.get("testdelegate") ); - param = task.createParam(); - param.setName("testlist.offset"); - param.setValue("" + i); + task.setDynamicAttribute( "test.file.list", "" + all[i]); - p.addTask(task); - } - - p.perform(); - -]]> </script> + task.setDynamicAttribute( "testlist.offset", "" + i ); + + p.addTask(task); + } + + p.perform(); + ]]> + </scriptdef> + + <target name="test" depends="build-test" description="Parallel Test Runner"> + <path id="all-test-classes-path"> + <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" /> + </path> + <property name="all-test-classes" refid="all-test-classes-path"/> + <testparallel testdelegate="testlist"/> </target> <!-- run microbenchmarks suite -->
