This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 5f4566af4c8d20d862cd9671eabec26c9f6d00fc Author: baunsgaard <[email protected]> AuthorDate: Tue Nov 8 15:47:47 2022 +0100 [MINOR] mvn parallel build This commit modify the pom.xml and adds a configuration to mvn to enable parallel building, and also adds a jvm config for mvn to reduce logging when creating shaded jars. all in all on my laptop build time for 'mvn clean package -P distribution' is reduced from 70 sec to 55 sec (hot) and 50 sec (cold). and if we do not do a clean build time is all the way down to 44 sec, from 65+. While doing this i have moved antler to a profile, since we do not need to run it every package since we include the antler generated code in github. This commit further fixes the compile for maven to reuse previous compiled classes if no changes were made. Unfortunately it was broken because of an empty file ColGroupDeltaDDC.java, that kept reporting stale for the compiler since it did not produce a .class file, and therefore forced the system to compile from scratch every execution. Before to run a single test from command line there was a compile overhead up to 40 sec, now it is reduced to 4 sec. Change SkLearn name for package to install. scikit-learn is the name of the python package not sklearn --- .github/workflows/python.yml | 2 +- .mvn/jvm.config | 1 + .mvn/maven.config | 1 + pom.xml | 79 +++++++++------------- .../compress/colgroup/ColGroupDeltaDDC.java | 4 +- 5 files changed, 38 insertions(+), 49 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index a968094b7b..d7abd26b9c 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -100,7 +100,7 @@ jobs: pip install --upgrade pip pip install --upgrade pip pip install wheel - pip install numpy py4j scipy sklearn requests pandas unittest-parallel + pip install numpy py4j scipy scikit-learn requests pandas unittest-parallel - name: Build Python Package run: | diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 0000000000..cc81b2be4a --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Dorg.slf4j.simpleLogger.showLogName=false -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade.mojo.ShadeMojo=off diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 0000000000..ac0bd15dc2 --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1 @@ +-T 1.5C diff --git a/pom.xml b/pom.xml index 560db63520..b44160c700 100644 --- a/pom.xml +++ b/pom.xml @@ -145,6 +145,10 @@ </includes> <targetPath>java/spoof</targetPath> </resource> + <resource> + <directory>src/test/config/hadoop_bin_windows/bin</directory> + <targetPath>../lib/hadoop/bin</targetPath> + </resource> </resources> <plugins> @@ -167,7 +171,6 @@ <type>jar</type> <overWrite>true</overWrite> <outputDirectory>${project.build.directory}/hadoop-test</outputDirectory> - <includes>**/*</includes> </artifactItem> </artifactItems> <overWriteReleases>false</overWriteReleases> @@ -254,50 +257,6 @@ </configuration> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-resources</id> - <phase>compile</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <resources> - <resource> - <directory>${basedir}/src/test/config/hadoop_bin_windows/bin</directory> - <filtering>false</filtering> - <includes> - <include>*.*</include> - </includes> - </resource> - </resources> - <outputDirectory>${basedir}/target/lib/hadoop/bin</outputDirectory> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.antlr</groupId> - <artifactId>antlr4-maven-plugin</artifactId> - <configuration> - <sourceDirectory>${basedir}/src/main/java</sourceDirectory> - <outputDirectory>${basedir}/src/main/java</outputDirectory> - </configuration> - <version>${antlr.version}</version> - <executions> - <execution> - <id>antlr</id> - <goals> - <goal>antlr4</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <!-- unit tests --> <groupId>org.apache.maven.plugins</groupId> @@ -310,8 +269,9 @@ <!-- 1C means the number of threads times 1 possible maximum forks for testing--> <forkCount>${test-forkCount}</forkCount> <reuseForks>false</reuseForks> - <reportFormat>brief</reportFormat> + <!-- <reportFormat>brief</reportFormat> --> <trimStackTrace>true</trimStackTrace> + <showSuccess>false</showSuccess> <rerunFailingTestsCount>${rerun.failing.tests.count}</rerunFailingTestsCount> </configuration> </plugin> @@ -589,6 +549,7 @@ <exclude>.gitmodules</exclude> <exclude>.repository/</exclude> <exclude>.idea/</exclude> + <exclude>.mvn/**</exclude> <exclude>.git</exclude> <exclude>.settings</exclude> <exclude>.classpath</exclude> @@ -693,6 +654,32 @@ </build> </profile> + <profile> + <id>antlr</id> + <build> + <plugins> + <plugin> + <groupId>org.antlr</groupId> + <artifactId>antlr4-maven-plugin</artifactId> + <configuration> + <sourceDirectory>${basedir}/src/main/java</sourceDirectory> + <outputDirectory>${basedir}/src/main/java</outputDirectory> + </configuration> + <version>${antlr.version}</version> + <executions> + <execution> + <id>antlr</id> + <goals> + <goal>antlr4</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + + </profile> + <profile> <!-- Profile to create binary distributions. Execute with `mvn clean package -P distribution` --> diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java index 2ec72f7cca..2666860ca6 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java @@ -23,7 +23,7 @@ package org.apache.sysds.runtime.compress.colgroup; * Class to encapsulate information about a column group that is first delta encoded then encoded with dense dictionary * encoding (DeltaDDC). */ -// public class ColGroupDeltaDDC extends ColGroupDDC { +public class ColGroupDeltaDDC { // extends ColGroupDDC // private static final long serialVersionUID = -1045556313148564147L; @@ -77,4 +77,4 @@ package org.apache.sysds.runtime.compress.colgroup; // public AColGroup scalarOperation(ScalarOperator op) { // return new ColGroupDeltaDDC(_colIndexes, _dict.applyScalarOp(op), _data, getCachedCounts()); // } -// } +}
