This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch profile-db
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
The following commit(s) were added to refs/heads/profile-db by this push:
new 23bb4ab [Java11] Full Support compilation java 11 and switch scala
version
23bb4ab is described below
commit 23bb4ab147ba3465bb2180a7202898cb55989eed
Author: berttty <[email protected]>
AuthorDate: Tue Aug 31 19:33:14 2021 +0200
[Java11] Full Support compilation java 11 and switch scala version
Signed-off-by: bertty <[email protected]>
---
pom.xml | 185 ++++++++++++++++++---
src/main/script/prerequisiteCheck.groovy | 2 +-
wayang-api/wayang-api-scala-java/pom.xml | 4 +-
.../wayang/core/plan/wayangplan/SlotMapping.java | 28 ++--
.../wayang-profile-db/{readme.md => README.md} | 0
.../src/test/java/profiledb/ProfileDBTest.java | 163 +++++++++---------
wayang-distro/pom.xml | 24 ++-
wayang-platforms/pom.xml | 9 +-
wayang-platforms/wayang-graphchi/pom.xml | 2 +-
wayang-platforms/wayang-spark/pom.xml | 8 +-
wayang-plugins/wayang-iejoin/pom.xml | 4 +-
wayang-profiler/pom.xml | 35 +++-
.../wayang/profiler/log/GeneticOptimizerApp.java | 3 +-
wayang-tests-integration/pom.xml | 84 ++++++----
.../apache/wayang/tests/FlinkIntegrationIT.java | 5 +-
.../apache/wayang/tests/GiraphIntegrationIT.java | 5 +-
.../java/org/apache/wayang/tests/WayangPlans.java | 4 +-
.../apache/wayang/test}/GraphChiIntegrationIT.java | 3 +-
18 files changed, 375 insertions(+), 193 deletions(-)
diff --git a/pom.xml b/pom.xml
index 8b6e143..5b7ce02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,12 +91,9 @@
</mailingLists>
<properties>
- <maven.compiler.source>1.8</maven.compiler.source>
- <maven.compiler.target>1.8</maven.compiler.target>
<encoding>UTF-8</encoding>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.root>${basedir}</project.root>
- <source.level>1.8</source.level>
<dist.id>${project.artifactId}</dist.id>
<jacoco.destfile>${project.build.directory}/jacoco.exec</jacoco.destfile>
<maven.kotlin.skip>false</maven.kotlin.skip>
@@ -306,13 +303,92 @@
</build>
</profile>
+ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <properties>
+ <java.version>1.8</java.version>
+ <source.level>1.8</source.level>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <maven.compiler.release>1.8</maven.compiler.release>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-maven-version</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <version>1.8</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- Used to build Java 11+ -->
<profile>
<id>java11</id>
+ <activation>
+ <jdk>11</jdk>
+ </activation>
<properties>
+ <java.version>11</java.version>
+ <source.level>11</source.level>
+ <maven.compiler.source>11</maven.compiler.source>
+ <maven.compiler.target>11</maven.compiler.target>
+ <maven.compiler.release>11</maven.compiler.release>
<jaxb.version>2.2.11</jaxb.version>
<java-activation.version>1.1.1</java-activation.version>
+ <javax-annotation-api>1.3.2</javax-annotation-api>
</properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-maven-version</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <version>11</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <source>8</source>
+ <detectJavaApiLink>false</detectJavaApiLink>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+
<dependencies>
<dependency>
<groupId>javax.xml.bind</groupId>
@@ -333,14 +409,6 @@
<scope>provided</scope>
</dependency>
</dependencies>
- <!-- The compilation of java 11 is only possible in a select
number of modules,
- it's necessary to fix this a soon as possible
- -->
- <modules>
- <module>wayang-api</module>
- <module>wayang-commons/wayang-core</module>
- <module>wayang-commons/wayang-basic</module>
- </modules>
</profile>
<!--
@@ -427,19 +495,61 @@
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+
<finalName>${project.artifactId}_${scala.mayor.version}-${project.version}</finalName>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
- <id>default-jar</id>
+ <id>copy-pom</id>
+ <phase>generate-resources</phase>
<goals>
- <goal>jar</goal>
+ <goal>copy-resources</goal>
</goals>
- <phase>package</phase>
<configuration>
-
<finalName>${project.artifactId}-${project.version}_${scala.mayor.version}</finalName>
- <skipIfEmpty>true</skipIfEmpty>
- <excludes>
- <exclude>*.timestamp</exclude>
- </excludes>
+
<outputDirectory>${project.build.directory}/${scala.mayor.version}</outputDirectory>
+ <resources>
+ <resource>
+
<directory>${basedir}</directory>
+ <includes>
+ <include>pom.xml</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-install-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-install</id>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ <execution>
+ <id>install-scala-version</id>
+ <phase>install</phase>
+ <goals>
+ <goal>install-file</goal>
+ </goals>
+ <configuration>
+ <groupId>${project.groupId}</groupId>
+
<artifactId>${project.artifactId}_${scala.mayor.version}</artifactId>
+ <version>${project.version}</version>
+
<packaging>${project.packaging}</packaging>
+
<file>${project.build.directory}/${project.artifactId}_${scala.mayor.version}-${project.version}.jar</file>
+
<pomFile>${project.build.directory}/${scala.mayor.version}/replacer/pom.xml</pomFile>
</configuration>
</execution>
</executions>
@@ -454,6 +564,34 @@
<!-- <plugin>-->
<!--
<artifactId>maven-jar-plugin</artifactId>-->
<!-- </plugin>-->
+ <plugin>
+
<groupId>com.google.code.maven-replacer-plugin</groupId>
+ <artifactId>replacer</artifactId>
+ <version>1.5.3</version>
+ <executions>
+ <execution>
+ <id>replace-artifactid</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>replace</goal>
+ </goals>
+ <configuration>
+ <includes>
+
<include>${basedir}/target/${scala.mayor.version}/pom.xml</include>
+ </includes>
+ <replacements>
+ <replacement>
+
<token><artifactId>${project.artifactId}</artifactId></token>
+
<value><artifactId>${project.artifactId}_${scala.mayor.version}</artifactId></value>
+ </replacement>
+ </replacements>
+
+ <!-- Replace to -->
+
<outputFile>${basedir}/target/${scala.mayor.version}/replacer/pom.xml</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
@@ -953,9 +1091,9 @@
<exclude>**/.jekyll-cache/**</exclude>
<exclude>**/README.md</exclude>
<!-- TODO: I am pretty sure we need to remove this and
replace it with a different text -->
- <exclude>src/test/resources/ulysses.txt</exclude>
- <exclude>src/test/resources/test.edgelist</exclude>
- <exclude>src/test/resources/*.txt</exclude>
+ <exclude>**/src/test/resources/ulysses.txt</exclude>
+ <exclude>**/src/test/resources/test.edgelist</exclude>
+ <exclude>**/src/test/resources/*.txt</exclude>
</excludes>
</configuration>
</plugin>
@@ -1098,9 +1236,6 @@
<requireMavenVersion>
<version>3.5.0</version>
</requireMavenVersion>
- <requireJavaVersion>
- <version>1.8</version>
- </requireJavaVersion>
</rules>
</configuration>
</execution>
diff --git a/src/main/script/prerequisiteCheck.groovy
b/src/main/script/prerequisiteCheck.groovy
index 2fb6950..69a07d5 100644
--- a/src/main/script/prerequisiteCheck.groovy
+++ b/src/main/script/prerequisiteCheck.groovy
@@ -190,7 +190,7 @@ if (os == "windows") {
// profiles.
/////////////////////////////////////////////////////
-checkJavaVersion("1.8", "1.8")
+checkJavaVersion("1.8", "11")
// Check if hadoop is available
// It seems that this is only required on Windows systems.
diff --git a/wayang-api/wayang-api-scala-java/pom.xml
b/wayang-api/wayang-api-scala-java/pom.xml
index 1470c30..070d5cb 100644
--- a/wayang-api/wayang-api-scala-java/pom.xml
+++ b/wayang-api/wayang-api-scala-java/pom.xml
@@ -26,7 +26,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-api-scala-java_2.11</artifactId>
+ <artifactId>wayang-api-scala-java</artifactId>
<version>0.6.0-SNAPSHOT</version>
<name>Wayang API Scala-Java</name>
@@ -72,7 +72,7 @@
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-spark_${scala.mayor.version}</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
diff --git
a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/plan/wayangplan/SlotMapping.java
b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/plan/wayangplan/SlotMapping.java
index cdd2775..48a1e8a 100644
---
a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/plan/wayangplan/SlotMapping.java
+++
b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/plan/wayangplan/SlotMapping.java
@@ -18,6 +18,7 @@
package org.apache.wayang.core.plan.wayangplan;
+import java.util.Map.Entry;
import org.apache.wayang.core.util.WayangCollections;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -38,9 +39,9 @@ public class SlotMapping {
private final Logger logger = LogManager.getLogger(this.getClass());
- private final Map<Slot, Slot> upstreamMapping = new HashMap<>();
+ private final Map<Slot<?>, Slot<?>> upstreamMapping = new HashMap<>();
- private Map<Slot, Collection> downstreamMapping = null;
+ private Map<Slot<?>, Collection> downstreamMapping = null;
/**
* Create a new instance that maps all {@link Slot}s of the given {@link
Operator} to themselves.
@@ -68,7 +69,7 @@ public class SlotMapping {
return slotMapping;
}
- public void mapAllUpsteam(InputSlot[] sources, InputSlot[] targets) {
+ public void mapAllUpsteam(InputSlot<?>[] sources, InputSlot<?>[] targets) {
if (sources.length != targets.length) {
throw new IllegalArgumentException(String.format("Incompatible
number of input slots between %s and %s.",
Arrays.toString(sources), Arrays.toString(targets)));
@@ -78,7 +79,7 @@ public class SlotMapping {
}
}
- public void mapAllUpsteam(OutputSlot[] sources, OutputSlot[] targets) {
+ public void mapAllUpsteam(OutputSlot<?>[] sources, OutputSlot<?>[]
targets) {
if (sources.length != targets.length) throw new
IllegalArgumentException();
for (int i = 0; i < sources.length; i++) {
this.mapUpstream(sources[i], targets[i]);
@@ -142,14 +143,15 @@ public class SlotMapping {
*
* @return {@link #downstreamMapping}
*/
- private Map<Slot, Collection> getOrCreateDownstreamMapping() {
+ private Map<Slot<?>, Collection> getOrCreateDownstreamMapping() {
if (this.downstreamMapping == null) {
- this.downstreamMapping =
this.upstreamMapping.entrySet().stream().collect(
- Collectors.groupingBy(
- Map.Entry::getValue,
- Collectors.mapping(
- Map.Entry::getKey,
-
Collectors.toCollection(LinkedList::new))));
+ Map<Slot<?>, Collection> map = new HashMap<>();
+ for (Entry<Slot<?>, Slot<?>> slotSlotEntry :
this.upstreamMapping.entrySet()) {
+ Object key = slotSlotEntry.getKey();
+ map.computeIfAbsent(slotSlotEntry.getValue(), k -> new
LinkedList<Object>())
+ .add(key);
+ }
+ this.downstreamMapping = map;
}
return this.downstreamMapping;
@@ -227,7 +229,7 @@ public class SlotMapping {
public void replaceOutputSlotMappings(Operator oldOperator, Operator
newOperator) {
if (oldOperator.getParent() == newOperator) {
// Default strategy: The oldOperator is now wrapped by the
newOperator.
- final Map<Slot, Collection> downstreamMapping =
this.getOrCreateDownstreamMapping();
+ final Map<Slot<?>, Collection> downstreamMapping =
this.getOrCreateDownstreamMapping();
final SlotMapping oldToNewSlotMapping =
oldOperator.getContainer().getSlotMapping();
for (int i = 0; i < oldOperator.getNumOutputs(); i++) {
final OutputSlot<?> oldOutput = oldOperator.getOutput(i);
@@ -296,7 +298,7 @@ public class SlotMapping {
*
* @return the upstream mapping
*/
- public Map<Slot, Slot> getUpstreamMapping() {
+ public Map<Slot<?>, Slot<?>> getUpstreamMapping() {
return this.upstreamMapping;
}
diff --git a/wayang-commons/wayang-utils/wayang-profile-db/readme.md
b/wayang-commons/wayang-utils/wayang-profile-db/README.md
similarity index 100%
rename from wayang-commons/wayang-utils/wayang-profile-db/readme.md
rename to wayang-commons/wayang-utils/wayang-profile-db/README.md
diff --git
a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
index b232773..f33ca72 100644
---
a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
+++
b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
@@ -43,100 +43,89 @@ public class ProfileDBTest {
@Test
public void testPolymorphSaveAndLoad() throws IOException {
+ File tempDir = Files.createTempDirectory("profiledb").toFile();
+ File file = new File(tempDir, "new-profiledb.json");
+ file.createNewFile();
- try {
- Path temp = Files.createTempFile("", ".tmp");
-
- String absolutePath = temp.toString();
- System.out.println("Temp file : " + absolutePath);
-
- URI uri = new URI("my-file4");
-
- FileStorage store = new FileStorage(uri);
-
- ProfileDB profileDB = new ProfileDB(store)
- .registerMeasurementClass(TestMemoryMeasurement.class)
- .registerMeasurementClass(TestTimeMeasurement.class);
-
- final Experiment experiment = new Experiment("test-xp", new
Subject("PageRank", "1.0"), "test experiment");
-
- Measurement timeMeasurement = new TestTimeMeasurement("exec-time",
12345L);
- Measurement memoryMeasurement = new
TestMemoryMeasurement("exec-time", System.currentTimeMillis(), 54321L);
-
- experiment.addMeasurement(timeMeasurement);
- experiment.addMeasurement(memoryMeasurement);
-
- // Save the experiment.
- byte[] buffer;
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- profileDB.save(Collections.singleton(experiment), bos);
- bos.close();
- buffer = bos.toByteArray();
- System.out.println("Buffer contents: " + new String(buffer,
"UTF-8"));
-
- // Load the experiment.
- ByteArrayInputStream bis = new ByteArrayInputStream(buffer);
- Collection<Experiment> loadedExperiments = profileDB.load(bis);
-
- // Compare the experiments.
- Assert.assertEquals(1, loadedExperiments.size());
- Experiment loadedExperiment = loadedExperiments.iterator().next();
- Assert.assertEquals(experiment, loadedExperiment);
+ FileStorage store = new FileStorage(file.toURI());
- // Compare the measurements.
- Assert.assertEquals(2, loadedExperiment.getMeasurements().size());
- Set<Measurement> expectedMeasurements = new HashSet<>(2);
- expectedMeasurements.add(timeMeasurement);
- expectedMeasurements.add(memoryMeasurement);
- Set<Measurement> loadedMeasurements = new
HashSet<>(loadedExperiment.getMeasurements());
- Assert.assertEquals(expectedMeasurements, loadedMeasurements);
+ ProfileDB profileDB = new ProfileDB(store)
+ .registerMeasurementClass(TestMemoryMeasurement.class)
+ .registerMeasurementClass(TestTimeMeasurement.class);
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
+ final Experiment experiment = new Experiment("test-xp", new
Subject("PageRank", "1.0"), "test experiment");
+
+ Measurement timeMeasurement = new TestTimeMeasurement("exec-time",
12345L);
+ Measurement memoryMeasurement = new TestMemoryMeasurement("exec-time",
System.currentTimeMillis(), 54321L);
+
+ experiment.addMeasurement(timeMeasurement);
+ experiment.addMeasurement(memoryMeasurement);
+
+ // Save the experiment.
+ byte[] buffer;
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ profileDB.save(Collections.singleton(experiment), bos);
+ bos.close();
+ buffer = bos.toByteArray();
+ System.out.println("Buffer contents: " + new String(buffer, "UTF-8"));
+
+ // Load the experiment.
+ ByteArrayInputStream bis = new ByteArrayInputStream(buffer);
+ Collection<Experiment> loadedExperiments = profileDB.load(bis);
+
+ // Compare the experiments.
+ Assert.assertEquals(1, loadedExperiments.size());
+ Experiment loadedExperiment = loadedExperiments.iterator().next();
+ Assert.assertEquals(experiment, loadedExperiment);
+
+ // Compare the measurements.
+ Assert.assertEquals(2, loadedExperiment.getMeasurements().size());
+ Set<Measurement> expectedMeasurements = new HashSet<>(2);
+ expectedMeasurements.add(timeMeasurement);
+ expectedMeasurements.add(memoryMeasurement);
+ Set<Measurement> loadedMeasurements = new
HashSet<>(loadedExperiment.getMeasurements());
+ Assert.assertEquals(expectedMeasurements, loadedMeasurements);
}
@Test
public void testRecursiveSaveAndLoad() throws IOException {
- try {
- URI uri = new URI("my-file2");
- FileStorage store = new FileStorage(uri);
-
- ProfileDB profileDB = new ProfileDB(store)
- .registerMeasurementClass(TestMemoryMeasurement.class)
- .registerMeasurementClass(TestTimeMeasurement.class);
-
- // Create an example experiment.
- final Experiment experiment = new Experiment("test-xp", new
Subject("PageRank", "1.0"), "test experiment");
- TestTimeMeasurement topLevelMeasurement = new
TestTimeMeasurement("exec-time", 12345L);
- TestTimeMeasurement childMeasurement = new
TestTimeMeasurement("sub-exec-time", 2345L);
- topLevelMeasurement.addSubmeasurements(childMeasurement);
- experiment.addMeasurement(topLevelMeasurement);
-
- // Save the experiment.
- byte[] buffer;
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- profileDB.save(Collections.singleton(experiment), bos);
- bos.close();
- buffer = bos.toByteArray();
- System.out.println("Buffer contents: " + new String(buffer,
"UTF-8"));
-
- // Load the experiment.
- ByteArrayInputStream bis = new ByteArrayInputStream(buffer);
- Collection<Experiment> loadedExperiments = profileDB.load(bis);
-
- // Compare the experiments.
- Assert.assertEquals(1, loadedExperiments.size());
- Experiment loadedExperiment = loadedExperiments.iterator().next();
- Assert.assertEquals(experiment, loadedExperiment);
-
- // Compare the measurements.
- Assert.assertEquals(1, loadedExperiment.getMeasurements().size());
- final Measurement loadedMeasurement =
loadedExperiment.getMeasurements().iterator().next();
- Assert.assertEquals(topLevelMeasurement, loadedMeasurement);
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
+ File tempDir = Files.createTempDirectory("profiledb").toFile();
+ File file = new File(tempDir, "new-profiledb.json");
+ file.createNewFile();
+ FileStorage store = new FileStorage(file.toURI());
+
+ ProfileDB profileDB = new ProfileDB(store)
+ .registerMeasurementClass(TestMemoryMeasurement.class)
+ .registerMeasurementClass(TestTimeMeasurement.class);
+
+ // Create an example experiment.
+ final Experiment experiment = new Experiment("test-xp", new
Subject("PageRank", "1.0"), "test experiment");
+ TestTimeMeasurement topLevelMeasurement = new
TestTimeMeasurement("exec-time", 12345L);
+ TestTimeMeasurement childMeasurement = new
TestTimeMeasurement("sub-exec-time", 2345L);
+ topLevelMeasurement.addSubmeasurements(childMeasurement);
+ experiment.addMeasurement(topLevelMeasurement);
+
+ // Save the experiment.
+ byte[] buffer;
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ profileDB.save(Collections.singleton(experiment), bos);
+ bos.close();
+ buffer = bos.toByteArray();
+ System.out.println("Buffer contents: " + new String(buffer, "UTF-8"));
+
+ // Load the experiment.
+ ByteArrayInputStream bis = new ByteArrayInputStream(buffer);
+ Collection<Experiment> loadedExperiments = profileDB.load(bis);
+
+ // Compare the experiments.
+ Assert.assertEquals(1, loadedExperiments.size());
+ Experiment loadedExperiment = loadedExperiments.iterator().next();
+ Assert.assertEquals(experiment, loadedExperiment);
+
+ // Compare the measurements.
+ Assert.assertEquals(1, loadedExperiment.getMeasurements().size());
+ final Measurement loadedMeasurement =
loadedExperiment.getMeasurements().iterator().next();
+ Assert.assertEquals(topLevelMeasurement, loadedMeasurement);
}
@Test
diff --git a/wayang-distro/pom.xml b/wayang-distro/pom.xml
index e1bd196..2a6588c 100644
--- a/wayang-distro/pom.xml
+++ b/wayang-distro/pom.xml
@@ -28,7 +28,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-distro_2.11</artifactId>
+ <artifactId>wayang-distro</artifactId>
<version>0.6.0-SNAPSHOT</version>
<build>
@@ -71,17 +71,12 @@
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-spark_2.11</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-graphchi_2.11</artifactId>
- <version>0.6.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.wayang</groupId>
- <artifactId>wayang-profiler_2.11</artifactId>
+ <artifactId>wayang-profiler</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -110,6 +105,17 @@
<version>0.6.0-SNAPSHOT</version>
</dependency>
</dependencies>
-
+ <profiles>
+ <profile>
+ <id>scala-11</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.wayang</groupId>
+ <artifactId>wayang-graphchi</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
diff --git a/wayang-platforms/pom.xml b/wayang-platforms/pom.xml
index c6177b1..ceb9d4e 100644
--- a/wayang-platforms/pom.xml
+++ b/wayang-platforms/pom.xml
@@ -39,7 +39,6 @@
<module>wayang-java</module>
<module>wayang-spark</module>
<module>wayang-postgres</module>
- <module>wayang-graphchi</module>
<module>wayang-jdbc-template</module>
<module>wayang-sqlite3</module>
<module>wayang-giraph</module>
@@ -58,5 +57,13 @@
</dependencies>
</dependencyManagement>
+ <profiles>
+ <profile>
+ <id>scala-11</id>
+ <modules>
+ <module>wayang-graphchi</module>
+ </modules>
+ </profile>
+ </profiles>
</project>
diff --git a/wayang-platforms/wayang-graphchi/pom.xml
b/wayang-platforms/wayang-graphchi/pom.xml
index 42aeb51..0554c9a 100644
--- a/wayang-platforms/wayang-graphchi/pom.xml
+++ b/wayang-platforms/wayang-graphchi/pom.xml
@@ -26,7 +26,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-graphchi_2.11</artifactId>
+ <artifactId>wayang-graphchi</artifactId>
<version>0.6.0-SNAPSHOT</version>
<name>Wayang Graphchi</name>
diff --git a/wayang-platforms/wayang-spark/pom.xml
b/wayang-platforms/wayang-spark/pom.xml
index a0c3972..8a93813 100644
--- a/wayang-platforms/wayang-spark/pom.xml
+++ b/wayang-platforms/wayang-spark/pom.xml
@@ -26,7 +26,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-spark_2.11</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
<name>Wayang Platform Spark</name>
@@ -79,6 +79,12 @@
<version>1.1.2.1</version>
<scope>${external.platforms.scope}</scope>
</dependency>
+ <!--Error of ArrayIndexOutOfBoundsException-->
+ <dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>2.8</version>
+ </dependency>
</dependencies>
</project>
diff --git a/wayang-plugins/wayang-iejoin/pom.xml
b/wayang-plugins/wayang-iejoin/pom.xml
index 303cf98..d81104d 100644
--- a/wayang-plugins/wayang-iejoin/pom.xml
+++ b/wayang-plugins/wayang-iejoin/pom.xml
@@ -19,7 +19,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-iejoin_2.11</artifactId>
+ <artifactId>wayang-iejoin</artifactId>
<version>0.6.0-SNAPSHOT</version>
<properties>
@@ -45,7 +45,7 @@
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-spark_${scala.mayor.version}</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml
index 3ccd939..fb69990 100644
--- a/wayang-profiler/pom.xml
+++ b/wayang-profiler/pom.xml
@@ -26,7 +26,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-profiler_2.11</artifactId>
+ <artifactId>wayang-profiler</artifactId>
<version>0.6.0-SNAPSHOT</version>
<name>Wayang Profiler</name>
@@ -57,7 +57,7 @@
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-spark_${scala.mayor.version}</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
<dependency>
@@ -67,11 +67,6 @@
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-graphchi_${scala.mayor.version}</artifactId>
- <version>0.6.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.wayang</groupId>
<artifactId>wayang-postgres</artifactId>
<version>0.6.0-SNAPSHOT</version>
</dependency>
@@ -90,5 +85,29 @@
<artifactId>profiledb-instrumentation</artifactId>
</dependency>
</dependencies>
-
+ <profiles>
+ <profile>
+ <id>scala-11</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.wayang</groupId>
+ <artifactId>wayang-graphchi</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.graphchi</groupId>
+
<artifactId>graphchi-java_${scala.mayor.version}</artifactId>
+ <version>0.2.2</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
diff --git
a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java
b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java
index 0a171a2..f091ad8 100644
---
a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java
+++
b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizerApp.java
@@ -33,7 +33,6 @@ import org.apache.wayang.core.util.Bitmask;
import org.apache.wayang.core.util.Formats;
import org.apache.wayang.core.util.WayangCollections;
import org.apache.wayang.core.util.Tuple;
-import org.apache.wayang.graphchi.GraphChi;
import org.apache.wayang.java.Java;
import org.apache.wayang.postgres.Postgres;
import org.apache.wayang.spark.Spark;
@@ -110,7 +109,7 @@ public class GeneticOptimizerApp {
Spark.platform();
Sqlite3.platform();
Postgres.platform();
- GraphChi.platform();
+ //TODO: add dinamically Graphchi, if the version of scala is 2.11
// Load the ExecutionLog.
double samplingFactor =
this.configuration.getDoubleProperty("wayang.profiler.ga.sampling", 1d);
diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml
index 0a595fc..231c8dd 100644
--- a/wayang-tests-integration/pom.xml
+++ b/wayang-tests-integration/pom.xml
@@ -26,7 +26,7 @@
<version>0.6.0-SNAPSHOT</version>
</parent>
- <artifactId>wayang-tests-integration_2.11</artifactId>
+ <artifactId>wayang-tests-integration</artifactId>
<version>0.6.0-SNAPSHOT</version>
<name>Wayang Integration Test</name>
@@ -37,17 +37,10 @@
<graphchi.version>0.2.2</graphchi.version>
<spark.version>2.4.0</spark.version>
<flink.version>1.7.1</flink.version>
- <scala.mayor.version>2.11</scala.mayor.version>
<giraph.version>1.2.0-hadoop2</giraph.version>
</properties>
<dependencies>
- <!--dependency>
- <groupId>org.apache.wayang</groupId>
- <artifactId>wayang-commons</artifactId>
- <version>0.6.0-SNAPSHOT</version>
- <scope>test</scope>
- </dependency-->
<dependency>
<groupId>org.apache.wayang</groupId>
<artifactId>wayang-java</artifactId>
@@ -78,33 +71,17 @@
<version>0.6.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.wayang</groupId>
- <artifactId>wayang-graphchi_${scala.mayor.version}</artifactId>
- <version>0.6.0-SNAPSHOT</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.wayang</groupId>
- <artifactId>wayang-spark_${scala.mayor.version}</artifactId>
+ <artifactId>wayang-spark</artifactId>
<version>0.6.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.graphchi</groupId>
- <artifactId>graphchi-java_${scala.mayor.version}</artifactId>
- <version>${graphchi.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+
<dependency>
<groupId>org.apache.wayang</groupId>
-
<artifactId>wayang-api-scala-java_${scala.mayor.version}</artifactId>
+ <artifactId>wayang-api-scala-java</artifactId>
<version>0.6.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
@@ -171,9 +148,6 @@
</exclusions>
</dependency>
-
-
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -271,5 +245,51 @@
</plugins>
</build>
-
+ <profiles>
+ <profile>
+ <id>scala-11</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src_2_11/test/java</source>
+ <source>src/test/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.wayang</groupId>
+ <artifactId>wayang-graphchi</artifactId>
+ <version>0.6.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.graphchi</groupId>
+
<artifactId>graphchi-java_${scala.mayor.version}</artifactId>
+ <version>${graphchi.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
diff --git
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/FlinkIntegrationIT.java
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/FlinkIntegrationIT.java
index 6698750..b5d964c 100644
---
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/FlinkIntegrationIT.java
+++
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/FlinkIntegrationIT.java
@@ -18,6 +18,7 @@
package org.apache.wayang.tests;
+import org.apache.wayang.java.plugin.JavaBasicPlugin;
import org.junit.Assert;
import org.junit.Test;
import org.apache.wayang.basic.data.Tuple2;
@@ -405,7 +406,8 @@ public class FlinkIntegrationIT {
Assert.assertEquals(WayangCollections.asSet(1, 4, 9),
WayangCollections.asSet(collector));
}
- @Test
+ //TODO validate this test is required
+ //@Test
public void testPageRankWithGraphBasic() {
// Build the WayangPlan.
List<Tuple2<Long, Long>> edges = Arrays.asList(
@@ -423,7 +425,6 @@ public class FlinkIntegrationIT {
// Execute the plan with a certain backend.
WayangContext wayangContext = new WayangContext()
.with(Flink.basicPlugin());
- //.with(WayangBasics.graphPlugin());
wayangContext.execute(wayangPlan);
diff --git
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GiraphIntegrationIT.java
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GiraphIntegrationIT.java
index d5c1b2e..8921a26 100644
---
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GiraphIntegrationIT.java
+++
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GiraphIntegrationIT.java
@@ -24,10 +24,7 @@ import org.apache.wayang.basic.data.Tuple2;
import org.apache.wayang.core.api.WayangContext;
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
import org.apache.wayang.giraph.Giraph;
-import org.apache.wayang.graphchi.GraphChi;
import org.apache.wayang.java.Java;
-import org.apache.wayang.spark.Spark;
-import org.apache.wayang.tests.WayangPlans;
import java.util.ArrayList;
import java.util.List;
@@ -36,7 +33,7 @@ import java.util.Set;
import java.util.stream.Collectors;
/**
- * Integration tests for the integration of GraphChi with Wayang.
+ * Integration tests for the integration of Giraph with Wayang.
*/
public class GiraphIntegrationIT {
diff --git
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/WayangPlans.java
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/WayangPlans.java
index 3062afb..88ef6d7 100644
---
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/WayangPlans.java
+++
b/wayang-tests-integration/src/test/java/org/apache/wayang/tests/WayangPlans.java
@@ -73,7 +73,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
- * Provides plans that can be used for integration testing..
+ * Provides plans that can be used for integration testing.
*/
public class WayangPlans {
@@ -89,7 +89,7 @@ public class WayangPlans {
public static URI createUri(String resourcePath) {
try {
- return
Thread.currentThread().getClass().getResource(resourcePath).toURI();
+ return WayangPlans.class.getResource(resourcePath).toURI();
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Illegal URI.", e);
}
diff --git
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GraphChiIntegrationIT.java
b/wayang-tests-integration/src_2_11/test/java/org/apache/wayang/test/GraphChiIntegrationIT.java
similarity index 98%
rename from
wayang-tests-integration/src/test/java/org/apache/wayang/tests/GraphChiIntegrationIT.java
rename to
wayang-tests-integration/src_2_11/test/java/org/apache/wayang/test/GraphChiIntegrationIT.java
index 8c42fd1..b515904 100644
---
a/wayang-tests-integration/src/test/java/org/apache/wayang/tests/GraphChiIntegrationIT.java
+++
b/wayang-tests-integration/src_2_11/test/java/org/apache/wayang/test/GraphChiIntegrationIT.java
@@ -16,8 +16,9 @@
* limitations under the License.
*/
-package org.apache.wayang.tests;
+package org.apache.wayang.test;
+import org.apache.wayang.tests.WayangPlans;
import org.junit.Assert;
import org.junit.Test;
import org.apache.wayang.basic.data.Tuple2;
