This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch s3
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
The following commit(s) were added to refs/heads/s3 by this push:
new b24c25a0 corrections
b24c25a0 is described below
commit b24c25a00906f71475f87acbb8792247d9350a59
Author: Bertty Contreras-Rojas <[email protected]>
AuthorDate: Tue Jul 5 11:57:51 2022 +0000
corrections
---
.mvn/wrapper/maven-wrapper.jar | Bin 0 -> 58727 bytes
.../main/java/org/apache/wayang/apps/sgd/SGDImpl.java | 7 ++++---
wayang-benchmark/pom.xml | 10 ++++++++++
.../org/apache/wayang/core/util/fs/S3FileSystem.java | 17 +++++++++++++++++
.../apache/wayang/spark/execution/SparkExecutor.java | 1 +
5 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar
new file mode 100644
index 00000000..c1dd12f1
Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ
diff --git
a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
index d5331c64..7585e735 100644
--- a/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
+++ b/wayang-benchmark/code/main/java/org/apache/wayang/apps/sgd/SGDImpl.java
@@ -119,15 +119,16 @@ class Transform implements
FunctionDescriptor.SerializableFunction<String, doubl
@Override
public double[] apply(String line) {
- String[] pointStr = line.split(" ");
+ String[] pointStr = line.split(",");
double[] point = new double[features + 1];
point[0] = Double.parseDouble(pointStr[0]);
for (int i = 1; i < pointStr.length; i++) {
- if (pointStr[i].equals("")) {
+/* if (pointStr[i].equals("")) {
continue;
}
String kv[] = pointStr[i].split(":", 2);
- point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]);
+ point[Integer.parseInt(kv[0]) - 1] = Double.parseDouble(kv[1]);*/
+ point[i] = Double.parseDouble(pointStr[i]);
}
return point;
}
diff --git a/wayang-benchmark/pom.xml b/wayang-benchmark/pom.xml
index 7df47adf..5cace142 100644
--- a/wayang-benchmark/pom.xml
+++ b/wayang-benchmark/pom.xml
@@ -64,6 +64,16 @@
<artifactId>hadoop-aws</artifactId>
<version>3.1.2</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>3.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>3.1.2</version>
+ </dependency>
</dependencies>
<modules>
diff --git
a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
index 99fe3cb8..d175e632 100644
---
a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
+++
b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/util/fs/S3FileSystem.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.wayang.core.util.fs;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
diff --git
a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
index 9aab425a..7100de44 100644
---
a/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
+++
b/wayang-platforms/wayang-spark/code/main/java/org/apache/wayang/spark/execution/SparkExecutor.java
@@ -195,6 +195,7 @@ public class SparkExecutor extends PushExecutorTemplate {
@Override
public void dispose() {
super.dispose();
+ this.sparkContextReference.get().stop();
this.sparkContextReference.noteDiscardedReference(true);
}
