eyalbenivri closed pull request #31: Amaterasu 47 URL: https://github.com/apache/incubator-amaterasu/pull/31
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/frameworks/spark/dispatcher/build.gradle b/frameworks/spark/dispatcher/build.gradle new file mode 100644 index 0000000..a95d958 --- /dev/null +++ b/frameworks/spark/dispatcher/build.gradle @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + id 'com.github.johnrengelman.shadow' version '1.2.4' + id 'com.github.maiflai.scalatest' version '0.22' + id 'scala' +} + +shadowJar { + zip64 true +} + +repositories { + maven { + url "https://plugins.gradle.org/m2/"; + } + mavenCentral() +} + +dependencies { + compile 'org.scala-lang:scala-library:2.11.8' +// compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8' +// compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8' + + compile project(':common') + compile project(':leader-common') + compile project(':amaterasu-sdk') +} + +task copyToHomeBin(type: Copy) { + dependsOn shadowJar + from 'build/libs' + into '../../../build/amaterasu/bin' +} + +task copyToHome() { + dependsOn copyToHomeBin +} \ No newline at end of file diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/SparkSetupProvider.scala similarity index 95% rename from leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala rename to frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/SparkSetupProvider.scala index f6dea22..5e26e45 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/SparkSetupProvider.scala @@ -14,17 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.leader.frameworks.spark +package org.apache.amaterasu.frameworks.spark.dispatcher import java.io.File import org.apache.amaterasu.common.configuration.ClusterConfig -import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser} +import org.apache.amaterasu.leader.common.utilities.{DataLoader, MemoryFormatParser} import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration import scala.collection.mutable +import scala.collection.mutable + class SparkSetupProvider extends FrameworkSetupProvider { diff --git a/frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java b/frameworks/spark/runner/src/main/java/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkEntryPoint.java similarity index 96% rename from frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java rename to frameworks/spark/runner/src/main/java/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkEntryPoint.java index 6b79b2f..6df5ef3 100755 --- a/frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java +++ b/frameworks/spark/runner/src/main/java/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkEntryPoint.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark; +package org.apache.amaterasu.frameworks.spark.runner.pyspark; import org.apache.amaterasu.common.runtime.Environment; -import org.apache.amaterasu.framework.spark.runtime.AmaContext; +import org.apache.amaterasu.frameworks.spark.runtime.AmaContext; import org.apache.spark.SparkConf; import org.apache.spark.SparkEnv; import org.apache.spark.api.java.JavaSparkContext; diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/SparkRunnersProvider.scala similarity index 94% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/SparkRunnersProvider.scala index 652f32b..e6c0a7d 100644 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/SparkRunnersProvider.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner +package org.apache.amaterasu.frameworks.spark.runner import java.io._ @@ -24,9 +24,10 @@ import org.apache.amaterasu.common.dataobjects.ExecData import org.apache.amaterasu.common.execution.actions.Notifier import org.apache.amaterasu.common.execution.dependencies.{Dependencies, PythonDependencies, PythonPackage} import org.apache.amaterasu.common.logging.Logging -import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunner -import org.apache.amaterasu.framework.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner} -import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunner +import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner} +import org.apache.amaterasu.frameworks.spark.runner.sparksql.SparkSqlRunner +import org.apache.amaterasu.frameworks.spark.runner.pyspark.PySparkRunner +import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner} import org.apache.amaterasu.sdk.{AmaterasuRunner, RunnersProvider} import org.eclipse.aether.util.artifact.JavaScopes import org.sonatype.aether.repository.RemoteRepository diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkExecutionQueue.scala similarity index 95% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkExecutionQueue.scala index ddcf66c..4a7c3a2 100755 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkExecutionQueue.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark +package org.apache.amaterasu.frameworks.spark.runner.pyspark import java.util import java.util.concurrent.{LinkedBlockingQueue, TimeUnit} diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkResultQueue.scala similarity index 88% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkResultQueue.scala index 16abbe3..1b17a81 100755 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkResultQueue.scala @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark +package org.apache.amaterasu.frameworks.spark.runner.pyspark -import org.apache.amaterasu.framework.spark.runner.pyspark.ResultType.ResultType +import org.apache.amaterasu.frameworks.spark.runner.pyspark.ResultType.ResultType object ResultType extends Enumeration { type ResultType = Value diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunner.scala similarity index 98% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunner.scala index c015ec5..90f8c68 100644 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunner.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark +package org.apache.amaterasu.frameworks.spark.runner.pyspark import java.io.File import java.util diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/ResultQueue.scala similarity index 95% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/ResultQueue.scala index d0cb4ae..daca0fc 100755 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/ResultQueue.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark +package org.apache.amaterasu.frameworks.spark.runner.pyspark import java.util.concurrent.{LinkedBlockingQueue, TimeUnit} diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/AmaSparkILoop.scala similarity index 95% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/AmaSparkILoop.scala index ec874b6..1f803dc 100755 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/AmaSparkILoop.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.repl +package org.apache.amaterasu.frameworks.spark.runner.repl import java.io.PrintWriter diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkRunnerHelper.scala similarity index 99% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkRunnerHelper.scala index 18658ec..acb5981 100644 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkRunnerHelper.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.repl +package org.apache.amaterasu.frameworks.spark.runner.repl import java.io.{ByteArrayOutputStream, File, PrintWriter} import java.nio.file.{Files, Paths} diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunner.scala similarity index 97% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunner.scala index 46d3077..f660939 100755 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunner.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.repl +package org.apache.amaterasu.frameworks.spark.runner.repl import java.io.ByteArrayOutputStream import java.util @@ -22,7 +22,7 @@ import java.util import org.apache.amaterasu.common.execution.actions.Notifier import org.apache.amaterasu.common.logging.Logging import org.apache.amaterasu.common.runtime.Environment -import org.apache.amaterasu.framework.spark.runtime.AmaContext +import org.apache.amaterasu.frameworks.spark.runtime.AmaContext import org.apache.amaterasu.sdk.AmaterasuRunner import org.apache.spark.sql.{Dataset, SparkSession} @@ -142,7 +142,7 @@ class SparkScalaRunner(var env: Environment, interpreter.interpret("import org.apache.spark.sql.SQLContext") interpreter.interpret("import org.apache.spark.sql.{ Dataset, SparkSession }") interpreter.interpret("import org.apache.spark.sql.SaveMode") - interpreter.interpret("import org.apache.amaterasu.framework.spark.runtime.AmaContext") + interpreter.interpret("import org.apache.amaterasu.frameworks.spark.runtime.AmaContext") interpreter.interpret("import org.apache.amaterasu.common.runtime.Environment") // creating a map (_contextStore) to hold the different spark contexts diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparkr/SparkRRunner.scala similarity index 96% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparkr/SparkRRunner.scala index 390b06a..2dc7b4e 100644 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparkr/SparkRRunner.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.sparkr +package org.apache.amaterasu.frameworks.spark.runner.sparkr import java.io.ByteArrayOutputStream import java.util diff --git a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunner.scala similarity index 97% rename from frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunner.scala index 62af197..1696174 100644 --- a/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala +++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunner.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.sparksql +package org.apache.amaterasu.frameworks.spark.runner.sparksql import java.io.File import java.util @@ -22,7 +22,7 @@ import java.util import org.apache.amaterasu.common.execution.actions.Notifier import org.apache.amaterasu.common.logging.Logging import org.apache.amaterasu.common.runtime.Environment -import org.apache.amaterasu.framework.spark.runtime.AmaContext +import org.apache.amaterasu.frameworks.spark.runtime.AmaContext import org.apache.amaterasu.sdk.AmaterasuRunner import org.apache.commons.io.FilenameUtils import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession} diff --git a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/RunnersLoadingTests.scala similarity index 96% rename from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/RunnersLoadingTests.scala index 3629674..dff1e4a 100644 --- a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala +++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/RunnersLoadingTests.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner +package org.apache.amaterasu.frameworks.spark.runner import org.apache.amaterasu.common.runtime.Environment import org.apache.amaterasu.executor.common.executors.ProvidersFactory diff --git a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/SparkTestsSuite.scala similarity index 90% rename from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/SparkTestsSuite.scala index 0214568..6de3643 100644 --- a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala +++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/SparkTestsSuite.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark +package org.apache.amaterasu.frameworks.spark.runner import java.io.{ByteArrayOutputStream, File} @@ -22,10 +22,9 @@ import org.apache.amaterasu.common.dataobjects.ExecData import org.apache.amaterasu.common.execution.dependencies._ import org.apache.amaterasu.common.runtime.Environment import org.apache.amaterasu.executor.common.executors.ProvidersFactory -import org.apache.amaterasu.framework.spark.runner.RunnersLoadingTests -import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunnerTests -import org.apache.amaterasu.framework.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests} -import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunnerTests +import org.apache.amaterasu.frameworks.spark.runner.pyspark.PySparkRunnerTests +import org.apache.amaterasu.frameworks.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests} +import org.apache.amaterasu.frameworks.spark.runner.sparksql.SparkSqlRunnerTests import org.apache.amaterasu.utilities.TestNotifier import org.apache.spark.sql.SparkSession import org.scalatest._ diff --git a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunnerTests.scala similarity index 98% rename from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunnerTests.scala index a320e56..1ed029f 100755 --- a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala +++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/pyspark/PySparkRunnerTests.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.pyspark +package org.apache.amaterasu.frameworks.spark.runner.pyspark import java.io.File diff --git a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunnerTests.scala similarity index 94% rename from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunnerTests.scala index 26f2ceb..90b0122 100755 --- a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala +++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/repl/SparkScalaRunnerTests.scala @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.repl +package org.apache.amaterasu.frameworks.spark.runner.repl import org.apache.amaterasu.executor.common.executors.ProvidersFactory -import org.apache.amaterasu.framework.spark.runtime.AmaContext +import org.apache.amaterasu.frameworks.spark.runtime.AmaContext import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers} import scala.collection.JavaConverters._ diff --git a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunnerTests.scala similarity index 99% rename from frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunnerTests.scala index abb5745..f189580 100644 --- a/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala +++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/frameworks/spark/runner/sparksql/SparkSqlRunnerTests.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runner.sparksql +package org.apache.amaterasu.frameworks.spark.runner.sparksql import org.apache.amaterasu.common.runtime.Environment import org.apache.amaterasu.executor.common.executors.ProvidersFactory diff --git a/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala b/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/frameworks/spark/runtime/AmaContext.scala similarity index 97% rename from frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala rename to frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/frameworks/spark/runtime/AmaContext.scala index cb2eccc..fc9fb94 100644 --- a/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala +++ b/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/frameworks/spark/runtime/AmaContext.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.framework.spark.runtime +package org.apache.amaterasu.frameworks.spark.runtime import org.apache.amaterasu.common.logging.Logging import org.apache.amaterasu.common.runtime.Environment diff --git a/leader-common/build.gradle b/leader-common/build.gradle new file mode 100644 index 0000000..3f3ac98 --- /dev/null +++ b/leader-common/build.gradle @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + id "com.github.johnrengelman.shadow" version "1.2.4" + id 'com.github.maiflai.scalatest' version '0.22' + id 'scala' + id 'java' +} + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +repositories { + maven { + url "https://plugins.gradle.org/m2/"; + } + mavenCentral() +} + +dependencies { + compile 'org.scala-lang:scala-library:2.11.8' + + compile project(':common') + + compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-scala_2.11', version: '2.6.3' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.6.4' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.6.4' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.6.4' + compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.6.4' +} \ No newline at end of file diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/DataLoader.scala b/leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/DataLoader.scala similarity index 97% rename from leader/src/main/scala/org/apache/amaterasu/leader/utilities/DataLoader.scala rename to leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/DataLoader.scala index f401f9b..7a1bc6c 100755 --- a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/DataLoader.scala +++ b/leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/DataLoader.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.leader.utilities +package org.apache.amaterasu.leader.common.utilities import java.io.{File, FileInputStream} import java.nio.file.{Files, Paths} diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/MemoryFormatParser.scala b/leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala similarity index 95% rename from leader/src/main/scala/org/apache/amaterasu/leader/utilities/MemoryFormatParser.scala rename to leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala index 7aaa752..cabe0e5 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/MemoryFormatParser.scala +++ b/leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/MemoryFormatParser.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.amaterasu.leader.utilities +package org.apache.amaterasu.leader.common.utilities object MemoryFormatParser { diff --git a/leader/build.gradle b/leader/build.gradle index 3f9276e..a0de6f5 100644 --- a/leader/build.gradle +++ b/leader/build.gradle @@ -34,9 +34,9 @@ repositories { dependencies { compile 'org.scala-lang:scala-library:2.11.8' compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8' -// compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8' compile project(':common') + compile project(':leader-common') compile project(':amaterasu-sdk') compile group: 'com.github.scopt', name: 'scopt_2.11', version: '3.3.0' @@ -52,7 +52,6 @@ dependencies { compile group: 'org.eclipse.jetty', name: 'jetty-http', version: '9.2.19.v20160908' compile group: 'org.eclipse.jetty', name: 'jetty-io', version: '9.2.19.v20160908' compile group: 'org.eclipse.jetty', name: 'jetty-servlet', version: '9.2.19.v20160908' - //compile group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0' compile group: 'org.eclipse.jetty.toolchain', name: 'jetty-test-helper', version: '4.0' compile group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '4.2.0.201601211800-r' compile group: 'org.yaml', name: 'snakeyaml', version: '1.18' diff --git a/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java b/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java index 8f16ee7..5e86188 100644 --- a/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java +++ b/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java @@ -26,21 +26,10 @@ import org.apache.curator.framework.recipes.barriers.DistributedBarrier; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.*; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -52,19 +41,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.jms.Connection; -import javax.jms.JMSException; -import javax.jms.MessageConsumer; -import javax.jms.Session; -import javax.jms.Topic; +import javax.jms.*; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import static java.lang.System.exit; @@ -132,7 +113,7 @@ private void run(JobOpts opts, String[] args) throws Exception { " env HADOOP_USER_NAME=" + UserGroupInformation.getCurrentUser().getUserName() + " $JAVA_HOME/bin/java" + " -Dscala.usejavacp=false" + - " -Xmx1G" + + " -Xmx2G" + " org.apache.amaterasu.leader.yarn.ApplicationMaster " + joinStrings(args) + newId + @@ -148,6 +129,7 @@ private void run(JobOpts opts, String[] args) throws Exception { // Setup local ama folder on hdfs. try { + System.out.println("===> " + jarPathQualified); if (!fs.exists(jarPathQualified)) { File home = new File(opts.home); fs.mkdirs(jarPathQualified); @@ -157,6 +139,7 @@ private void run(JobOpts opts, String[] args) throws Exception { } // setup frameworks + System.out.println("===> setting up frameworks"); FrameworkProvidersFactory frameworkFactory = FrameworkProvidersFactory.apply(opts.env, config); for (String group : frameworkFactory.groups()) { System.out.println("===> setting up " + group); @@ -208,7 +191,15 @@ private void run(JobOpts opts, String[] args) throws Exception { // set local resource on master container Map<String, LocalResource> localResources = new HashMap<>(); - localResources.put("leader.jar", leaderJar); + //localResources.put("leader.jar", leaderJar); + // making the bin folder's content available to the appMaster + RemoteIterator<LocatedFileStatus> bin = fs.listFiles(Path.mergePaths(jarPath, new Path("/bin")), true); + + while (bin.hasNext()){ + LocatedFileStatus binFile = bin.next(); + localResources.put(binFile.getPath().getName(), setLocalResourceFromPath(binFile.getPath())); + } + localResources.put("amaterasu.properties", propFile); localResources.put("log4j.properties", log4jPropFile); amContainer.setLocalResources(localResources); diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala index 19b7ec3..1bb82ff 100755 --- a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala @@ -29,9 +29,10 @@ import org.apache.amaterasu.common.configuration.enums.ActionStatus.ActionStatus import org.apache.amaterasu.common.dataobjects.ActionData import org.apache.amaterasu.common.execution.actions.NotificationLevel.NotificationLevel import org.apache.amaterasu.common.execution.actions.{Notification, NotificationLevel, NotificationType} +import org.apache.amaterasu.leader.common.utilities.DataLoader import org.apache.amaterasu.leader.execution.frameworks.FrameworkProvidersFactory import org.apache.amaterasu.leader.execution.{JobLoader, JobManager} -import org.apache.amaterasu.leader.utilities.{DataLoader, HttpServer} +import org.apache.amaterasu.leader.utilities.HttpServer import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} import org.apache.curator.retry.ExponentialBackoffRetry import org.apache.log4j.LogManager diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala index 406c150..1f1aa25 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala @@ -21,16 +21,17 @@ import java.net.{InetAddress, ServerSocket, URLEncoder} import java.nio.ByteBuffer import java.util import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue} - import javax.jms.Session + import org.apache.activemq.ActiveMQConnectionFactory import org.apache.activemq.broker.BrokerService import org.apache.amaterasu.common.configuration.ClusterConfig import org.apache.amaterasu.common.dataobjects.ActionData import org.apache.amaterasu.common.logging.Logging +import org.apache.amaterasu.leader.common.utilities.DataLoader import org.apache.amaterasu.leader.execution.frameworks.FrameworkProvidersFactory import org.apache.amaterasu.leader.execution.{JobLoader, JobManager} -import org.apache.amaterasu.leader.utilities.{ActiveReportListener, Args, DataLoader} +import org.apache.amaterasu.leader.utilities.{ActiveReportListener, Args} import org.apache.curator.framework.recipes.barriers.DistributedBarrier import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} import org.apache.curator.retry.ExponentialBackoffRetry @@ -251,7 +252,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging { "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ", s"/bin/bash spark/bin/load-spark-env.sh && ", s"java -cp spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " + - "-Xmx1G " + + "-Xmx2G " + "-Dscala.usejavacp=true " + "-Dhdp.version=2.6.1.0-129 " + "org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher " + diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala index b178f52..24f28cc 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala @@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap import com.google.gson.Gson import org.apache.amaterasu.common.configuration.ClusterConfig import org.apache.amaterasu.common.logging.Logging +import org.apache.amaterasu.leader.common.utilities.DataLoader import org.apache.amaterasu.leader.execution.JobManager -import org.apache.amaterasu.leader.utilities.DataLoader import org.apache.hadoop.yarn.api.records._ import org.apache.hadoop.yarn.client.api.async.{AMRMClientAsync, NMClientAsync} import org.apache.hadoop.yarn.util.Records diff --git a/leader/src/main/scripts/ama-start-yarn.sh b/leader/src/main/scripts/ama-start-yarn.sh index f6af18f..fe14f6e 100755 --- a/leader/src/main/scripts/ama-start-yarn.sh +++ b/leader/src/main/scripts/ama-start-yarn.sh @@ -102,6 +102,8 @@ done echo "repo: ${REPO} " echo "force-bin: ${FORCE_BIN}" export HADOOP_USER_CLASSPATH_FIRST=true +export YARN_USER_CLASSPATH=${YARN_USER_CLASSPATH}:bin/* + CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-rc4-all.jar org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}" if [ -n "$REPO" ]; then diff --git a/leader/src/main/scripts/amaterasu.properties b/leader/src/main/scripts/amaterasu.properties index 0bd7afd..5c55d79 100755 --- a/leader/src/main/scripts/amaterasu.properties +++ b/leader/src/main/scripts/amaterasu.properties @@ -22,8 +22,10 @@ webserver.root=dist spark.version=2.2.1-bin-hadoop2.7 yarn.queue=default yarn.jarspath=hdfs:///apps/amaterasu -spark.home=/usr/hdp/current/spark2-client +spark.home=/usr/lib/spark #spark.home=/opt/cloudera/parcels/SPARK2-2.1.0.cloudera2-1.cdh5.7.0.p0.171658/lib/spark2 yarn.hadoop.home.dir=/etc/hadoop spark.opts.spark.yarn.am.extraJavaOptions="-Dhdp.version=2.6.1.0-129" spark.opts.spark.driver.extraJavaOptions="-Dhdp.version=2.6.1.0-129" +yarn.master.memoryMB=2048 +yarn.worker.memoryMB=2048 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index c222795..9343001 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,9 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +// Core include 'leader' project(':leader') +include 'leader-common' +project(':leader-common') + include 'common' project(':common') @@ -26,9 +31,13 @@ project(':executor') include 'sdk' findProject(':sdk')?.name = 'amaterasu-sdk' -//Spark +// Frameworks +// Spark include 'spark-runner' project(':spark-runner').projectDir=file("frameworks/spark/runner") include 'spark-runtime' project(':spark-runtime').projectDir=file("frameworks/spark/runtime") +include 'spark-dispatcher' +project(':spark-dispatcher').projectDir=file("frameworks/spark/dispatcher") + ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
