http://git-wip-us.apache.org/repos/asf/spark/blob/4af9de7d/repl/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala b/repl/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala deleted file mode 100644 index 13cd2b7..0000000 --- a/repl/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala +++ /dev/null @@ -1,232 +0,0 @@ -// scalastyle:off - -/* NSC -- new Scala compiler - * Copyright 2005-2013 LAMP/EPFL - * @author Martin Odersky - */ - -package org.apache.spark.repl - -import scala.tools.nsc._ -import scala.tools.nsc.interpreter._ - -import scala.collection.{ mutable, immutable } -import scala.PartialFunction.cond -import scala.reflect.internal.Chars -import scala.reflect.internal.Flags._ -import scala.language.implicitConversions - -trait SparkMemberHandlers { - val intp: SparkIMain - - import intp.{ Request, global, naming } - import global._ - import naming._ - - private def codegenln(leadingPlus: Boolean, xs: String*): String = codegen(leadingPlus, (xs ++ Array("\n")): _*) - private def codegenln(xs: String*): String = codegenln(true, xs: _*) - - private def codegen(xs: String*): String = codegen(true, xs: _*) - private def codegen(leadingPlus: Boolean, xs: String*): String = { - val front = if (leadingPlus) "+ " else "" - front + (xs map string2codeQuoted mkString " + ") - } - private implicit def name2string(name: Name) = name.toString - - /** A traverser that finds all mentioned identifiers, i.e. things - * that need to be imported. It might return extra names. - */ - private class ImportVarsTraverser extends Traverser { - val importVars = new mutable.HashSet[Name]() - - override def traverse(ast: Tree) = ast match { - case Ident(name) => - // XXX this is obviously inadequate but it's going to require some effort - // to get right. - if (name.toString startsWith "x$") () - else importVars += name - case _ => super.traverse(ast) - } - } - private object ImportVarsTraverser { - def apply(member: Tree) = { - val ivt = new ImportVarsTraverser() - ivt traverse member - ivt.importVars.toList - } - } - - def chooseHandler(member: Tree): MemberHandler = member match { - case member: DefDef => new DefHandler(member) - case member: ValDef => new ValHandler(member) - case member: Assign => new AssignHandler(member) - case member: ModuleDef => new ModuleHandler(member) - case member: ClassDef => new ClassHandler(member) - case member: TypeDef => new TypeAliasHandler(member) - case member: Import => new ImportHandler(member) - case DocDef(_, documented) => chooseHandler(documented) - case member => new GenericHandler(member) - } - - sealed abstract class MemberDefHandler(override val member: MemberDef) extends MemberHandler(member) { - def symbol = if (member.symbol eq null) NoSymbol else member.symbol - def name: Name = member.name - def mods: Modifiers = member.mods - def keyword = member.keyword - def prettyName = name.decode - - override def definesImplicit = member.mods.isImplicit - override def definesTerm: Option[TermName] = Some(name.toTermName) filter (_ => name.isTermName) - override def definesType: Option[TypeName] = Some(name.toTypeName) filter (_ => name.isTypeName) - override def definedSymbols = if (symbol eq NoSymbol) Nil else List(symbol) - } - - /** Class to handle one member among all the members included - * in a single interpreter request. - */ - sealed abstract class MemberHandler(val member: Tree) { - def definesImplicit = false - def definesValue = false - def isLegalTopLevel = false - - def definesTerm = Option.empty[TermName] - def definesType = Option.empty[TypeName] - - lazy val referencedNames = ImportVarsTraverser(member) - def importedNames = List[Name]() - def definedNames = definesTerm.toList ++ definesType.toList - def definedOrImported = definedNames ++ importedNames - def definedSymbols = List[Symbol]() - - def extraCodeToEvaluate(req: Request): String = "" - def resultExtractionCode(req: Request): String = "" - - private def shortName = this.getClass.toString split '.' last - override def toString = shortName + referencedNames.mkString(" (refs: ", ", ", ")") - } - - class GenericHandler(member: Tree) extends MemberHandler(member) - - class ValHandler(member: ValDef) extends MemberDefHandler(member) { - val maxStringElements = 1000 // no need to mkString billions of elements - override def definesValue = true - - override def resultExtractionCode(req: Request): String = { - val isInternal = isUserVarName(name) && req.lookupTypeOf(name) == "Unit" - if (!mods.isPublic || isInternal) "" - else { - // if this is a lazy val we avoid evaluating it here - val resultString = - if (mods.isLazy) codegenln(false, "<lazy>") - else any2stringOf(req fullPath name, maxStringElements) - - val vidString = - if (replProps.vids) """" + " @ " + "%%8x".format(System.identityHashCode(%s)) + " """.trim.format(req fullPath name) - else "" - - """ + "%s%s: %s = " + %s""".format(string2code(prettyName), vidString, string2code(req typeOf name), resultString) - } - } - } - - class DefHandler(member: DefDef) extends MemberDefHandler(member) { - private def vparamss = member.vparamss - private def isMacro = member.symbol hasFlag MACRO - // true if not a macro and 0-arity - override def definesValue = !isMacro && flattensToEmpty(vparamss) - override def resultExtractionCode(req: Request) = - if (mods.isPublic) codegenln(name, ": ", req.typeOf(name)) else "" - } - - class AssignHandler(member: Assign) extends MemberHandler(member) { - val Assign(lhs, rhs) = member - val name = newTermName(freshInternalVarName()) - - override def definesTerm = Some(name) - override def definesValue = true - override def extraCodeToEvaluate(req: Request) = - """val %s = %s""".format(name, lhs) - - /** Print out lhs instead of the generated varName */ - override def resultExtractionCode(req: Request) = { - val lhsType = string2code(req lookupTypeOf name) - val res = string2code(req fullPath name) - """ + "%s: %s = " + %s + "\n" """.format(string2code(lhs.toString), lhsType, res) + "\n" - } - } - - class ModuleHandler(module: ModuleDef) extends MemberDefHandler(module) { - override def definesTerm = Some(name) - override def definesValue = true - override def isLegalTopLevel = true - - override def resultExtractionCode(req: Request) = codegenln("defined module ", name) - } - - class ClassHandler(member: ClassDef) extends MemberDefHandler(member) { - override def definesType = Some(name.toTypeName) - override def definesTerm = Some(name.toTermName) filter (_ => mods.isCase) - override def isLegalTopLevel = true - - override def resultExtractionCode(req: Request) = - codegenln("defined %s %s".format(keyword, name)) - } - - class TypeAliasHandler(member: TypeDef) extends MemberDefHandler(member) { - private def isAlias = mods.isPublic && treeInfo.isAliasTypeDef(member) - override def definesType = Some(name.toTypeName) filter (_ => isAlias) - - override def resultExtractionCode(req: Request) = - codegenln("defined type alias ", name) + "\n" - } - - class ImportHandler(imp: Import) extends MemberHandler(imp) { - val Import(expr, selectors) = imp - def targetType: Type = intp.typeOfExpression("" + expr) - override def isLegalTopLevel = true - - def createImportForName(name: Name): String = { - selectors foreach { - case sel @ ImportSelector(old, _, `name`, _) => return "import %s.{ %s }".format(expr, sel) - case _ => () - } - "import %s.%s".format(expr, name) - } - // TODO: Need to track these specially to honor Predef masking attempts, - // because they must be the leading imports in the code generated for each - // line. We can use the same machinery as Contexts now, anyway. - def isPredefImport = isReferenceToPredef(expr) - - // wildcard imports, e.g. import foo._ - private def selectorWild = selectors filter (_.name == nme.USCOREkw) - // renamed imports, e.g. import foo.{ bar => baz } - private def selectorRenames = selectors map (_.rename) filterNot (_ == null) - - /** Whether this import includes a wildcard import */ - val importsWildcard = selectorWild.nonEmpty - - /** Whether anything imported is implicit .*/ - def importsImplicit = implicitSymbols.nonEmpty - - def implicitSymbols = importedSymbols filter (_.isImplicit) - def importedSymbols = individualSymbols ++ wildcardSymbols - - lazy val individualSymbols: List[Symbol] = - beforePickler(individualNames map (targetType nonPrivateMember _)) - - lazy val wildcardSymbols: List[Symbol] = - if (importsWildcard) beforePickler(targetType.nonPrivateMembers.toList) - else Nil - - /** Complete list of names imported by a wildcard */ - lazy val wildcardNames: List[Name] = wildcardSymbols map (_.name) - lazy val individualNames: List[Name] = selectorRenames filterNot (_ == nme.USCOREkw) flatMap (_.bothNames) - - /** The names imported by this statement */ - override lazy val importedNames: List[Name] = wildcardNames ++ individualNames - lazy val importsSymbolNamed: Set[String] = importedNames map (_.toString) toSet - - def importString = imp.toString - override def resultExtractionCode(req: Request) = codegenln(importString) + "\n" - } -}
http://git-wip-us.apache.org/repos/asf/spark/blob/4af9de7d/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala b/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala deleted file mode 100644 index 7fd5fbb..0000000 --- a/repl/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.repl - -import scala.tools.nsc.Settings - -/** - * <i>scala.tools.nsc.Settings</i> implementation adding Spark-specific REPL - * command line options. - */ -class SparkRunnerSettings(error: String => Unit) extends Settings(error){ - - val loadfiles = MultiStringSetting( - "-i", - "file", - "load a file (assumes the code is given interactively)") -} http://git-wip-us.apache.org/repos/asf/spark/blob/4af9de7d/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala ---------------------------------------------------------------------- diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala deleted file mode 100644 index 91c9c52..0000000 --- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.repl - -import java.io._ -import java.net.URLClassLoader - -import scala.collection.mutable.ArrayBuffer - -import org.scalatest.FunSuite -import org.apache.spark.SparkContext -import org.apache.commons.lang3.StringEscapeUtils -import org.apache.spark.util.Utils - - -class ReplSuite extends FunSuite { - - def runInterpreter(master: String, input: String): String = { - val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath" - - val in = new BufferedReader(new StringReader(input + "\n")) - val out = new StringWriter() - val cl = getClass.getClassLoader - var paths = new ArrayBuffer[String] - if (cl.isInstanceOf[URLClassLoader]) { - val urlLoader = cl.asInstanceOf[URLClassLoader] - for (url <- urlLoader.getURLs) { - if (url.getProtocol == "file") { - paths += url.getFile - } - } - } - val classpath = paths.mkString(File.pathSeparator) - - val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH) - System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath) - - val interp = new SparkILoop(in, new PrintWriter(out), master) - org.apache.spark.repl.Main.interp = interp - interp.process(Array("-classpath", classpath)) - org.apache.spark.repl.Main.interp = null - if (interp.sparkContext != null) { - interp.sparkContext.stop() - } - if (oldExecutorClasspath != null) { - System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath) - } else { - System.clearProperty(CONF_EXECUTOR_CLASSPATH) - } - return out.toString - } - - def assertContains(message: String, output: String) { - val isContain = output.contains(message) - assert(isContain, - "Interpreter output did not contain '" + message + "':\n" + output) - } - - def assertDoesNotContain(message: String, output: String) { - val isContain = output.contains(message) - assert(!isContain, - "Interpreter output contained '" + message + "':\n" + output) - } - - test("propagation of local properties") { - // A mock ILoop that doesn't install the SIGINT handler. - class ILoop(out: PrintWriter) extends SparkILoop(None, out, None) { - settings = new scala.tools.nsc.Settings - settings.usejavacp.value = true - org.apache.spark.repl.Main.interp = this - override def createInterpreter() { - intp = new SparkILoopInterpreter - intp.setContextClassLoader() - } - } - - val out = new StringWriter() - val interp = new ILoop(new PrintWriter(out)) - interp.sparkContext = new SparkContext("local", "repl-test") - interp.createInterpreter() - interp.intp.initialize() - interp.sparkContext.setLocalProperty("someKey", "someValue") - - // Make sure the value we set in the caller to interpret is propagated in the thread that - // interprets the command. - interp.interpret("org.apache.spark.repl.Main.interp.sparkContext.getLocalProperty(\"someKey\")") - assert(out.toString.contains("someValue")) - - interp.sparkContext.stop() - System.clearProperty("spark.driver.port") - } - - test("simple foreach with accumulator") { - val output = runInterpreter("local", - """ - |val accum = sc.accumulator(0) - |sc.parallelize(1 to 10).foreach(x => accum += x) - |accum.value - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res1: Int = 55", output) - } - - test("external vars") { - val output = runInterpreter("local", - """ - |var v = 7 - |sc.parallelize(1 to 10).map(x => v).collect.reduceLeft(_+_) - |v = 10 - |sc.parallelize(1 to 10).map(x => v).collect.reduceLeft(_+_) - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 70", output) - assertContains("res1: Int = 100", output) - } - - test("external classes") { - val output = runInterpreter("local", - """ - |class C { - |def foo = 5 - |} - |sc.parallelize(1 to 10).map(x => (new C).foo).collect.reduceLeft(_+_) - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 50", output) - } - - test("external functions") { - val output = runInterpreter("local", - """ - |def double(x: Int) = x + x - |sc.parallelize(1 to 10).map(x => double(x)).collect.reduceLeft(_+_) - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 110", output) - } - - test("external functions that access vars") { - val output = runInterpreter("local", - """ - |var v = 7 - |def getV() = v - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - |v = 10 - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 70", output) - assertContains("res1: Int = 100", output) - } - - test("broadcast vars") { - // Test that the value that a broadcast var had when it was created is used, - // even if that variable is then modified in the driver program - // TODO: This doesn't actually work for arrays when we run in local mode! - val output = runInterpreter("local", - """ - |var array = new Array[Int](5) - |val broadcastArray = sc.broadcast(array) - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - |array(0) = 5 - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Array[Int] = Array(0, 0, 0, 0, 0)", output) - assertContains("res2: Array[Int] = Array(5, 0, 0, 0, 0)", output) - } - - test("interacting with files") { - val tempDir = Utils.createTempDir() - val out = new FileWriter(tempDir + "/input") - out.write("Hello world!\n") - out.write("What's up?\n") - out.write("Goodbye\n") - out.close() - val output = runInterpreter("local", - """ - |var file = sc.textFile("%s").cache() - |file.count() - |file.count() - |file.count() - """.stripMargin.format(StringEscapeUtils.escapeJava( - tempDir.getAbsolutePath + File.separator + "input"))) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Long = 3", output) - assertContains("res1: Long = 3", output) - assertContains("res2: Long = 3", output) - Utils.deleteRecursively(tempDir) - } - - test("local-cluster mode") { - val output = runInterpreter("local-cluster[1,1,512]", - """ - |var v = 7 - |def getV() = v - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - |v = 10 - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - |var array = new Array[Int](5) - |val broadcastArray = sc.broadcast(array) - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - |array(0) = 5 - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 70", output) - assertContains("res1: Int = 100", output) - assertContains("res2: Array[Int] = Array(0, 0, 0, 0, 0)", output) - assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output) - } - - test("SPARK-1199 two instances of same class don't type check.") { - val output = runInterpreter("local-cluster[1,1,512]", - """ - |case class Sum(exp: String, exp2: String) - |val a = Sum("A", "B") - |def b(a: Sum): String = a match { case Sum(_, _) => "Found Sum" } - |b(a) - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - } - - test("SPARK-2452 compound statements.") { - val output = runInterpreter("local", - """ - |val x = 4 ; def f() = x - |f() - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - } - - test("SPARK-2576 importing SQLContext.createSchemaRDD.") { - // We need to use local-cluster to test this case. - val output = runInterpreter("local-cluster[1,1,512]", - """ - |val sqlContext = new org.apache.spark.sql.SQLContext(sc) - |import sqlContext.createSchemaRDD - |case class TestCaseClass(value: Int) - |sc.parallelize(1 to 10).map(x => TestCaseClass(x)).toSchemaRDD.collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - } - - test("SPARK-2632 importing a method from non serializable class and not using it.") { - val output = runInterpreter("local", - """ - |class TestClass() { def testMethod = 3 } - |val t = new TestClass - |import t.testMethod - |case class TestCaseClass(value: Int) - |sc.parallelize(1 to 10).map(x => TestCaseClass(x)).collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - } - - if (System.getenv("MESOS_NATIVE_LIBRARY") != null) { - test("running on Mesos") { - val output = runInterpreter("localquiet", - """ - |var v = 7 - |def getV() = v - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - |v = 10 - |sc.parallelize(1 to 10).map(x => getV()).collect.reduceLeft(_+_) - |var array = new Array[Int](5) - |val broadcastArray = sc.broadcast(array) - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - |array(0) = 5 - |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("res0: Int = 70", output) - assertContains("res1: Int = 100", output) - assertContains("res2: Array[Int] = Array(0, 0, 0, 0, 0)", output) - assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output) - } - } - - test("collecting objects of class defined in repl") { - val output = runInterpreter("local[2]", - """ - |case class Foo(i: Int) - |val ret = sc.parallelize((1 to 100).map(Foo), 10).collect - """.stripMargin) - assertDoesNotContain("error:", output) - assertDoesNotContain("Exception", output) - assertContains("ret: Array[Foo] = Array(Foo(1),", output) - } -} http://git-wip-us.apache.org/repos/asf/spark/blob/4af9de7d/sql/catalyst/pom.xml ---------------------------------------------------------------------- diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 0d756f8..0cc3175 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -44,11 +44,7 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-reflect</artifactId> </dependency> - <dependency> - <groupId>org.scalamacros</groupId> - <artifactId>quasiquotes_${scala.binary.version}</artifactId> - <version>${scala.macros.version}</version> - </dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.binary.version}</artifactId> @@ -103,4 +99,27 @@ </plugin> </plugins> </build> + <profiles> + <profile> + <id>scala-2.10</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.scalamacros</groupId> + <artifactId>quasiquotes_${scala.binary.version}</artifactId> + <version>${scala.macros.version}</version> + </dependency> + </dependencies> + </profile> + <profile> + <id>scala-2.11</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. --> + </profile> + + </profiles> </project> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
