Repository: spark Updated Branches: refs/heads/master 23af2d79a -> 633ffd816
rename the file. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/633ffd81 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/633ffd81 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/633ffd81 Branch: refs/heads/master Commit: 633ffd816d285480bab1f346471135b10ec092bb Parents: 23af2d7 Author: gatorsmile <[email protected]> Authored: Tue Oct 10 11:01:02 2017 -0700 Committer: gatorsmile <[email protected]> Committed: Tue Oct 10 11:01:02 2017 -0700 ---------------------------------------------------------------------- .../codegen/BufferHolderSparkSubmitSuite.scala | 78 ++++++++++++++++++++ .../codegen/BufferHolderSparkSubmitSutie.scala | 78 -------------------- 2 files changed, 78 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/633ffd81/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala new file mode 100644 index 0000000..1167d2f --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions.codegen + +import org.scalatest.{BeforeAndAfterEach, Matchers} +import org.scalatest.concurrent.Timeouts + +import org.apache.spark.{SparkFunSuite, TestUtils} +import org.apache.spark.deploy.SparkSubmitSuite +import org.apache.spark.sql.catalyst.expressions.UnsafeRow +import org.apache.spark.unsafe.array.ByteArrayMethods +import org.apache.spark.util.ResetSystemProperties + +// A test for growing the buffer holder to nearly 2GB. Due to the heap size limitation of the Spark +// unit tests JVM, the actually test code is running as a submit job. +class BufferHolderSparkSubmitSuite + extends SparkFunSuite + with Matchers + with BeforeAndAfterEach + with ResetSystemProperties + with Timeouts { + + test("SPARK-22222: Buffer holder should be able to allocate memory larger than 1GB") { + val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + + val argsForSparkSubmit = Seq( + "--class", BufferHolderSparkSubmitSuite.getClass.getName.stripSuffix("$"), + "--name", "SPARK-22222", + "--master", "local-cluster[2,1,1024]", + "--driver-memory", "4g", + "--conf", "spark.ui.enabled=false", + "--conf", "spark.master.rest.enabled=false", + "--conf", "spark.driver.extraJavaOptions=-ea", + unusedJar.toString) + SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..") + } +} + +object BufferHolderSparkSubmitSuite { + + def main(args: Array[String]): Unit = { + + val ARRAY_MAX = ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH + + val holder = new BufferHolder(new UnsafeRow(1000)) + + holder.reset() + holder.grow(roundToWord(ARRAY_MAX / 2)) + + holder.reset() + holder.grow(roundToWord(ARRAY_MAX / 2 + 8)) + + holder.reset() + holder.grow(roundToWord(Integer.MAX_VALUE / 2)) + + holder.reset() + holder.grow(roundToWord(Integer.MAX_VALUE)) + } + + private def roundToWord(len: Int): Int = { + ByteArrayMethods.roundNumberOfBytesToNearestWord(len) + } +} http://git-wip-us.apache.org/repos/asf/spark/blob/633ffd81/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSutie.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSutie.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSutie.scala deleted file mode 100644 index 1167d2f..0000000 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSutie.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.expressions.codegen - -import org.scalatest.{BeforeAndAfterEach, Matchers} -import org.scalatest.concurrent.Timeouts - -import org.apache.spark.{SparkFunSuite, TestUtils} -import org.apache.spark.deploy.SparkSubmitSuite -import org.apache.spark.sql.catalyst.expressions.UnsafeRow -import org.apache.spark.unsafe.array.ByteArrayMethods -import org.apache.spark.util.ResetSystemProperties - -// A test for growing the buffer holder to nearly 2GB. Due to the heap size limitation of the Spark -// unit tests JVM, the actually test code is running as a submit job. -class BufferHolderSparkSubmitSuite - extends SparkFunSuite - with Matchers - with BeforeAndAfterEach - with ResetSystemProperties - with Timeouts { - - test("SPARK-22222: Buffer holder should be able to allocate memory larger than 1GB") { - val unusedJar = TestUtils.createJarWithClasses(Seq.empty) - - val argsForSparkSubmit = Seq( - "--class", BufferHolderSparkSubmitSuite.getClass.getName.stripSuffix("$"), - "--name", "SPARK-22222", - "--master", "local-cluster[2,1,1024]", - "--driver-memory", "4g", - "--conf", "spark.ui.enabled=false", - "--conf", "spark.master.rest.enabled=false", - "--conf", "spark.driver.extraJavaOptions=-ea", - unusedJar.toString) - SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..") - } -} - -object BufferHolderSparkSubmitSuite { - - def main(args: Array[String]): Unit = { - - val ARRAY_MAX = ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH - - val holder = new BufferHolder(new UnsafeRow(1000)) - - holder.reset() - holder.grow(roundToWord(ARRAY_MAX / 2)) - - holder.reset() - holder.grow(roundToWord(ARRAY_MAX / 2 + 8)) - - holder.reset() - holder.grow(roundToWord(Integer.MAX_VALUE / 2)) - - holder.reset() - holder.grow(roundToWord(Integer.MAX_VALUE)) - } - - private def roundToWord(len: Int): Int = { - ByteArrayMethods.roundNumberOfBytesToNearestWord(len) - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
