[S2GRAPH-117] moved non-play tests to s2core
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5b5b4334 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5b5b4334 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5b5b4334 Branch: refs/heads/master Commit: 5b5b4334aa8c7075fc9a2c3a4bc607cad7c8c024 Parents: fcb041b Author: Jong Wook Kim <[email protected]> Authored: Mon Oct 10 01:34:06 2016 -0400 Committer: Jong Wook Kim <[email protected]> Committed: Mon Oct 10 01:44:34 2016 -0400 ---------------------------------------------------------------------- build.sbt | 1 + conf/log4j.properties | 27 ++++ conf/logback.xml | 32 ----- loader/build.sbt | 2 +- loader/src/main/resources/log4j.properties | 27 ---- loader/src/test/resources/log4j.properties | 27 ++++ project/Common.scala | 3 +- s2core/build.sbt | 10 +- s2core/src/main/resources/logback.xml | 45 ------ s2core/src/test/resources/log4j.properties | 27 ++++ .../s2graph/core/Integrate/CrudTest.scala | 2 +- .../s2graph/core/Integrate/QueryTest.scala | 10 +- .../apache/s2graph/core/PostProcessSpec.scala | 130 +++++++++++++++++ .../core/benchmark/BenchmarkCommon.scala | 48 +++++++ .../s2graph/core/benchmark/GraphUtilSpec.scala | 142 ++++++++++++++++++ .../core/benchmark/JsonBenchmarkSpec.scala | 64 +++++++++ .../benchmark/OrderingUtilBenchmarkSpec.scala | 117 +++++++++++++++ .../core/benchmark/SamplingBenchmarkSpec.scala | 102 +++++++++++++ s2counter_core/build.sbt | 4 +- s2rest_netty/build.sbt | 2 +- s2rest_play/build.sbt | 6 +- .../rest/play/benchmark/BenchmarkCommon.scala | 48 ------- .../rest/play/benchmark/GraphUtilSpec.scala | 143 ------------------- .../rest/play/benchmark/JsonBenchmarkSpec.scala | 64 --------- .../benchmark/OrderingUtilBenchmarkSpec.scala | 117 --------------- .../play/benchmark/SamplingBenchmarkSpec.scala | 104 -------------- .../rest/play/controllers/PostProcessSpec.scala | 131 ----------------- spark/build.sbt | 12 +- 28 files changed, 713 insertions(+), 734 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index d2df8a6..36b56fd 100755 --- a/build.sbt +++ b/build.sbt @@ -31,6 +31,7 @@ lazy val commonSettings = Seq( testOptions in Test += Tests.Argument("-oDF"), concurrentRestrictions in Global += Tags.limit(Tags.Test, 1), parallelExecution in Test := false, + libraryDependencies ++= Common.loggingRuntime, resolvers ++= Seq( Resolver.mavenLocal ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/conf/log4j.properties b/conf/log4j.properties new file mode 100644 index 0000000..c13e516 --- /dev/null +++ b/conf/log4j.properties @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=${root.logger} +root.logger=INFO,console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# The noisier spark/kafka logs are suppressed +log4j.logger.org.apache.spark=WARN,console +log4j.logger.org.apache.kafka=WARN,console http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/conf/logback.xml ---------------------------------------------------------------------- diff --git a/conf/logback.xml b/conf/logback.xml deleted file mode 100644 index d3f09bc..0000000 --- a/conf/logback.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> - <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> - <layout class="ch.qos.logback.classic.PatternLayout"> - <Pattern> - %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n - </Pattern> - </layout> - </appender> - <root level="INFO"> - <appender-ref ref="STDOUT" /> - </root> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/loader/build.sbt ---------------------------------------------------------------------- diff --git a/loader/build.sbt b/loader/build.sbt index 0312a3b..73144bb 100644 --- a/loader/build.sbt +++ b/loader/build.sbt @@ -35,7 +35,7 @@ libraryDependencies ++= Seq( "org.apache.spark" %% "spark-hive" % sparkVersion % "provided", "org.apache.spark" %% "spark-streaming-kafka" % sparkVersion, "org.apache.httpcomponents" % "fluent-hc" % "4.2.5", - "org.specs2" %% "specs2-core" % "2.4.11" % "test", + "org.specs2" %% "specs2-core" % specs2Version % "test", "org.scalatest" %% "scalatest" % "2.2.1" % "test", "org.apache.hadoop" % "hadoop-distcp" % hadoopVersion ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/loader/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/loader/src/main/resources/log4j.properties b/loader/src/main/resources/log4j.properties deleted file mode 100644 index 9a4b8ef..0000000 --- a/loader/src/main/resources/log4j.properties +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -log4j.rootLogger=${root.logger} -root.logger=WARN,console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# The noisier spark logs go to file only -log4j.logger.org.apache.spark=WARN,console -log4j.logger.org.apache.kafka=WARN,console http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/loader/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/loader/src/test/resources/log4j.properties b/loader/src/test/resources/log4j.properties new file mode 100644 index 0000000..9a4b8ef --- /dev/null +++ b/loader/src/test/resources/log4j.properties @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=${root.logger} +root.logger=WARN,console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# The noisier spark logs go to file only +log4j.logger.org.apache.spark=WARN,console +log4j.logger.org.apache.kafka=WARN,console http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index fb4af84..f3dfc68 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -22,6 +22,7 @@ import sbt._ object Common { val sparkVersion = "1.4.1" val playVersion = "2.5.9" + val specs2Version = "3.8.5" val hbaseVersion = "1.2.2" val hadoopVersion = "2.7.3" @@ -32,7 +33,7 @@ object Common { "org.slf4j" % "slf4j-log4j12" % "1.7.21", "org.slf4j" % "jcl-over-slf4j" % "1.7.21", "org.slf4j" % "jul-to-slf4j" % "1.7.21" - ).map(_ % "runtime") + ).flatMap(dep => Seq(dep % "test", dep % "runtime")) /** rules to exclude logging backends and bridging libraries from dependency */ val loggingExcludes = Seq( http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/build.sbt ---------------------------------------------------------------------- diff --git a/s2core/build.sbt b/s2core/build.sbt index 0e19a86..80f37b0 100644 --- a/s2core/build.sbt +++ b/s2core/build.sbt @@ -23,7 +23,7 @@ name := """s2core""" scalacOptions ++= Seq("-deprecation") -libraryDependencies ++= loggingRuntime ++ Seq( +libraryDependencies ++= Seq( "org.slf4j" % "slf4j-api" % "1.7.21", "com.typesafe" % "config" % "1.2.1", "com.typesafe.play" %% "play-json" % playVersion, @@ -34,15 +34,17 @@ libraryDependencies ++= loggingRuntime ++ Seq( "org.apache.hbase" % "hbase-server" % hbaseVersion excludeLogging() exclude("com.google.protobuf", "protobuf*"), "org.apache.hbase" % "hbase-hadoop-compat" % hbaseVersion excludeLogging(), "org.apache.hbase" % "hbase-hadoop2-compat" % hbaseVersion excludeLogging(), - "org.apache.kafka" % "kafka-clients" % "0.8.2.0" excludeAll(loggingExcludes: _*) exclude("com.sun.jdmk", "j*") exclude("com.sun.jmx", "j*") exclude("javax.jms", "j*"), + "org.apache.kafka" % "kafka-clients" % "0.8.2.0" excludeLogging() exclude("com.sun.jdmk", "j*") exclude("com.sun.jmx", "j*") exclude("javax.jms", "j*"), "commons-pool" % "commons-pool" % "1.6", - "org.scalatest" %% "scalatest" % "2.2.4" % "test", "org.scalikejdbc" %% "scalikejdbc" % "2.1.4", "com.h2database" % "h2" % "1.4.192", "com.stumbleupon" % "async" % "1.4.1", "io.netty" % "netty" % "3.9.4.Final" force(), "org.hbase" % "asynchbase" % "1.7.2" excludeLogging(), - "net.bytebuddy" % "byte-buddy" % "1.4.26" + "net.bytebuddy" % "byte-buddy" % "1.4.26", + + "org.scalatest" %% "scalatest" % "2.2.4" % "test", + "org.specs2" %% "specs2-core" % specs2Version % "test" ) libraryDependencies := { http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/main/resources/logback.xml ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/logback.xml b/s2core/src/main/resources/logback.xml deleted file mode 100644 index 7c2a495..0000000 --- a/s2core/src/main/resources/logback.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> - -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -<configuration> - <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> - <withJansi>true</withJansi> - <encoder> - <pattern> - %d{ISO8601} [%highlight(%-5level)] [%gray(%logger{0})] [%thread] - %msg%n - </pattern> - </encoder> - </appender> - - <!--<root level="INFO">--> - <!--<appender-ref ref="STDOUT"/>--> - <!--</root>--> - - <logger name="application" level="DEBUG"> - <appender-ref ref="STDOUT"/> - </logger> - - <logger name="error" level="DEBUG"> - <appender-ref ref="STDOUT"/> - </logger> - -</configuration> - http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/s2core/src/test/resources/log4j.properties b/s2core/src/test/resources/log4j.properties new file mode 100644 index 0000000..c13e516 --- /dev/null +++ b/s2core/src/test/resources/log4j.properties @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=${root.logger} +root.logger=INFO,console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# The noisier spark/kafka logs are suppressed +log4j.logger.org.apache.spark=WARN,console +log4j.logger.org.apache.kafka=WARN,console http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala index 3911e0d..fda9991 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/CrudTest.scala @@ -223,7 +223,7 @@ class CrudTest extends IntegrateCommon { for ((key, expectedVal) <- expected) { propsLs.last.as[JsObject].keys.contains(key) should be(true) - (propsLs.last \ key).toString should be(expectedVal) + (propsLs.last \ key).get.toString should be(expectedVal) } } } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala index c0071fa..9c52b32 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala @@ -54,16 +54,16 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { """) var edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 1001)) // test interval on timestamp index - (edges \ "size").toString should be("1") + (edges \ "size").get.toString should be("1") edges = getEdgesSync(queryWithInterval(0, index2, "_timestamp", 1000, 2000)) // test interval on timestamp index - (edges \ "size").toString should be("2") + (edges \ "size").get.toString should be("2") edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 11)) // test interval on weight index - (edges \ "size").toString should be("1") + (edges \ "size").get.toString should be("1") edges = getEdgesSync(queryWithInterval(2, index1, "weight", 10, 20)) // test interval on weight index - (edges \ "size").toString should be("2") + (edges \ "size").get.toString should be("2") } test("get edge with where condition") { @@ -604,7 +604,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { logger.debug(Json.prettyPrint(rs)) val results = (rs \ "results").as[List[JsValue]] results.size should be(1) - (results(0) \ "to").toString should be("555") + (results(0) \ "to").get.toString should be("555") } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/PostProcessSpec.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/PostProcessSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/PostProcessSpec.scala new file mode 100644 index 0000000..cd809f6 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/PostProcessSpec.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core + +import org.specs2.mutable.Specification +import play.api.libs.json.{JsNumber, JsString, JsValue} + +class PostProcessSpec extends Specification { + import OrderingUtil._ + + "test order by json" >> { + val jsLs: Seq[Seq[JsValue]] = Seq( + Seq(JsNumber(0), JsString("a")), + Seq(JsNumber(0), JsString("b")), + Seq(JsNumber(1), JsString("a")), + Seq(JsNumber(1), JsString("b")), + Seq(JsNumber(2), JsString("c")) + ) + + // number descending, string ascending + val sortedJsLs: Seq[Seq[JsValue]] = Seq( + Seq(JsNumber(2), JsString("c")), + Seq(JsNumber(1), JsString("a")), + Seq(JsNumber(1), JsString("b")), + Seq(JsNumber(0), JsString("a")), + Seq(JsNumber(0), JsString("b")) + ) + + val orderParam: Seq[Boolean] = Seq(false, true) + val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] { + override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = { + val xe = x.iterator + val ye = y.iterator + val oe = orderParam.iterator + + while (xe.hasNext && ye.hasNext && oe.hasNext) { + val (xev, yev) = oe.next() match { + case true => xe.next() -> ye.next() + case false => ye.next() -> xe.next() + } + val res = (xev, yev) match { + case (JsNumber(xv), JsNumber(yv)) => + Ordering[BigDecimal].compare(xv, yv) + case (JsString(xv), JsString(yv)) => + Ordering[String].compare(xv, yv) + case _ => throw new Exception("type mismatch") + } + if (res != 0) return res + } + + Ordering.Boolean.compare(xe.hasNext, ye.hasNext) + } + }) + + resultJsLs.toString() must_== sortedJsLs.toString + } + + "test order by primitive type" >> { + val jsLs: Seq[Seq[Any]] = Seq( + Seq(0, "a"), + Seq(0, "b"), + Seq(1, "a"), + Seq(1, "b"), + Seq(2, "c") + ) + + // number descending, string ascending + val sortedJsLs: Seq[Seq[Any]] = Seq( + Seq(2, "c"), + Seq(1, "a"), + Seq(1, "b"), + Seq(0, "a"), + Seq(0, "b") + ) + + val ascendingLs: Seq[Boolean] = Seq(false, true) + val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs)) + + resultJsLs.toString() must_== sortedJsLs.toString + } + + "test order by primitive type with short ascending list" >> { + val jsLs: Seq[Seq[Any]] = Seq( + Seq(0, "a"), + Seq(1, "b"), + Seq(0, "b"), + Seq(1, "a"), + Seq(2, "c"), + Seq(1, "c"), + Seq(1, "d"), + Seq(1, "f"), + Seq(1, "e") + ) + + // number descending, string ascending(default) + val sortedJsLs: Seq[Seq[Any]] = Seq( + Seq(2, "c"), + Seq(1, "a"), + Seq(1, "b"), + Seq(1, "c"), + Seq(1, "d"), + Seq(1, "e"), + Seq(1, "f"), + Seq(0, "a"), + Seq(0, "b") + ) + + val ascendingLs: Seq[Boolean] = Seq(false) + val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs)) + + resultJsLs.toString() must_== sortedJsLs.toString + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/benchmark/BenchmarkCommon.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/benchmark/BenchmarkCommon.scala b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/BenchmarkCommon.scala new file mode 100644 index 0000000..9e220cf --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/BenchmarkCommon.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.benchmark + +import com.typesafe.config.{ConfigFactory, Config} +import org.apache.s2graph.core.{Management, Graph} +import org.specs2.mutable.Specification +import scalikejdbc.AutoSession + +import scala.concurrent.ExecutionContext + +trait BenchmarkCommon extends Specification { + val wrapStr = s"\n==================================================" + + def duration[T](prefix: String = "")(block: => T) = { + val startTs = System.currentTimeMillis() + val ret = block + val endTs = System.currentTimeMillis() + println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr") + ret + } + + def durationWithReturn[T](prefix: String = "")(block: => T): (T, Long) = { + val startTs = System.currentTimeMillis() + val ret = block + val endTs = System.currentTimeMillis() + val duration = endTs - startTs +// println(s"$wrapStr\n$prefix: took $duration ms$wrapStr") + (ret, duration) + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/benchmark/GraphUtilSpec.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/benchmark/GraphUtilSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/GraphUtilSpec.scala new file mode 100644 index 0000000..3cb216c --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/GraphUtilSpec.scala @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.benchmark + +import org.apache.hadoop.hbase.util.Bytes +import org.apache.s2graph.core.GraphUtil +import org.apache.s2graph.core.types.{HBaseType, InnerVal, SourceVertexId} + +import scala.collection.mutable +import scala.collection.mutable.ListBuffer + +class GraphUtilSpec extends BenchmarkCommon { + + def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean = + Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0 + + def betweenShort(value: Short, start: Short, end: Short): Boolean = + start <= value && value <= end + + + "GraphUtil" should { + "test murmur3 hash function distribution" in { + val testNum = 1000000 + val bucketSize = Short.MaxValue / 40 + val countsNew = new mutable.HashMap[Int, Int]() + val counts = new mutable.HashMap[Int, Int]() + for { + i <- (0 until testNum) + } { + val h = GraphUtil.murmur3(i.toString) / bucketSize + val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize + counts += (h -> (counts.getOrElse(h, 0) + 1)) + countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1)) + } + val all = counts.toList.sortBy { case (bucket, count) => count }.reverse + val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse + val top = all.take(10) + val bottom = all.takeRight(10) + val topNew = allNew.take(10) + val bottomNew = allNew.takeRight(10) + println(s"Top: $top") + println(s"Bottom: $bottom") + println("-" * 50) + println(s"TopNew: $topNew") + println(s"Bottom: $bottomNew") + true + } + + "test murmur hash skew2" in { + + import HBaseType._ + val testNum = 1000000L + val regionCount = 40 + val window = Int.MaxValue / regionCount + val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]() + for { + i <- (0 until regionCount) + } yield { + val startKey = Bytes.toBytes(i * window) + val endKey = Bytes.toBytes((i + 1) * window) + rangeBytes += (startKey.toList -> endKey.toList) + } + + + + val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]() + val counts = new collection.mutable.HashMap[Short, Long]() + stats += (0 -> (rangeBytes.head -> 0L)) + + for (i <- (0L until testNum)) { + val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION)) + val bytes = vertexId.bytes + val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString()) + val shortVal = counts.getOrElse(shortKey, 0L) + 1L + counts += (shortKey -> shortVal) + var j = 0 + var found = false + while (j < rangeBytes.size && !found) { + val (start, end) = rangeBytes(j) + if (between(bytes, start.toArray, end.toArray)) { + found = true + } + j += 1 + } + val head = rangeBytes(j - 1) + val key = j - 1 + val value = stats.get(key) match { + case None => 0L + case Some(v) => v._2 + 1 + } + stats += (key ->(head, value)) + } + val sorted = stats.toList.sortBy(kv => kv._2._2).reverse + println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") + sorted.foreach { case (idx, ((start, end), cnt)) => + val startShort = Bytes.toShort(start.take(2).toArray) + val endShort = Bytes.toShort(end.take(2).toArray) + val count = counts.count(t => startShort <= t._1 && t._1 < endShort) + println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") + + } + println("\n" * 10) + println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") + stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) => + val startShort = Bytes.toShort(start.take(2).toArray) + val endShort = Bytes.toShort(end.take(2).toArray) + val count = counts.count(t => startShort <= t._1 && t._1 < endShort) + println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") + + } + + true + } + + "Bytes compareTo" in { + val x = Array[Byte](11, -12, -26, -14, -23) + val startKey = Array[Byte](0, 0, 0, 0) + val endKey = Array[Byte](12, -52, -52, -52) + println(Bytes.compareTo(startKey, x)) + println(Bytes.compareTo(endKey, x)) + true + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/benchmark/JsonBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/benchmark/JsonBenchmarkSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/JsonBenchmarkSpec.scala new file mode 100644 index 0000000..8ba9ea2 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/JsonBenchmarkSpec.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.benchmark + +import play.api.libs.json.JsNumber +import play.libs.Json + +class JsonBenchmarkSpec extends BenchmarkCommon { + "to json" >> { + "json benchmark" >> { + + duration("map to json") { + (0 to 10) foreach { n => + val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap + Json.toJson(numberMaps) + } + } + + duration("directMakeJson") { + (0 to 10) foreach { n => + var jsObj = play.api.libs.json.Json.obj() + (0 to 10).foreach { n => + jsObj += (n.toString -> JsNumber(n * n)) + } + } + } + + duration("map to json 2") { + (0 to 50) foreach { n => + val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap + Json.toJson(numberMaps) + } + } + + duration("directMakeJson 2") { + (0 to 50) foreach { n => + var jsObj = play.api.libs.json.Json.obj() + (0 to 10).foreach { n => + jsObj += (n.toString -> JsNumber(n * n)) + } + } + } + true + } + true + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/benchmark/OrderingUtilBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/benchmark/OrderingUtilBenchmarkSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/OrderingUtilBenchmarkSpec.scala new file mode 100644 index 0000000..3cb0543 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/OrderingUtilBenchmarkSpec.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.benchmark + +import org.apache.s2graph.core.OrderingUtil._ +import org.apache.s2graph.core.{OrderingUtil, SeqMultiOrdering} +import play.api.libs.json.{JsNumber, JsValue} + +import scala.util.Random + +class OrderingUtilBenchmarkSpec extends BenchmarkCommon { + "OrderingUtilBenchmarkSpec" should { + + "performance MultiOrdering any" >> { + val tupLs = (0 until 10) map { i => + Random.nextDouble() -> Random.nextLong() + } + + val seqLs = tupLs.map { tup => + Seq(tup._1, tup._2) + } + + val sorted1 = duration("TupleOrdering double,long") { + (0 until 1000) foreach { _ => + tupLs.sortBy { case (x, y) => + -x -> -y + } + } + tupLs.sortBy { case (x, y) => + -x -> -y + } + }.map { x => x._1 } + + val sorted2 = duration("MultiOrdering double,long") { + (0 until 1000) foreach { _ => + seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false))) + } + seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false))) + }.map { x => x.head } + + sorted1.toString() must_== sorted2.toString() + } + + "performance MultiOrdering double" >> { + val tupLs = (0 until 50) map { i => + Random.nextDouble() -> Random.nextDouble() + } + + val seqLs = tupLs.map { tup => + Seq(tup._1, tup._2) + } + + duration("MultiOrdering double") { + (0 until 1000) foreach { _ => + seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false))) + } + } + + duration("TupleOrdering double") { + (0 until 1000) foreach { _ => + tupLs.sortBy { case (x, y) => + -x -> -y + } + } + } + + 1 must_== 1 + } + + "performance MultiOrdering jsvalue" >> { + val tupLs = (0 until 50) map { i => + Random.nextDouble() -> Random.nextLong() + } + + val seqLs = tupLs.map { tup => + Seq(JsNumber(tup._1), JsNumber(tup._2)) + } + + val sorted1 = duration("TupleOrdering double,long") { + (0 until 1000) foreach { _ => + tupLs.sortBy { case (x, y) => + -x -> -y + } + } + tupLs.sortBy { case (x, y) => + -x -> -y + } + } + + val sorted2 = duration("MultiOrdering jsvalue") { + (0 until 1000) foreach { _ => + seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false))) + } + seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false))) + } + + 1 must_== 1 + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2core/src/test/scala/org/apache/s2graph/core/benchmark/SamplingBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/benchmark/SamplingBenchmarkSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/SamplingBenchmarkSpec.scala new file mode 100644 index 0000000..a8777fb --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/benchmark/SamplingBenchmarkSpec.scala @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.core.benchmark + +import scala.annotation.tailrec +import scala.util.Random + +class SamplingBenchmarkSpec extends BenchmarkCommon { + "sample" should { + + "sample benchmark" in { + @tailrec + def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = { + if (set.size == n) set + else randomInt(n, range, set + Random.nextInt(range)) + } + + // sample using random array + def randomArraySample[T](num: Int, ls: List[T]): List[T] = { + val randomNum = randomInt(num, ls.size) + var sample = List.empty[T] + var idx = 0 + ls.foreach { e => + if (randomNum.contains(idx)) sample = e :: sample + idx += 1 + } + sample + } + + // sample using shuffle + def shuffleSample[T](num: Int, ls: List[T]): List[T] = { + Random.shuffle(ls).take(num) + } + + // sample using random number generation + def rngSample[T](num: Int, ls: List[T]): List[T] = { + var sampled = List.empty[T] + val N = ls.size // population + var t = 0 // total input records dealt with + var m = 0 // number of items selected so far + + while (m < num) { + val u = Random.nextDouble() + if ((N - t) * u < num - m) { + sampled = ls(t) :: sampled + m += 1 + } + t += 1 + } + sampled + } + + // test data + val testLimit = 10000 + val testNum = 10 + val testData = (0 to 1000).toList + + // dummy for warm-up + (0 to testLimit) foreach { n => + randomArraySample(testNum, testData) + shuffleSample(testNum, testData) + rngSample(testNum, testData) + } + + duration("Random Array Sampling") { + (0 to testLimit) foreach { _ => + val sampled = randomArraySample(testNum, testData) + } + } + + duration("Shuffle Sampling") { + (0 to testLimit) foreach { _ => + val sampled = shuffleSample(testNum, testData) + } + } + + duration("RNG Sampling") { + (0 to testLimit) foreach { _ => + val sampled = rngSample(testNum, testData) + } + } + true + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2counter_core/build.sbt ---------------------------------------------------------------------- diff --git a/s2counter_core/build.sbt b/s2counter_core/build.sbt index 28a368d..b248b43 100644 --- a/s2counter_core/build.sbt +++ b/s2counter_core/build.sbt @@ -25,11 +25,11 @@ scalacOptions ++= Seq("-feature", "-deprecation", "-language:existentials") scalacOptions in Test ++= Seq("-Yrangepos") -libraryDependencies ++= loggingRuntime ++ Seq( +libraryDependencies ++= Seq( "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "com.typesafe.play" %% "play-ws" % playVersion excludeLogging(), "org.apache.hadoop" % "hadoop-common" % hadoopVersion excludeLogging, "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion excludeLogging, "redis.clients" % "jedis" % "2.6.0", - "org.specs2" %% "specs2-core" % "3.8.5" % "test" + "org.specs2" %% "specs2-core" % specs2Version % "test" ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_netty/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_netty/build.sbt b/s2rest_netty/build.sbt index 422cfef..975ced5 100644 --- a/s2rest_netty/build.sbt +++ b/s2rest_netty/build.sbt @@ -22,7 +22,7 @@ name := "s2rest_netty" enablePlugins(JavaAppPackaging) -libraryDependencies ++= loggingRuntime ++ Seq( +libraryDependencies ++= Seq( "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility "io.netty" % "netty-all" % "4.0.33.Final" ) http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/build.sbt ---------------------------------------------------------------------- diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt index 77e13d4..7583ad9 100644 --- a/s2rest_play/build.sbt +++ b/s2rest_play/build.sbt @@ -22,9 +22,9 @@ name := "s2rest_play" scalacOptions in Test ++= Seq("-Yrangepos") -libraryDependencies := loggingRuntime ++ (libraryDependencies.value ++ Seq(ws, filters, specs2 % Test)).map(_.excludeLogging()) ++ Seq( - "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility - "org.specs2" %% "specs2-core" % "3.8.5" % "test" +libraryDependencies := (libraryDependencies.value ++ Seq(ws, filters, specs2 % Test)).map(_.excludeLogging()) ++ Seq( + "com.google.guava" % "guava" % "12.0.1" force() // use this old version of guava to avoid incompatibility + //"org.specs2" %% "specs2-core" % specs2Version % "test" ) routesGenerator := StaticRoutesGenerator http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala deleted file mode 100644 index 240421c..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.benchmark - -import com.typesafe.config.{ConfigFactory, Config} -import org.apache.s2graph.core.{Management, Graph} -import org.specs2.mutable.Specification -import scalikejdbc.AutoSession - -import scala.concurrent.ExecutionContext - -trait BenchmarkCommon extends Specification { - val wrapStr = s"\n==================================================" - - def duration[T](prefix: String = "")(block: => T) = { - val startTs = System.currentTimeMillis() - val ret = block - val endTs = System.currentTimeMillis() - println(s"$wrapStr\n$prefix: took ${endTs - startTs} ms$wrapStr") - ret - } - - def durationWithReturn[T](prefix: String = "")(block: => T): (T, Long) = { - val startTs = System.currentTimeMillis() - val ret = block - val endTs = System.currentTimeMillis() - val duration = endTs - startTs -// println(s"$wrapStr\n$prefix: took $duration ms$wrapStr") - (ret, duration) - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala deleted file mode 100644 index 737b828..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.benchmark - -import org.apache.hadoop.hbase.util.Bytes -import org.apache.s2graph.core.GraphUtil -import org.apache.s2graph.core.types.{HBaseType, InnerVal, SourceVertexId} -import play.api.test.PlaySpecification - -import scala.collection.mutable -import scala.collection.mutable.ListBuffer - -class GraphUtilSpec extends BenchmarkCommon with PlaySpecification { - - def between(bytes: Array[Byte], startKey: Array[Byte], endKey: Array[Byte]): Boolean = - Bytes.compareTo(startKey, bytes) <= 0 && Bytes.compareTo(endKey, bytes) >= 0 - - def betweenShort(value: Short, start: Short, end: Short): Boolean = - start <= value && value <= end - - - "GraphUtil" should { - "test murmur3 hash function distribution" in { - val testNum = 1000000 - val bucketSize = Short.MaxValue / 40 - val countsNew = new mutable.HashMap[Int, Int]() - val counts = new mutable.HashMap[Int, Int]() - for { - i <- (0 until testNum) - } { - val h = GraphUtil.murmur3(i.toString) / bucketSize - val hNew = GraphUtil.murmur3Int(i.toString) / bucketSize - counts += (h -> (counts.getOrElse(h, 0) + 1)) - countsNew += (hNew -> (countsNew.getOrElse(hNew, 0) + 1)) - } - val all = counts.toList.sortBy { case (bucket, count) => count }.reverse - val allNew = countsNew.toList.sortBy { case (bucket, count) => count }.reverse - val top = all.take(10) - val bottom = all.takeRight(10) - val topNew = allNew.take(10) - val bottomNew = allNew.takeRight(10) - println(s"Top: $top") - println(s"Bottom: $bottom") - println("-" * 50) - println(s"TopNew: $topNew") - println(s"Bottom: $bottomNew") - true - } - - "test murmur hash skew2" in { - - import HBaseType._ - val testNum = 1000000L - val regionCount = 40 - val window = Int.MaxValue / regionCount - val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]() - for { - i <- (0 until regionCount) - } yield { - val startKey = Bytes.toBytes(i * window) - val endKey = Bytes.toBytes((i + 1) * window) - rangeBytes += (startKey.toList -> endKey.toList) - } - - - - val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]() - val counts = new collection.mutable.HashMap[Short, Long]() - stats += (0 -> (rangeBytes.head -> 0L)) - - for (i <- (0L until testNum)) { - val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION)) - val bytes = vertexId.bytes - val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString()) - val shortVal = counts.getOrElse(shortKey, 0L) + 1L - counts += (shortKey -> shortVal) - var j = 0 - var found = false - while (j < rangeBytes.size && !found) { - val (start, end) = rangeBytes(j) - if (between(bytes, start.toArray, end.toArray)) { - found = true - } - j += 1 - } - val head = rangeBytes(j - 1) - val key = j - 1 - val value = stats.get(key) match { - case None => 0L - case Some(v) => v._2 + 1 - } - stats += (key ->(head, value)) - } - val sorted = stats.toList.sortBy(kv => kv._2._2).reverse - println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") - sorted.foreach { case (idx, ((start, end), cnt)) => - val startShort = Bytes.toShort(start.take(2).toArray) - val endShort = Bytes.toShort(end.take(2).toArray) - val count = counts.count(t => startShort <= t._1 && t._1 < endShort) - println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") - - } - println("\n" * 10) - println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") - stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) => - val startShort = Bytes.toShort(start.take(2).toArray) - val endShort = Bytes.toShort(end.take(2).toArray) - val count = counts.count(t => startShort <= t._1 && t._1 < endShort) - println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") - - } - - true - } - - "Bytes compareTo" in { - val x = Array[Byte](11, -12, -26, -14, -23) - val startKey = Array[Byte](0, 0, 0, 0) - val endKey = Array[Byte](12, -52, -52, -52) - println(Bytes.compareTo(startKey, x)) - println(Bytes.compareTo(endKey, x)) - true - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala deleted file mode 100644 index 5733f19..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/JsonBenchmarkSpec.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.benchmark - -import play.api.libs.json.JsNumber -import play.libs.Json - -class JsonBenchmarkSpec extends BenchmarkCommon { - "to json" >> { - "json benchmark" >> { - - duration("map to json") { - (0 to 10) foreach { n => - val numberMaps = (0 to 100).map { n => (n.toString -> JsNumber(n * n)) }.toMap - Json.toJson(numberMaps) - } - } - - duration("directMakeJson") { - (0 to 10) foreach { n => - var jsObj = play.api.libs.json.Json.obj() - (0 to 10).foreach { n => - jsObj += (n.toString -> JsNumber(n * n)) - } - } - } - - duration("map to json 2") { - (0 to 50) foreach { n => - val numberMaps = (0 to 10).map { n => (n.toString -> JsNumber(n * n)) }.toMap - Json.toJson(numberMaps) - } - } - - duration("directMakeJson 2") { - (0 to 50) foreach { n => - var jsObj = play.api.libs.json.Json.obj() - (0 to 10).foreach { n => - jsObj += (n.toString -> JsNumber(n * n)) - } - } - } - true - } - true - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala deleted file mode 100644 index fb80dc4..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/OrderingUtilBenchmarkSpec.scala +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.benchmark - -import org.apache.s2graph.core.OrderingUtil._ -import org.apache.s2graph.core.{OrderingUtil, SeqMultiOrdering} -import play.api.libs.json.{JsNumber, JsValue} - -import scala.util.Random - -class OrderingUtilBenchmarkSpec extends BenchmarkCommon { - "OrderingUtilBenchmarkSpec" should { - - "performance MultiOrdering any" >> { - val tupLs = (0 until 10) map { i => - Random.nextDouble() -> Random.nextLong() - } - - val seqLs = tupLs.map { tup => - Seq(tup._1, tup._2) - } - - val sorted1 = duration("TupleOrdering double,long") { - (0 until 1000) foreach { _ => - tupLs.sortBy { case (x, y) => - -x -> -y - } - } - tupLs.sortBy { case (x, y) => - -x -> -y - } - }.map { x => x._1 } - - val sorted2 = duration("MultiOrdering double,long") { - (0 until 1000) foreach { _ => - seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false))) - } - seqLs.sorted(new SeqMultiOrdering[Any](Seq(false, false))) - }.map { x => x.head } - - sorted1.toString() must_== sorted2.toString() - } - - "performance MultiOrdering double" >> { - val tupLs = (0 until 50) map { i => - Random.nextDouble() -> Random.nextDouble() - } - - val seqLs = tupLs.map { tup => - Seq(tup._1, tup._2) - } - - duration("MultiOrdering double") { - (0 until 1000) foreach { _ => - seqLs.sorted(new SeqMultiOrdering[Double](Seq(false, false))) - } - } - - duration("TupleOrdering double") { - (0 until 1000) foreach { _ => - tupLs.sortBy { case (x, y) => - -x -> -y - } - } - } - - 1 must_== 1 - } - - "performance MultiOrdering jsvalue" >> { - val tupLs = (0 until 50) map { i => - Random.nextDouble() -> Random.nextLong() - } - - val seqLs = tupLs.map { tup => - Seq(JsNumber(tup._1), JsNumber(tup._2)) - } - - val sorted1 = duration("TupleOrdering double,long") { - (0 until 1000) foreach { _ => - tupLs.sortBy { case (x, y) => - -x -> -y - } - } - tupLs.sortBy { case (x, y) => - -x -> -y - } - } - - val sorted2 = duration("MultiOrdering jsvalue") { - (0 until 1000) foreach { _ => - seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false))) - } - seqLs.sorted(new SeqMultiOrdering[JsValue](Seq(false, false))) - } - - 1 must_== 1 - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala deleted file mode 100644 index 07e0b13..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.benchmark - -import play.api.test.{FakeApplication, PlaySpecification, WithApplication} - -import scala.annotation.tailrec -import scala.util.Random - -class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification { - "sample" should { - - "sample benchmark" in { - @tailrec - def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = { - if (set.size == n) set - else randomInt(n, range, set + Random.nextInt(range)) - } - - // sample using random array - def randomArraySample[T](num: Int, ls: List[T]): List[T] = { - val randomNum = randomInt(num, ls.size) - var sample = List.empty[T] - var idx = 0 - ls.foreach { e => - if (randomNum.contains(idx)) sample = e :: sample - idx += 1 - } - sample - } - - // sample using shuffle - def shuffleSample[T](num: Int, ls: List[T]): List[T] = { - Random.shuffle(ls).take(num) - } - - // sample using random number generation - def rngSample[T](num: Int, ls: List[T]): List[T] = { - var sampled = List.empty[T] - val N = ls.size // population - var t = 0 // total input records dealt with - var m = 0 // number of items selected so far - - while (m < num) { - val u = Random.nextDouble() - if ((N - t) * u < num - m) { - sampled = ls(t) :: sampled - m += 1 - } - t += 1 - } - sampled - } - - // test data - val testLimit = 10000 - val testNum = 10 - val testData = (0 to 1000).toList - - // dummy for warm-up - (0 to testLimit) foreach { n => - randomArraySample(testNum, testData) - shuffleSample(testNum, testData) - rngSample(testNum, testData) - } - - duration("Random Array Sampling") { - (0 to testLimit) foreach { _ => - val sampled = randomArraySample(testNum, testData) - } - } - - duration("Shuffle Sampling") { - (0 to testLimit) foreach { _ => - val sampled = shuffleSample(testNum, testData) - } - } - - duration("RNG Sampling") { - (0 to testLimit) foreach { _ => - val sampled = rngSample(testNum, testData) - } - } - true - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala ---------------------------------------------------------------------- diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala deleted file mode 100644 index adc3b02..0000000 --- a/s2rest_play/test/org/apache/s2graph/rest/play/controllers/PostProcessSpec.scala +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.s2graph.rest.play.controllers - -import org.apache.s2graph.core.{OrderingUtil, SeqMultiOrdering} -import play.api.libs.json.{JsNumber, JsString, JsValue} -import play.api.test.PlaySpecification - -class PostProcessSpec extends PlaySpecification { - import OrderingUtil._ - - "test order by json" >> { - val jsLs: Seq[Seq[JsValue]] = Seq( - Seq(JsNumber(0), JsString("a")), - Seq(JsNumber(0), JsString("b")), - Seq(JsNumber(1), JsString("a")), - Seq(JsNumber(1), JsString("b")), - Seq(JsNumber(2), JsString("c")) - ) - - // number descending, string ascending - val sortedJsLs: Seq[Seq[JsValue]] = Seq( - Seq(JsNumber(2), JsString("c")), - Seq(JsNumber(1), JsString("a")), - Seq(JsNumber(1), JsString("b")), - Seq(JsNumber(0), JsString("a")), - Seq(JsNumber(0), JsString("b")) - ) - - val orderParam: Seq[Boolean] = Seq(false, true) - val resultJsLs = jsLs.sorted(new Ordering[Seq[JsValue]] { - override def compare(x: Seq[JsValue], y: Seq[JsValue]): Int = { - val xe = x.iterator - val ye = y.iterator - val oe = orderParam.iterator - - while (xe.hasNext && ye.hasNext && oe.hasNext) { - val (xev, yev) = oe.next() match { - case true => xe.next() -> ye.next() - case false => ye.next() -> xe.next() - } - val res = (xev, yev) match { - case (JsNumber(xv), JsNumber(yv)) => - Ordering[BigDecimal].compare(xv, yv) - case (JsString(xv), JsString(yv)) => - Ordering[String].compare(xv, yv) - case _ => throw new Exception("type mismatch") - } - if (res != 0) return res - } - - Ordering.Boolean.compare(xe.hasNext, ye.hasNext) - } - }) - - resultJsLs.toString() must_== sortedJsLs.toString - } - - "test order by primitive type" >> { - val jsLs: Seq[Seq[Any]] = Seq( - Seq(0, "a"), - Seq(0, "b"), - Seq(1, "a"), - Seq(1, "b"), - Seq(2, "c") - ) - - // number descending, string ascending - val sortedJsLs: Seq[Seq[Any]] = Seq( - Seq(2, "c"), - Seq(1, "a"), - Seq(1, "b"), - Seq(0, "a"), - Seq(0, "b") - ) - - val ascendingLs: Seq[Boolean] = Seq(false, true) - val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs)) - - resultJsLs.toString() must_== sortedJsLs.toString - } - - "test order by primitive type with short ascending list" >> { - val jsLs: Seq[Seq[Any]] = Seq( - Seq(0, "a"), - Seq(1, "b"), - Seq(0, "b"), - Seq(1, "a"), - Seq(2, "c"), - Seq(1, "c"), - Seq(1, "d"), - Seq(1, "f"), - Seq(1, "e") - ) - - // number descending, string ascending(default) - val sortedJsLs: Seq[Seq[Any]] = Seq( - Seq(2, "c"), - Seq(1, "a"), - Seq(1, "b"), - Seq(1, "c"), - Seq(1, "d"), - Seq(1, "e"), - Seq(1, "f"), - Seq(0, "a"), - Seq(0, "b") - ) - - val ascendingLs: Seq[Boolean] = Seq(false) - val resultJsLs = jsLs.sorted(new SeqMultiOrdering[Any](ascendingLs)) - - resultJsLs.toString() must_== sortedJsLs.toString - } -} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5b5b4334/spark/build.sbt ---------------------------------------------------------------------- diff --git a/spark/build.sbt b/spark/build.sbt index 1ab6ecb..14a673e 100644 --- a/spark/build.sbt +++ b/spark/build.sbt @@ -17,16 +17,18 @@ * under the License. */ +import Common._ + name := "s2spark" scalacOptions ++= Seq("-deprecation", "-feature") libraryDependencies ++= Seq( "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility - "org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided", - "org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided", - "org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion, - "com.typesafe.play" %% "play-json" % Common.playVersion, - "org.specs2" %% "specs2-core" % "3.8.5" % "test", + "org.apache.spark" %% "spark-core" % sparkVersion % "provided", + "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided", + "org.apache.spark" %% "spark-streaming-kafka" % sparkVersion, + "com.typesafe.play" %% "play-json" % playVersion, + "org.specs2" %% "specs2-core" % specs2Version % "test", "org.scalatest" %% "scalatest" % "2.2.1" % "test" )
