Repository: incubator-predictionio Updated Branches: refs/heads/develop b9db14535 -> ada0591f2
Improve insert performance for ES storage Closes #373 Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/ada0591f Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/ada0591f Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/ada0591f Branch: refs/heads/develop Commit: ada0591f2afe617dc212e06fafbc2286beb190a6 Parents: b9db145 Author: Shinsuke Sugaya <[email protected]> Authored: Tue Apr 25 16:25:16 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Tue Apr 25 16:25:16 2017 -0700 ---------------------------------------------------------------------- .../predictionio/data/storage/elasticsearch/ESLEvents.scala | 2 +- .../predictionio/data/storage/elasticsearch/ESSequences.scala | 6 ++++-- tests/pio_tests/engines/recommendation-engine/build.sbt | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ada0591f/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala ---------------------------------------------------------------------- diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala index f379bb8..18e2fed 100644 --- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala +++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala @@ -74,7 +74,7 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St ("entityId" -> ("type" -> "keyword")) ~ ("targetEntityType" -> ("type" -> "keyword")) ~ ("targetEntityId" -> ("type" -> "keyword")) ~ - ("properties" -> ("type" -> "keyword")) ~ + ("properties" -> ("enabled" -> 0)) ~ ("eventTime" -> ("type" -> "date")) ~ ("tags" -> ("type" -> "keyword")) ~ ("prId" -> ("type" -> "keyword")) ~ http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ada0591f/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala ---------------------------------------------------------------------- diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala index 29c8c33..8a6a047 100644 --- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala +++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala @@ -46,7 +46,9 @@ class ESSequences(client: ESClient, config: StorageClientConfig, index: String) ESUtils.getNumberOfReplicas(config, index.toUpperCase)) val mappingJson = (estype -> - ("_all" -> ("enabled" -> 0))) + ("_all" -> ("enabled" -> 0)) ~ + ("properties" -> + ("n" -> ("enabled" -> 0)))) ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson))) } finally { restClient.close() @@ -59,7 +61,7 @@ class ESSequences(client: ESClient, config: StorageClientConfig, index: String) val response = restClient.performRequest( "POST", s"/$index/$estype/$name", - Map("refresh" -> "true").asJava, + Map("refresh" -> "false").asJava, entity) val jsonResponse = parse(EntityUtils.toString(response.getEntity)) val result = (jsonResponse \ "result").extract[String] http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ada0591f/tests/pio_tests/engines/recommendation-engine/build.sbt ---------------------------------------------------------------------- diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt index 5785f39..e9e2436 100644 --- a/tests/pio_tests/engines/recommendation-engine/build.sbt +++ b/tests/pio_tests/engines/recommendation-engine/build.sbt @@ -26,6 +26,6 @@ name := "template-scala-parallel-recommendation" organization := "org.apache.predictionio" libraryDependencies ++= Seq( - "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided", + "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.1-SNAPSHOT" % "provided", "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided", "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided")
