This is an automated email from the ASF dual-hosted git repository. markusthoemmes pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new bcba2d1 Allow Elasticsearch client to skip query result hits. (#3849) bcba2d1 is described below commit bcba2d1d2228e17c5bdb6e9d3cf666fd946b9216 Author: James Dubee <jwdu...@us.ibm.com> AuthorDate: Thu Jul 12 03:55:53 2018 -0400 Allow Elasticsearch client to skip query result hits. (#3849) --- .../logging/ElasticSearchRestClient.scala | 11 ++----- .../logging/ElasticSearchLogStoreTests.scala | 3 +- .../logging/ElasticSearchRestClientTests.scala | 36 +++++++++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala index 0290e96..9218958 100644 --- a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala +++ b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala @@ -57,7 +57,6 @@ case class EsQueryAggs(aggField: String, agg: EsAgg, field: String) case class EsQueryRange(key: String, range: EsRange, value: String) case class EsQueryBoolMatch(key: String, value: String) case class EsQueryOrder(field: String, kind: EsOrder) -case class EsQuerySize(size: Integer) case class EsQueryAll() extends EsQueryMethod case class EsQueryMust(matches: Vector[EsQueryBoolMatch], range: Vector[EsQueryRange] = Vector.empty) extends EsQueryMethod @@ -66,7 +65,8 @@ case class EsQueryTerm(key: String, value: String) extends EsQueryMethod case class EsQueryString(queryString: String) extends EsQueryMethod case class EsQuery(query: EsQueryMethod, sort: Option[EsQueryOrder] = None, - size: Option[EsQuerySize] = None, + size: Option[Int] = None, + from: Int = 0, aggs: Option[EsQueryAggs] = None) // Schema of ES query results @@ -121,11 +121,6 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol { JsArray(JsObject(query.field -> JsObject("order" -> query.kind.toString.toJson))) } - implicit object EsQuerySizeJsonFormat extends RootJsonFormat[EsQuerySize] { - def read(query: JsValue) = ??? - def write(query: EsQuerySize) = JsNumber(query.size) - } - implicit object EsQueryAggsJsonFormat extends RootJsonFormat[EsQueryAggs] { def read(query: JsValue) = ??? def write(query: EsQueryAggs) = @@ -148,7 +143,7 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol { } } - implicit val esQueryFormat = jsonFormat4(EsQuery.apply) + implicit val esQueryFormat = jsonFormat5(EsQuery.apply) implicit val esSearchHitFormat = jsonFormat(EsSearchHit.apply _, "_source") implicit val esSearchHitsFormat = jsonFormat2(EsSearchHits.apply) implicit val esSearchResultFormat = jsonFormat1(EsSearchResult.apply) diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala index bd3e0a0..91db374 100644 --- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala +++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala @@ -79,7 +79,8 @@ class ElasticSearchLogStoreTests "query" -> JsObject( "query_string" -> JsObject("query" -> JsString( s"_type: ${defaultConfig.logSchema.userLogs} AND ${defaultConfig.logSchema.activationId}: $activationId"))), - "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" -> JsString("asc"))))).compactPrint + "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" -> JsString("asc")))), + "from" -> JsNumber(0)).compactPrint private val defaultHttpRequest = HttpRequest( POST, Uri(s"/whisk_user_logs/_search"), diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala index 040b0fc..2a43b79 100644 --- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala +++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala @@ -87,7 +87,8 @@ class ElasticSearchRestClientTests "must" -> JsArray( JsObject("match" -> JsObject("someKey1" -> JsString("someValue1"))), - JsObject("match" -> JsObject("someKey2" -> JsString("someValue2"))))))) + JsObject("match" -> JsObject("someKey2" -> JsString("someValue2")))))), + "from" -> 0.toJson) // Test must with ranges Seq((EsRangeGte, "gte"), (EsRangeGt, "gt"), (EsRangeLte, "lte"), (EsRangeLt, "lt")).foreach { @@ -109,7 +110,8 @@ class ElasticSearchRestClientTests "filter" -> JsArray( JsObject("range" -> JsObject("someKey1" -> JsObject(rangeValue -> "someValue1".toJson))), - JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue -> "someValue2".toJson))))))) + JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue -> "someValue2".toJson)))))), + "from" -> 0.toJson) } } @@ -120,7 +122,8 @@ class ElasticSearchRestClientTests EsQuery(EsQueryAll(), aggs = Some(queryAgg)).toJson shouldBe JsObject( "query" -> JsObject("match_all" -> JsObject.empty), - "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field" -> "someField".toJson)))) + "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field" -> "someField".toJson))), + "from" -> 0.toJson) } } @@ -128,7 +131,8 @@ class ElasticSearchRestClientTests val queryMatch = EsQueryMatch("someField", "someValue") EsQuery(queryMatch).toJson shouldBe JsObject( - "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson)))) + "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson))), + "from" -> 0.toJson) // Test match with types Seq((EsMatchPhrase, "phrase"), (EsMatchPhrasePrefix, "phrase_prefix")).foreach { @@ -137,21 +141,25 @@ class ElasticSearchRestClientTests EsQuery(queryMatch).toJson shouldBe JsObject( "query" -> JsObject( - "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson, "type" -> typeValue.toJson)))) + "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson, "type" -> typeValue.toJson))), + "from" -> 0.toJson) } } it should "construct a query with term" in { val queryTerm = EsQueryTerm("user", "someUser") - EsQuery(queryTerm).toJson shouldBe JsObject("query" -> JsObject("term" -> JsObject("user" -> JsString("someUser")))) + EsQuery(queryTerm).toJson shouldBe JsObject( + "query" -> JsObject("term" -> JsObject("user" -> JsString("someUser"))), + "from" -> 0.toJson) } it should "construct a query with query string" in { val queryString = EsQueryString("_type: someType") EsQuery(queryString).toJson shouldBe JsObject( - "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type: someType")))) + "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type: someType"))), + "from" -> 0.toJson) } it should "construct a query with order" in { @@ -161,16 +169,22 @@ class ElasticSearchRestClientTests EsQuery(EsQueryAll(), Some(queryOrder)).toJson shouldBe JsObject( "query" -> JsObject("match_all" -> JsObject.empty), - "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson)))) + "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson))), + "from" -> 0.toJson) } } it should "construct query with size" in { - val querySize = EsQuerySize(1) + EsQuery(EsQueryAll(), size = Some(1)).toJson shouldBe JsObject( + "query" -> JsObject("match_all" -> JsObject.empty), + "size" -> 1.toJson, + "from" -> 0.toJson) + } - EsQuery(EsQueryAll(), size = Some(querySize)).toJson shouldBe JsObject( + it should "construct query with from" in { + EsQuery(EsQueryAll(), from = 1).toJson shouldBe JsObject( "query" -> JsObject("match_all" -> JsObject.empty), - "size" -> JsNumber(1)) + "from" -> 1.toJson) } it should "error when search response does not match expected type" in {