This is an automated email from the ASF dual-hosted git repository. cbickel pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 20a5a60 Enable new views. (#3155) 20a5a60 is described below commit 20a5a6083d1630037740b4cf62f0d4e8fbc5c829 Author: rodric rabbah <rod...@gmail.com> AuthorDate: Tue Jan 16 02:09:31 2018 -0500 Enable new views. (#3155) Support ?count to retrieve document count for a collection. Use public-package view for listing package in other namespaces. Fix logs playbook for new view. Update wskadmin to set reduce=false. Use Scalatest theSameElementsAs for comparing List responses. Remove old views. --- ...ign_document_for_activations_db_filters_v2.json | 9 -- ...isks_design_document_for_activations_db_v2.json | 9 -- .../whisks_design_document_for_entities_db_v2.json | 21 ----- ansible/logs.yml | 2 +- ansible/tasks/recreateViews.yml | 7 +- common/scala/src/main/resources/application.conf | 6 +- .../scala/whisk/core/database/ArtifactStore.scala | 16 ++++ .../whisk/core/database/CouchDbRestStore.scala | 99 +++++++++++++++------- .../main/scala/whisk/core/entity/WhiskAction.scala | 20 ++--- .../scala/whisk/core/entity/WhiskActivation.scala | 2 +- .../main/scala/whisk/core/entity/WhiskEntity.scala | 7 +- .../scala/whisk/core/entity/WhiskPackage.scala | 12 ++- .../main/scala/whisk/core/entity/WhiskStore.scala | 48 ++++++++--- .../main/scala/whisk/core/controller/Actions.scala | 22 ++--- .../scala/whisk/core/controller/Activations.scala | 19 +++-- .../scala/whisk/core/controller/ApiUtils.scala | 52 +++++------- .../scala/whisk/core/controller/Entities.scala | 10 +-- .../scala/whisk/core/controller/Packages.scala | 38 ++++----- .../main/scala/whisk/core/controller/Rules.scala | 16 ++-- .../scala/whisk/core/controller/Triggers.scala | 17 ++-- .../core/controller/test/ActionsApiTests.scala | 12 +-- .../core/controller/test/ActivationsApiTests.scala | 63 +++++++------- .../core/controller/test/PackagesApiTests.scala | 75 ++++++++++------ .../whisk/core/controller/test/RulesApiTests.scala | 18 ++-- .../core/controller/test/TriggersApiTests.scala | 12 +-- .../SequenceActionApiMigrationTests.scala | 5 +- .../scala/whisk/core/database/test/DbUtils.scala | 14 +-- tools/admin/wskadmin | 2 +- 28 files changed, 333 insertions(+), 300 deletions(-) diff --git a/ansible/files/whisks_design_document_for_activations_db_filters_v2.json b/ansible/files/whisks_design_document_for_activations_db_filters_v2.json deleted file mode 100644 index 14eb0b0..0000000 --- a/ansible/files/whisks_design_document_for_activations_db_filters_v2.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "_id": "_design/whisks-filters.v2", - "language": "javascript", - "views": { - "activations": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n var isActivation = function (doc) { return (doc.activationId !== undefined) };\n var summarize = function (doc) {\n var endtime = doc.end !== 0 ? doc.end : undefined;\n return {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations,\n activationId: doc.activationId,\n start: doc.start,\n end: endtim [...] - } - } -} diff --git a/ansible/files/whisks_design_document_for_activations_db_v2.json b/ansible/files/whisks_design_document_for_activations_db_v2.json deleted file mode 100644 index be57879..0000000 --- a/ansible/files/whisks_design_document_for_activations_db_v2.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "_id": "_design/whisks.v2", - "language": "javascript", - "views": { - "activations": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n var isActivation = function (doc) { return (doc.activationId !== undefined) };\n var summarize = function (doc) {\n var endtime = doc.end !== 0 ? doc.end : undefined;\n return {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations,\n activationId: doc.activationId,\n start: doc.start,\n end: endtim [...] - } - } -} diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.json b/ansible/files/whisks_design_document_for_entities_db_v2.json deleted file mode 100644 index 97ed91c..0000000 --- a/ansible/files/whisks_design_document_for_entities_db_v2.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "_id": "_design/whisks.v2", - "language": "javascript", - "views": { - "rules": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n var isRule = function (doc) { return (doc.trigger !== undefined) };\n if (isRule(doc)) try {\n var ns = doc.namespace.split(PATHSEP);\n var root = ns[0];\n var date = doc.updated;\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations\n };\n emit([doc.namespace, date], value);\n if (root !== doc.nam [...] - }, - "all": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n\n var isPackage = function (doc) { return (doc.binding !== undefined) };\n var isAction = function (doc) { return (doc.exec !== undefined) };\n var isTrigger = function (doc) { return (doc.exec === undefined && doc.binding === undefined && doc.parameters !== undefined) };\n var isRule = function (doc) { return (doc.trigger !== undefined) };\n \n var collection = function (doc) {\n if (isPackage(doc)) return \"packages\"; [...] - }, - "packages": { - "map": "function (doc) {\n var isPackage = function (doc) { return (doc.binding !== undefined) };\n if (isPackage(doc)) try {\n var date = doc.updated;\n emit([doc.namespace, date], {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations,\n binding: Object.keys(doc.binding).length !== 0\n });\n } catch (e) {}\n}" - }, - "actions": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var ns = doc.namespace.split(PATHSEP);\n var root = ns[0];\n var date = doc.updated;\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations\n };\n emit([doc.namespace, date], value);\n if (root !== doc.nam [...] - }, - "triggers": { - "map": "function (doc) {\n var PATHSEP = \"/\";\n var isTrigger = function (doc) { return (doc.exec === undefined && doc.binding === undefined && doc.parameters !== undefined) };\n if (isTrigger(doc)) try {\n var ns = doc.namespace.split(PATHSEP);\n var root = ns[0];\n var date = doc.updated;\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations\n };\n [...] - } - } -} \ No newline at end of file diff --git a/ansible/logs.yml b/ansible/logs.yml index 3dc1b1e..545e69b 100644 --- a/ansible/logs.yml +++ b/ansible/logs.yml @@ -10,7 +10,7 @@ - name: create "logs" folder file: path="{{ openwhisk_home }}/logs" state=directory - name: dump entity views - local_action: shell "{{ openwhisk_home }}/bin/wskadmin" db get whisks --docs --view whisks.v2/{{ item }} | tail -n +2 > "{{ openwhisk_home }}/logs/db-{{ item }}.log" + local_action: shell "{{ openwhisk_home }}/bin/wskadmin" db get whisks --docs --view whisks.v2.1.0/{{ item }} | tail -n +2 > "{{ openwhisk_home }}/logs/db-{{ item }}.log" with_items: - actions - triggers diff --git a/ansible/tasks/recreateViews.yml b/ansible/tasks/recreateViews.yml index a5b6048..7868f85 100644 --- a/ansible/tasks/recreateViews.yml +++ b/ansible/tasks/recreateViews.yml @@ -6,7 +6,8 @@ dbName: "{{ db.whisk.actions }}" doc: "{{ lookup('file', '{{ item }}') }}" with_items: - - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_entities_db_v2.json" + - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json" + - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_all_entities_db_v2.1.0.json" - "{{ openwhisk_home }}/ansible/files/filter_design_document.json" - include: db/recreateDoc.yml @@ -14,8 +15,8 @@ dbName: "{{ db.whisk.activations }}" doc: "{{ lookup('file', '{{ item }}') }}" with_items: - - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_activations_db_v2.json" - - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_activations_db_filters_v2.json" + - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_activations_db_v2.1.0.json" + - "{{ openwhisk_home }}/ansible/files/whisks_design_document_for_activations_db_filters_v2.1.0.json" - "{{ openwhisk_home }}/ansible/files/filter_design_document.json" - "{{ openwhisk_home }}/ansible/files/activations_design_document_for_activations_db.json" - "{{ openwhisk_home }}/ansible/files/logCleanup_design_document_for_activations_db.json" diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf index 7de4b3e..0288106 100644 --- a/common/scala/src/main/resources/application.conf +++ b/common/scala/src/main/resources/application.conf @@ -78,9 +78,9 @@ whisk { } # db related configuration db { - actions-ddoc = "whisks.v2" - activations-ddoc = "whisks.v2" - activations-filter-ddoc = "whisks-filters.v2" + actions-ddoc = "whisks.v2.1.0" + activations-ddoc = "whisks.v2.1.0" + activations-filter-ddoc = "whisks-filters.v2.1.0" } # transaction ID related configuration transactions { diff --git a/common/scala/src/main/scala/whisk/core/database/ArtifactStore.scala b/common/scala/src/main/scala/whisk/core/database/ArtifactStore.scala index 2a61730..33ea805 100644 --- a/common/scala/src/main/scala/whisk/core/database/ArtifactStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/ArtifactStore.scala @@ -88,6 +88,7 @@ trait ArtifactStore[DocumentAbstraction] { * @param includeDocs include full documents matching query iff true (shall not be used with reduce) * @param descending reverse results iff true * @param reduce apply reduction associated with query to the result iff true + * @param stale a flag to permit a stale view result to be returned * @param transid the transaction id for logging * @return a future that completes with List[JsObject] of all documents from view between start and end key (list may be empty) */ @@ -102,6 +103,21 @@ trait ArtifactStore[DocumentAbstraction] { stale: StaleParameter)(implicit transid: TransactionId): Future[List[JsObject]] /** + * Counts all documents from database view that match a start key, up to an end key, using a future. + * If the operation is successful, the promise completes with Long. + * + * @param table the name of the table to query + * @param startKey to starting key to query the view for + * @param endKey to starting key to query the view for + * @param skip the number of record to skip (for pagination) + * @param stale a flag to permit a stale view result to be returned + * @param transid the transaction id for logging + * @return a future that completes with Long that is the number of documents from view between start and end key (count may be zero) + */ + protected[core] def count(table: String, startKey: List[Any], endKey: List[Any], skip: Int, stale: StaleParameter)( + implicit transid: TransactionId): Future[Long] + + /** * Attaches a "file" of type `contentType` to an existing document. The revision for the document must be set. */ protected[core] def attach(doc: DocInfo, name: String, contentType: ContentType, docStream: Source[ByteString, _])( diff --git a/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala b/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala index 7625e22..6832bc3 100644 --- a/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala +++ b/common/scala/src/main/scala/whisk/core/database/CouchDbRestStore.scala @@ -288,40 +288,40 @@ class CouchDbRestStore[DocumentAbstraction <: DocumentSerializer](dbProtocol: St (startKey, endKey) } - val parts = table.split("/") + val Array(firstPart, secondPart) = table.split("/") val start = transid.started(this, LoggingMarkers.DATABASE_QUERY, s"[QUERY] '$dbName' searching '$table") - val f = for (eitherResponse <- client.executeView(parts(0), parts(1))( - startKey = realStartKey, - endKey = realEndKey, - skip = Some(skip), - limit = Some(limit), - stale = stale, - includeDocs = includeDocs, - descending = descending, - reduce = reduce)) - yield - eitherResponse match { - case Right(response) => - val rows = response.fields("rows").convertTo[List[JsObject]] - - val out = if (reduce && !rows.isEmpty) { - assert(rows.length == 1, s"result of reduced view contains more than one value: '$rows'") - rows.head.fields("value").convertTo[List[JsObject]] - } else if (reduce) { - List(JsObject()) - } else { - rows - } - - transid.finished(this, start, s"[QUERY] '$dbName' completed: matched ${out.size}") - out - - case Left(code) => - transid.failed(this, start, s"Unexpected http response code: $code", ErrorLevel) - throw new Exception("Unexpected http response code: " + code) - } + val f = client + .executeView(firstPart, secondPart)( + startKey = realStartKey, + endKey = realEndKey, + skip = Some(skip), + limit = Some(limit), + stale = stale, + includeDocs = includeDocs, + descending = descending, + reduce = reduce) + .map { + case Right(response) => + val rows = response.fields("rows").convertTo[List[JsObject]] + + val out = if (reduce && !rows.isEmpty) { + assert(rows.length == 1, s"result of reduced view contains more than one value: '$rows'") + rows.head.fields("value").convertTo[List[JsObject]] + } else if (reduce) { + List(JsObject()) + } else { + rows + } + + transid.finished(this, start, s"[QUERY] '$dbName' completed: matched ${out.size}") + out + + case Left(code) => + transid.failed(this, start, s"Unexpected http response code: $code", ErrorLevel) + throw new Exception("Unexpected http response code: " + code) + } reportFailure( f, @@ -329,6 +329,43 @@ class CouchDbRestStore[DocumentAbstraction <: DocumentSerializer](dbProtocol: St transid.failed(this, start, s"[QUERY] '$dbName' internal error, failure: '${failure.getMessage}'", ErrorLevel)) } + protected[core] def count(table: String, startKey: List[Any], endKey: List[Any], skip: Int, stale: StaleParameter)( + implicit transid: TransactionId): Future[Long] = { + + val Array(firstPart, secondPart) = table.split("/") + + val start = transid.started(this, LoggingMarkers.DATABASE_QUERY, s"[COUNT] '$dbName' searching '$table") + + val f = client + .executeView(firstPart, secondPart)( + startKey = startKey, + endKey = endKey, + skip = Some(skip), + stale = stale, + reduce = true) + .map { + case Right(response) => + val rows = response.fields("rows").convertTo[List[JsObject]] + + val out = if (!rows.isEmpty) { + assert(rows.length == 1, s"result of reduced view contains more than one value: '$rows'") + rows.head.fields("value").convertTo[Long] + } else 0L + + transid.finished(this, start, s"[COUNT] '$dbName' completed: count $out") + out + + case Left(code) => + transid.failed(this, start, s"Unexpected http response code: $code", ErrorLevel) + throw new Exception("Unexpected http response code: " + code) + } + + reportFailure( + f, + failure => + transid.failed(this, start, s"[COUNT] '$dbName' internal error, failure: '${failure.getMessage}'", ErrorLevel)) + } + override protected[core] def attach( doc: DocInfo, name: String, diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala index 904cd62..adddf81 100644 --- a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala @@ -167,19 +167,15 @@ case class WhiskAction(namespace: EntityPath, * Strictly used in view testing to enforce alignment. */ override def summaryAsJson = { - if (WhiskEntityQueries.designDoc.endsWith("v2")) { - super.summaryAsJson - } else { - val binary = exec match { - case c: CodeExec[_] => c.binary - case _ => false - } - - JsObject( - super.summaryAsJson.fields + - ("limits" -> limits.toJson) + - ("exec" -> JsObject("binary" -> JsBoolean(binary)))) + val binary = exec match { + case c: CodeExec[_] => c.binary + case _ => false } + + JsObject( + super.summaryAsJson.fields + + ("limits" -> limits.toJson) + + ("exec" -> JsObject("binary" -> JsBoolean(binary)))) } } diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskActivation.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskActivation.scala index 7556abb..f6026ef 100644 --- a/common/scala/src/main/scala/whisk/core/entity/WhiskActivation.scala +++ b/common/scala/src/main/scala/whisk/core/entity/WhiskActivation.scala @@ -162,7 +162,7 @@ object WhiskActivation * A view for activations in a namespace additionally keyed by action name * (and package name if present) sorted by date. */ - val filtersView = WhiskEntityQueries.view(filtersDdoc, collectionName) + lazy val filtersView = WhiskEntityQueries.view(filtersDdoc, collectionName) override implicit val serdes = jsonFormat13(WhiskActivation.apply) diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskEntity.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskEntity.scala index 501f597..0f1e5e1 100644 --- a/common/scala/src/main/scala/whisk/core/entity/WhiskEntity.scala +++ b/common/scala/src/main/scala/whisk/core/entity/WhiskEntity.scala @@ -87,18 +87,15 @@ abstract class WhiskEntity protected[entity] (en: EntityName, val entityType: St * This should be synchronized with the views computed in the databse. * Strictly used in view testing to enforce alignment. */ - def summaryAsJson = { + def summaryAsJson: JsObject = { import WhiskActivation.instantSerdes - val base = Map( + JsObject( "namespace" -> namespace.toJson, "name" -> name.toJson, "version" -> version.toJson, WhiskEntity.sharedFieldName -> JsBoolean(publish), "annotations" -> annotations.toJsArray, "updated" -> updated.toJson) - if (WhiskEntityQueries.designDoc.endsWith("v2")) { - JsObject(base - "updated") - } else JsObject(base) } } diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskPackage.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskPackage.scala index 92b0e56..57390ae 100644 --- a/common/scala/src/main/scala/whisk/core/entity/WhiskPackage.scala +++ b/common/scala/src/main/scala/whisk/core/entity/WhiskPackage.scala @@ -136,13 +136,9 @@ case class WhiskPackage(namespace: EntityPath, * Strictly used in view testing to enforce alignment. */ override def summaryAsJson = { - if (WhiskEntityQueries.designDoc.endsWith("v2")) { - JsObject(super.summaryAsJson.fields + (WhiskPackage.bindingFieldName -> binding.isDefined.toJson)) - } else { - JsObject( - super.summaryAsJson.fields + - (WhiskPackage.bindingFieldName -> binding.map(Binding.serdes.write(_)).getOrElse(JsBoolean(false)))) - } + JsObject( + super.summaryAsJson.fields + + (WhiskPackage.bindingFieldName -> binding.map(Binding.serdes.write(_)).getOrElse(JsBoolean(false)))) } } @@ -205,6 +201,8 @@ object WhiskPackage } override val cacheEnabled = true + + lazy val publicPackagesView: View = WhiskEntityQueries.view(collection = s"$collectionName-public") } /** diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskStore.scala b/common/scala/src/main/scala/whisk/core/entity/WhiskStore.scala index 94be958..7eaf1a1 100644 --- a/common/scala/src/main/scala/whisk/core/entity/WhiskStore.scala +++ b/common/scala/src/main/scala/whisk/core/entity/WhiskStore.scala @@ -24,6 +24,7 @@ import scala.language.postfixOps import scala.util.Try import akka.actor.ActorSystem import akka.stream.ActorMaterializer +import spray.json.JsNumber import spray.json.JsObject import spray.json.JsString import spray.json.RootJsonFormat @@ -202,7 +203,7 @@ object WhiskEntityQueries { * Name of view in design-doc that lists all entities in that views regardless of types. * This is uses in the namespace API, and also in tests to check preconditions. */ - val viewAll: View = view(collection = "all") + lazy val viewAll: View = view(ddoc = s"all-$designDoc", collection = "all") /** * Queries the datastore for all entities in a namespace, and converts the list of entities @@ -242,19 +243,42 @@ trait WhiskEntityQueries[T] { * @return list of records as JSON object if docs parameter is false, as Left * and a list of the records as their type T if including the full record, as Right */ - def listCollectionInNamespace[A <: WhiskEntity](db: ArtifactStore[A], - path: EntityPath, // could be a namesapce or namespace + package name - skip: Int, - limit: Int, - includeDocs: Boolean = false, - since: Option[Instant] = None, - upto: Option[Instant] = None, - stale: StaleParameter = StaleParameter.No)( - implicit transid: TransactionId): Future[Either[List[JsObject], List[T]]] = { + def listCollectionInNamespace[A <: WhiskEntity]( + db: ArtifactStore[A], + path: EntityPath, // could be a namesapce or namespace + package name + skip: Int, + limit: Int, + includeDocs: Boolean = false, + since: Option[Instant] = None, + upto: Option[Instant] = None, + stale: StaleParameter = StaleParameter.No, + viewName: View = view)(implicit transid: TransactionId): Future[Either[List[JsObject], List[T]]] = { val convert = if (includeDocs) Some((o: JsObject) => Try { serdes.read(o) }) else None val startKey = List(path.asString, since map { _.toEpochMilli } getOrElse 0) val endKey = List(path.asString, upto map { _.toEpochMilli } getOrElse TOP, TOP) - query(db, view, startKey, endKey, skip, limit, reduce = false, stale, convert) + query(db, viewName, startKey, endKey, skip, limit, reduce = false, stale, convert) + } + + /** + * Queries the datastore for the records count in a specific collection (i.e., type) matching + * the given path (which should be one namespace, or namespace + package name). + * + * @return JSON object with a single key, the collection name, and a value equal to the view length + */ + def countCollectionInNamespace[A <: WhiskEntity]( + db: ArtifactStore[A], + path: EntityPath, // could be a namespace or namespace + package name + skip: Int, + since: Option[Instant] = None, + upto: Option[Instant] = None, + stale: StaleParameter = StaleParameter.No, + viewName: View = view)(implicit transid: TransactionId): Future[JsObject] = { + implicit val ec = db.executionContext + val startKey = List(path.asString, since map { _.toEpochMilli } getOrElse 0) + val endKey = List(path.asString, upto map { _.toEpochMilli } getOrElse TOP, TOP) + db.count(viewName.name, startKey, endKey, skip, stale) map { count => + JsObject(collectionName -> JsNumber(count)) + } } protected[entity] def query[A <: WhiskEntity]( @@ -269,7 +293,7 @@ trait WhiskEntityQueries[T] { convert: Option[JsObject => Try[T]])(implicit transid: TransactionId): Future[Either[List[JsObject], List[T]]] = { implicit val ec = db.executionContext val includeDocs = convert.isDefined - db.query(view.name, startKey, endKey, skip, limit, includeDocs, true, reduce, stale) map { rows => + db.query(view.name, startKey, endKey, skip, limit, includeDocs, descending = true, reduce, stale) map { rows => convert map { fn => Right(rows flatMap { row => fn(row.fields("doc").asJsObject) toOption diff --git a/core/controller/src/main/scala/whisk/core/controller/Actions.scala b/core/controller/src/main/scala/whisk/core/controller/Actions.scala index e08d0e4..d0a43c9 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Actions.scala @@ -318,14 +318,19 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 200 [] or [WhiskAction as JSON] * - 500 Internal Server Error */ - override def list(user: Identity, namespace: EntityPath, excludePrivate: Boolean)(implicit transid: TransactionId) = { + override def list(user: Identity, namespace: EntityPath)(implicit transid: TransactionId) = { parameter('skip ? 0, 'limit.as[ListLimit] ? ListLimit(collection.defaultListLimit), 'count ? false) { (skip, limit, count) => - listEntities { - WhiskAction.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = false) map { - list => - val actions = list.fold((js) => js, (as) => as.map(WhiskAction.serdes.write(_))) - FilterEntityList.filter(actions, excludePrivate) + if (!count) { + listEntities { + WhiskAction.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = false) map { + list => + list.fold((js) => js, (as) => as.map(WhiskAction.serdes.write(_))) + } + } + } else { + countEntities { + WhiskAction.countCollectionInNamespace(entityStore, namespace, skip) } } } @@ -518,10 +523,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with pkgName.path.addPath(wp.name) } // list actions in resolved namespace - // NOTE: excludePrivate is false since the subject is authorize to access - // the package; in the future, may wish to exclude private actions in a - // public package instead - list(user, pkgns, excludePrivate = false) + list(user, pkgns) }) } diff --git a/core/controller/src/main/scala/whisk/core/controller/Activations.scala b/core/controller/src/main/scala/whisk/core/controller/Activations.scala index c136eb2..f7ea7b8 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Activations.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Activations.scala @@ -151,9 +151,20 @@ trait WhiskActivationsApi extends Directives with AuthenticatedRouteProvider wit 'name.as[Option[EntityPath]] ?, 'since.as[Instant] ?, 'upto.as[Instant] ?) { (skip, limit, count, docs, name, since, upto) => - val invalidDocs = count && docs - // regardless of limit, cap at maxActivationLimit (200) records, client must paginate - if (!invalidDocs) { + if (count && !docs) { + countEntities { + WhiskActivation.countCollectionInNamespace( + activationStore, + name.flatten.map(p => namespace.addPath(p)).getOrElse(namespace), + skip, + since, + upto, + StaleParameter.UpdateAfter, + viewName = name.flatten.map(_ => WhiskActivation.filtersView).getOrElse(WhiskActivation.view)) + } + } else if (count && docs) { + terminate(BadRequest, Messages.docsNotAllowedWithCount) + } else { val activations = name.flatten match { case Some(action) => WhiskActivation.listActivationsMatchingName( @@ -178,8 +189,6 @@ trait WhiskActivationsApi extends Directives with AuthenticatedRouteProvider wit StaleParameter.UpdateAfter) } listEntities(activations map (_.fold((js) => js, (wa) => wa.map(_.toExtendedJson)))) - } else { - terminate(BadRequest, Messages.docsNotAllowedWithCount) } } } diff --git a/core/controller/src/main/scala/whisk/core/controller/ApiUtils.scala b/core/controller/src/main/scala/whisk/core/controller/ApiUtils.scala index 4b44ddf..0f5b16c 100644 --- a/core/controller/src/main/scala/whisk/core/controller/ApiUtils.scala +++ b/core/controller/src/main/scala/whisk/core/controller/ApiUtils.scala @@ -31,7 +31,6 @@ import akka.http.scaladsl.server.Directives import akka.http.scaladsl.server.RequestContext import akka.http.scaladsl.server.RouteResult import spray.json.DefaultJsonProtocol._ -import spray.json.JsBoolean import spray.json.JsObject import spray.json.JsValue import spray.json.RootJsonFormat @@ -41,7 +40,6 @@ import whisk.core.controller.PostProcess.PostProcessEntity import whisk.core.database._ import whisk.core.entity.DocId import whisk.core.entity.WhiskDocument -import whisk.core.entity.WhiskEntity import whisk.http.ErrorResponse import whisk.http.ErrorResponse.terminate import whisk.http.Messages._ @@ -70,30 +68,6 @@ protected[core] object RejectRequest { } } -protected[controller] object FilterEntityList { - import WhiskEntity.sharedFieldName - - /** - * Filters from a list of entities serialized to JsObjects only those - * that have the shared field ("publish") equal to true and excludes - * all others. - */ - protected[controller] def filter(resources: List[JsValue], - excludePrivate: Boolean, - additionalFilter: JsObject => Boolean = (_ => true)) = { - if (excludePrivate) { - resources filter { - case obj: JsObject => - obj.fields.get(sharedFieldName) match { - case Some(JsBoolean(true)) => additionalFilter(obj) // a shared entity - case _ => false - } - case _ => false // only expecting JsObject instances - } - } else resources - } -} - /** * A convenient typedef for functions that post process an entity * on an operation and terminate the HTTP request. @@ -114,13 +88,7 @@ trait ReadOps extends Directives { import RestApiCommons.jsonDefaultResponsePrinter /** - * Get all entities of type A from datastore that match key. Terminates HTTP request. - * - * @param factory the factory that can fetch entities of type A from datastore - * @param datastore the client to the database - * @param key the key to use to match records in the view, optional, if not defined, use namespace - * @param view the view to query - * @param filter a function List[A] => List[A] that filters the results + * Terminates HTTP request for list requests. * * Responses are one of (Code, Message) * - 200 entity A [] as JSON [] @@ -138,6 +106,24 @@ trait ReadOps extends Directives { } /** + * Terminates HTTP request for list count requests. + * + * Responses are one of (Code, Message) + * - 200 JSON object + * - 500 Internal Server Error + */ + protected def countEntities(count: Future[JsValue])(implicit transid: TransactionId) = { + onComplete(count) { + case Success(c) => + logging.info(this, s"[COUNT] count success") + complete(OK, c) + case Failure(t: Throwable) => + logging.error(this, s"[COUNT] count failed: ${t.getMessage}") + terminate(InternalServerError) + } + } + + /** * Gets an entity of type A from datastore. Terminates HTTP request. * * @param factory the factory that can fetch entity of type A from datastore diff --git a/core/controller/src/main/scala/whisk/core/controller/Entities.scala b/core/controller/src/main/scala/whisk/core/controller/Entities.scala index fff8269..cd9a72a 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Entities.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Entities.scala @@ -94,12 +94,9 @@ trait WhiskCollectionAPI implicit transid: TransactionId): RequestContext => Future[RouteResult] /** Gets all entities from namespace. If necessary filter only entities that are shared. Terminates HTTP request. */ - protected def list(user: Identity, path: EntityPath, excludePrivate: Boolean)( + protected def list(user: Identity, path: EntityPath)( implicit transid: TransactionId): RequestContext => Future[RouteResult] - /** Indicates if listing entities in collection requires filtering out private entities. */ - protected val listRequiresPrivateEntityFilter = false // currently supported on PACKAGES only - /** Dispatches resource to the proper handler depending on context. */ protected override def dispatchOp(user: Identity, op: Privilege, resource: Resource)( implicit transid: TransactionId) = { @@ -131,9 +128,8 @@ trait WhiskCollectionAPI // produce all entities in the requested namespace UNLESS the subject is // entitled to them which for now means they own the namespace. If the // subject does not own the namespace, then exclude packages that are private - val excludePrivate = listRequiresPrivateEntityFilter && resource.namespace.root != user.namespace - logging.info(this, s"[LIST] exclude private entities: required == $excludePrivate") - list(user, resource.namespace, excludePrivate) + // in the API handler + list(user, resource.namespace) case _ => reject } diff --git a/core/controller/src/main/scala/whisk/core/controller/Packages.scala b/core/controller/src/main/scala/whisk/core/controller/Packages.scala index d469b0e..6a646c0 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Packages.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Packages.scala @@ -25,8 +25,6 @@ import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.server.{RequestContext, RouteResult} import akka.http.scaladsl.unmarshalling.Unmarshaller -import spray.json._ - import whisk.common.TransactionId import whisk.core.controller.RestApiCommons.ListLimit import whisk.core.database.{CacheChangeNotification, DocumentTypeMismatchException, NoDocumentException} @@ -50,9 +48,6 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { /** Route directives for API. The methods that are supported on packages. */ protected override lazy val entityOps = put | get | delete - /** Must exclude any private packages when listing those in a namespace unless owned by subject. */ - protected override val listRequiresPrivateEntityFilter = true - /** JSON response formatter. */ import RestApiCommons.jsonDefaultResponsePrinter @@ -173,24 +168,25 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { * - 200 [] or [WhiskPackage as JSON] * - 500 Internal Server Error */ - override def list(user: Identity, namespace: EntityPath, excludePrivate: Boolean)(implicit transid: TransactionId) = { + override def list(user: Identity, namespace: EntityPath)(implicit transid: TransactionId) = { parameter('skip ? 0, 'limit.as[ListLimit] ? ListLimit(collection.defaultListLimit), 'count ? false) { (skip, limit, count) => - listEntities { - WhiskPackage.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = false) map { - list => - // any subject is entitled to list packages in any namespace - // however, they shall only observe public packages if the packages - // are not in one of the namespaces the subject is entitled to - val packages = list.fold((js) => js, (ps) => ps.map(WhiskPackage.serdes.write(_))) - - FilterEntityList.filter(packages, excludePrivate, additionalFilter = { - // additionally exclude bindings - _.fields.get(WhiskPackage.bindingFieldName) match { - case Some(JsBoolean(isbinding)) => !isbinding - case _ => false // exclude anything that does not conform - } - }) + val viewName = if (user.namespace.toPath == namespace) WhiskPackage.view else WhiskPackage.publicPackagesView + if (!count) { + listEntities { + WhiskPackage + .listCollectionInNamespace( + entityStore, + namespace, + skip, + limit.n, + includeDocs = false, + viewName = viewName) + .map(_.fold((js) => js, (ps) => ps.map(WhiskPackage.serdes.write(_)))) + } + } else { + countEntities { + WhiskPackage.countCollectionInNamespace(entityStore, namespace, skip, viewName = viewName) } } } diff --git a/core/controller/src/main/scala/whisk/core/controller/Rules.scala b/core/controller/src/main/scala/whisk/core/controller/Rules.scala index 0540e7d..9561d37 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Rules.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Rules.scala @@ -234,14 +234,18 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { * - 200 [] or [WhiskRule as JSON] * - 500 Internal Server Error */ - override def list(user: Identity, namespace: EntityPath, excludePrivate: Boolean)(implicit transid: TransactionId) = { + override def list(user: Identity, namespace: EntityPath)(implicit transid: TransactionId) = { parameter('skip ? 0, 'limit.as[ListLimit] ? ListLimit(collection.defaultListLimit), 'count ? false) { (skip, limit, count) => - val includeDocs = WhiskEntityQueries.designDoc.endsWith("v2.1.0") - listEntities { - WhiskRule.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs) map { list => - val rules = list.fold((js) => js, (rls) => rls.map(WhiskRule.serdes.write(_))) - FilterEntityList.filter(rules, excludePrivate) + if (!count) { + listEntities { + WhiskRule.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = true) map { list => + list.fold((js) => js, (rls) => rls.map(WhiskRule.serdes.write(_))) + } + } + } else { + countEntities { + WhiskRule.countCollectionInNamespace(entityStore, namespace, skip) } } } diff --git a/core/controller/src/main/scala/whisk/core/controller/Triggers.scala b/core/controller/src/main/scala/whisk/core/controller/Triggers.scala index 8c71fce..3893f57 100644 --- a/core/controller/src/main/scala/whisk/core/controller/Triggers.scala +++ b/core/controller/src/main/scala/whisk/core/controller/Triggers.scala @@ -235,14 +235,19 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { * - 200 [] or [WhiskTrigger as JSON] * - 500 Internal Server Error */ - override def list(user: Identity, namespace: EntityPath, excludePrivate: Boolean)(implicit transid: TransactionId) = { + override def list(user: Identity, namespace: EntityPath)(implicit transid: TransactionId) = { parameter('skip ? 0, 'limit.as[ListLimit] ? ListLimit(collection.defaultListLimit), 'count ? false) { (skip, limit, count) => - listEntities { - WhiskTrigger.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = false) map { - list => - val triggers = list.fold((js) => js, (ts) => ts.map(WhiskTrigger.serdes.write(_))) - FilterEntityList.filter(triggers, excludePrivate) + if (!count) { + listEntities { + WhiskTrigger.listCollectionInNamespace(entityStore, namespace, skip, limit.n, includeDocs = false) map { + list => + list.fold((js) => js, (ts) => ts.map(WhiskTrigger.serdes.write(_))) + } + } + } else { + countEntities { + WhiskTrigger.countCollectionInNamespace(entityStore, namespace, skip) } } } diff --git a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala index 6c2dcd1..3566713 100644 --- a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala @@ -85,9 +85,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { status should be(OK) val response = responseAs[List[JsObject]] actions.length should be(response.length) - actions forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs actions.map(_.summaryAsJson) } } @@ -114,9 +112,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { status should be(OK) val response = responseAs[List[WhiskAction]] actions.length should be(response.length) - actions forall { a => - response contains a - } should be(true) + response should contain theSameElementsAs actions.map(_.summaryAsJson) } } @@ -131,9 +127,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { status should be(OK) val response = responseAs[List[JsObject]] actions.length should be(response.length) - actions forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs actions.map(_.summaryAsJson) } // it should "reject list action with explicit namespace not owned by subject" in { diff --git a/tests/src/test/scala/whisk/core/controller/test/ActivationsApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/ActivationsApiTests.scala index d37dae7..cc16220 100644 --- a/tests/src/test/scala/whisk/core/controller/test/ActivationsApiTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/ActivationsApiTests.scala @@ -58,6 +58,18 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi val collectionPath = s"/${EntityPath.DEFAULT}/${collection.path}" def aname() = MakeName.next("activations_tests") + def checkCount(filter: String, expected: Int, user: Identity = creds) = { + implicit val tid = transid() + withClue(s"count did not match for filter: $filter") { + whisk.utils.retry { + Get(s"$collectionPath?count=true&$filter") ~> Route.seal(routes(user)) ~> check { + status should be(OK) + responseAs[JsObject] shouldBe JsObject(collection.path -> JsNumber(expected)) + } + } + } + } + //// GET /activations it should "get summary activation by namespace" in { implicit val tid = transid() @@ -82,13 +94,10 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi whisk.utils.retry { Get(s"$collectionPath") ~> Route.seal(routes(creds)) ~> check { status should be(OK) - val rawResponse = responseAs[List[JsObject]] val response = responseAs[List[JsObject]] activations.length should be(response.length) - activations forall { a => - response contains a.summaryAsJson - } should be(true) - rawResponse forall { a => + response should contain theSameElementsAs activations.map(_.summaryAsJson) + response forall { a => a.getFields("for") match { case Seq(JsString(n)) => n == actionName.asString case _ => false @@ -101,13 +110,10 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi whisk.utils.retry { Get(s"/$namespace/${collection.path}") ~> Route.seal(routes(creds)) ~> check { status should be(OK) - val rawResponse = responseAs[List[JsObject]] val response = responseAs[List[JsObject]] activations.length should be(response.length) - activations forall { a => - response contains a.summaryAsJson - } should be(true) - rawResponse forall { a => + response should contain theSameElementsAs activations.map(_.summaryAsJson) + response forall { a => a.getFields("for") match { case Seq(JsString(n)) => n == actionName.asString case _ => false @@ -162,14 +168,14 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi activations foreach { put(activationStore, _) } waitOnView(activationStore, namespace.root, 2, WhiskActivation.view) + checkCount("", 2) + whisk.utils.retry { Get(s"$collectionPath?docs=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] activations.length should be(response.length) - activations forall { a => - response contains a.toExtendedJson - } should be(true) + response should contain theSameElementsAs activations.map(_.toExtendedJson) } } } @@ -238,14 +244,14 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi (e.start.equals(since) || e.start.equals(upto) || (e.start.isAfter(since) && e.start.isBefore(upto))) } + checkCount(filter, expected.length) + whisk.utils.retry { Get(s"$collectionPath?docs=true&$filter") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] expected.length should be(response.length) - expected forall { a => - response contains a.toExtendedJson - } should be(true) + response should contain theSameElementsAs expected.map(_.toExtendedJson) } } } @@ -254,14 +260,14 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi val expected = activations.filter(e => e.start.equals(upto) || e.start.isBefore(upto)) val filter = s"upto=${upto.toEpochMilli}" + checkCount(filter, expected.length) + whisk.utils.retry { Get(s"$collectionPath?docs=true&$filter") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] expected.length should be(response.length) - expected forall { a => - response contains a.toExtendedJson - } should be(true) + response should contain theSameElementsAs expected.map(_.toExtendedJson) } } } @@ -271,13 +277,13 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi val expected = activations.filter(e => e.start.equals(since) || e.start.isAfter(since)) val filter = s"since=${since.toEpochMilli}" + checkCount(filter, expected.length) + Get(s"$collectionPath?docs=true&$filter") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] expected.length should be(response.length) - expected forall { a => - response contains a.toExtendedJson - } should be(true) + response should contain theSameElementsAs expected.map(_.toExtendedJson) } } } @@ -343,26 +349,25 @@ class ActivationsApiTests extends ControllerTestCommon with WhiskActivationsApi activationsInPackage.length, WhiskActivation.filtersView) + checkCount("name=xyz", activations.length) + whisk.utils.retry { Get(s"$collectionPath?name=xyz") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] activations.length should be(response.length) - activations forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs activations.map(_.summaryAsJson) } } - // this is not yet ready, the v2 views must be activated + checkCount("name=pkg/xyz", activations.length) + whisk.utils.retry { Get(s"$collectionPath?name=pkg/xyz") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] activationsInPackage.length should be(response.length) - activationsInPackage forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs activationsInPackage.map(_.summaryAsJson) } } } diff --git a/tests/src/test/scala/whisk/core/controller/test/PackagesApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/PackagesApiTests.scala index 0fd2825..54e3823 100644 --- a/tests/src/test/scala/whisk/core/controller/test/PackagesApiTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/PackagesApiTests.scala @@ -58,6 +58,18 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { Parameters(WhiskPackage.bindingFieldName, Binding.serdes.write(binding)) } + def checkCount(path: String = collectionPath, expected: Long, user: Identity = creds) = { + implicit val tid = transid() + withClue(s"count did not match") { + whisk.utils.retry { + Get(s"$path?count=true") ~> Route.seal(routes(user)) ~> check { + status should be(OK) + responseAs[JsObject].fields(collection.path).convertTo[Long] shouldBe (expected) + } + } + } + } + //// GET /packages it should "list all packages/references" in { implicit val tid = transid() @@ -72,21 +84,25 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { }.toList providers foreach { put(entityStore, _) } waitOnView(entityStore, WhiskPackage, namespace, providers.length) + + checkCount(expected = providers.length) whisk.utils.retry { Get(s"$collectionPath") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] providers.length should be(response.length) - providers forall { p => - response contains p.summaryAsJson - } should be(true) + response should contain theSameElementsAs providers.map(_.summaryAsJson) } } - val auser = WhiskAuthHelpers.newIdentity() - Get(s"/$namespace/${collection.path}") ~> Route.seal(routes(auser)) ~> check { - val response = responseAs[List[JsObject]] - response should be(List()) // cannot list packages that are private in another namespace + { + val path = s"/$namespace/${collection.path}" + val auser = WhiskAuthHelpers.newIdentity() + checkCount(path, 0, auser) + Get(path) ~> Route.seal(routes(auser)) ~> check { + val response = responseAs[List[JsObject]] + response should be(List()) // cannot list packages that are private in another namespace + } } } @@ -107,29 +123,31 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { waitOnView(entityStore, WhiskPackage, namespaces(1), 1) waitOnView(entityStore, WhiskPackage, namespaces(2), 1) waitOnView(entityStore, WhiskPackage, namespaces(0), 1 + 4) - Get(s"$collectionPath") ~> Route.seal(routes(creds)) ~> check { - status should be(OK) - val response = responseAs[List[JsObject]] - val expected = providers.filter { _.namespace == namespace } ++ references - response.length should be >= (expected.length) - expected forall { p => - (response contains p.summaryAsJson) - } should be(true) + + { + val expected = providers.filter(_.namespace == namespace) ++ references + checkCount(expected = expected.length) + Get(s"$collectionPath") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[List[JsObject]] + response should have size expected.size + response should contain theSameElementsAs expected.map(_.summaryAsJson) + } } - val auser = WhiskAuthHelpers.newIdentity() - Get(s"/$namespace/${collection.path}") ~> Route.seal(routes(auser)) ~> check { - status should be(OK) - val response = responseAs[List[JsObject]] - val expected = providers.filter { p => - p.namespace == namespace && p.publish - } ++ references.filter { p => - p.publish && p.binding == None + { + val path = s"/$namespace/${collection.path}" + val auser = WhiskAuthHelpers.newIdentity() + val expected = providers.filter(p => p.namespace == namespace && p.publish) ++ + references.filter(p => p.publish && p.binding == None) + + checkCount(path, expected.length, auser) + Get(path) ~> Route.seal(routes(auser)) ~> check { + status should be(OK) + val response = responseAs[List[JsObject]] + response should have size expected.size + response should contain theSameElementsAs expected.map(_.summaryAsJson) } - response.length should be >= (expected.length) - expected forall { p => - (response contains p.summaryAsJson) - } should be(true) } } @@ -213,10 +231,11 @@ class PackagesApiTests extends ControllerTestCommon with WhiskPackagesApi { waitOnView(entityStore, WhiskPackage, namespaces(0), 1) waitOnView(entityStore, WhiskPackage, namespaces(1), 1) waitOnView(entityStore, WhiskPackage, namespaces(2), 1) + val expected = providers filter (_.namespace == creds.namespace.toPath) + Get(s"$collectionPath?public=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] - val expected = providers filter { _.namespace == creds.namespace.toPath } response.length should be >= (expected.length) expected forall { p => (response contains p.summaryAsJson) && p.binding == None diff --git a/tests/src/test/scala/whisk/core/controller/test/RulesApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/RulesApiTests.scala index 60804b7..54fb047 100644 --- a/tests/src/test/scala/whisk/core/controller/test/RulesApiTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/RulesApiTests.scala @@ -70,24 +70,20 @@ class RulesApiTests extends ControllerTestCommon with WhiskRulesApi { WhiskRule(namespace, aname(), afullname(namespace, "bogus trigger"), afullname(namespace, "bogus action")) }.toList rules foreach { put(entityStore, _) } - waitOnView(entityStore, WhiskRule, namespace, 2) + waitOnView(entityStore, WhiskRule, namespace, 2, includeDocs = true) Get(s"$collectionPath") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] rules.length should be(response.length) - rules forall { r => - response contains r.summaryAsJson - } should be(true) + response should contain theSameElementsAs rules.map(_.toJson) } - // it should "list trirulesggers with explicit namespace owned by subject" in { + // it should "list rules with explicit namespace owned by subject" in { Get(s"/$namespace/${collection.path}") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] rules.length should be(response.length) - rules forall { r => - response contains r.summaryAsJson - } should be(true) + response should contain theSameElementsAs rules.map(_.toJson) } // it should "reject list rules with explicit namespace not owned by subject" in { @@ -116,14 +112,12 @@ class RulesApiTests extends ControllerTestCommon with WhiskRulesApi { WhiskRule(namespace, aname(), afullname(namespace, "bogus trigger"), afullname(namespace, "bogus action")) }.toList rules foreach { put(entityStore, _) } - waitOnView(entityStore, WhiskRule, namespace, 2) + waitOnView(entityStore, WhiskRule, namespace, 2, includeDocs = true) Get(s"$collectionPath?docs=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[WhiskRule]] rules.length should be(response.length) - rules forall { r => - response contains r - } should be(true) + response should contain theSameElementsAs rules.map(_.toJson) } } diff --git a/tests/src/test/scala/whisk/core/controller/test/TriggersApiTests.scala b/tests/src/test/scala/whisk/core/controller/test/TriggersApiTests.scala index 513c05f..200086f 100644 --- a/tests/src/test/scala/whisk/core/controller/test/TriggersApiTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/TriggersApiTests.scala @@ -78,9 +78,7 @@ class TriggersApiTests extends ControllerTestCommon with WhiskTriggersApi { status should be(OK) val response = responseAs[List[JsObject]] triggers.length should be(response.length) - triggers forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs triggers.map(_.summaryAsJson) } // it should "list triggers with explicit namespace owned by subject" in { @@ -88,9 +86,7 @@ class TriggersApiTests extends ControllerTestCommon with WhiskTriggersApi { status should be(OK) val response = responseAs[List[JsObject]] triggers.length should be(response.length) - triggers forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs triggers.map(_.summaryAsJson) } // it should "reject list triggers with explicit namespace not owned by subject" in { @@ -123,9 +119,7 @@ class TriggersApiTests extends ControllerTestCommon with WhiskTriggersApi { status should be(OK) val response = responseAs[List[WhiskTrigger]] triggers.length should be(response.length) - triggers forall { a => - response contains a - } should be(true) + response should contain theSameElementsAs triggers.map(_.summaryAsJson) } } diff --git a/tests/src/test/scala/whisk/core/controller/test/migration/SequenceActionApiMigrationTests.scala b/tests/src/test/scala/whisk/core/controller/test/migration/SequenceActionApiMigrationTests.scala index 4d0f035..dbeb29e 100644 --- a/tests/src/test/scala/whisk/core/controller/test/migration/SequenceActionApiMigrationTests.scala +++ b/tests/src/test/scala/whisk/core/controller/test/migration/SequenceActionApiMigrationTests.scala @@ -67,11 +67,8 @@ class SequenceActionApiMigrationTests Get(s"/$namespace/${collection.path}") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[List[JsObject]] - actions.length should be(response.length) - actions forall { a => - response contains a.summaryAsJson - } should be(true) + response should contain theSameElementsAs actions.map(_.summaryAsJson) } } diff --git a/tests/src/test/scala/whisk/core/database/test/DbUtils.scala b/tests/src/test/scala/whisk/core/database/test/DbUtils.scala index 85b69fb..c7a75b7 100644 --- a/tests/src/test/scala/whisk/core/database/test/DbUtils.scala +++ b/tests/src/test/scala/whisk/core/database/test/DbUtils.scala @@ -125,13 +125,15 @@ trait DbUtils extends TransactionCounter { * This uses retry above, where the step performs a collection-specific view query using the collection * factory. The result count from the view is checked against the given value. */ - def waitOnView(db: EntityStore, factory: WhiskEntityQueries[_], namespace: EntityPath, count: Int)( - implicit context: ExecutionContext, - transid: TransactionId, - timeout: Duration) = { + def waitOnView( + db: EntityStore, + factory: WhiskEntityQueries[_], + namespace: EntityPath, + count: Int, + includeDocs: Boolean = false)(implicit context: ExecutionContext, transid: TransactionId, timeout: Duration) = { val success = retry(() => { - factory.listCollectionInNamespace(db, namespace, 0, 0) map { l => - if (l.left.get.length < count) { + factory.listCollectionInNamespace(db, namespace, 0, 0, includeDocs) map { l => + if (l.fold(_.length, _.length) < count) { throw RetryOp() } else true } diff --git a/tools/admin/wskadmin b/tools/admin/wskadmin index c23a7a9..861c9a9 100755 --- a/tools/admin/wskadmin +++ b/tools/admin/wskadmin @@ -599,7 +599,7 @@ def getDbCmd(args, props): print('view name "%s" is not formatted correctly, should be design/view' % args.view) return 2 - url = '%(protocol)s://%(host)s:%(port)s/%(database)s%(design)s/%(index)s?include_docs=%(docs)s' % { + url = '%(protocol)s://%(host)s:%(port)s/%(database)s%(design)s/%(index)s?reduce=false&include_docs=%(docs)s' % { 'protocol': protocol, 'host' : host, 'port' : port, -- To stop receiving notification emails like this one, please contact ['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].