chetanmeh commented on a change in pull request #3517: MemoryArtifactStore for unit testing and ArtifactStore SPI Validation URL: https://github.com/apache/incubator-openwhisk/pull/3517#discussion_r181288490
########## File path: common/scala/src/main/scala/whisk/core/database/DocumentHandler.scala ########## @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.database + +import spray.json._ +import spray.json.DefaultJsonProtocol._ +import whisk.common.TransactionId +import whisk.core.entity.{DocId, UserLimits} +import whisk.core.entity.EntityPath.PATHSEP +import whisk.utils.JsHelpers + +import scala.collection.immutable.Seq +import scala.concurrent.Future +import scala.concurrent.ExecutionContext + +/** + * Simple abstraction allow accessing a document just by _id. This would be used + * to perform queries related to join support + */ +trait DocumentProvider { + protected[database] def get(id: DocId)(implicit transid: TransactionId): Future[Option[JsObject]] +} + +trait DocumentHandler { + + /** + * Returns a JsObject having computed fields. This is a substitution for fields + * computed in CouchDB views + */ + def computedFields(js: JsObject): JsObject = JsObject.empty + + /** + * Returns the set of field names (including sub document field) which needs to be fetched as part of + * query made for the given view. + */ + def fieldsRequiredForView(ddoc: String, view: String): Set[String] = Set.empty + + /** + * Transforms the query result instance from artifact store as per view requirements. Some view computation + * may result in performing a join operation. + * + * If the passed instance does not confirm to view conditions that transformed result would be None. This + * would be the case if view condition cannot be completed handled in query made to artifact store + */ + def transformViewResult( + ddoc: String, + view: String, + startKey: List[Any], + endKey: List[Any], + includeDocs: Boolean, + js: JsObject, + provider: DocumentProvider)(implicit transid: TransactionId, ec: ExecutionContext): Future[Seq[JsObject]] + + /** + * Determines if the complete document should be fetched even if `includeDocs` is set to true. For some view computation + * complete document (including sub documents) may be needed and for them its required that complete document must be + * fetched as part of query response + */ + def shouldAlwaysIncludeDocs(ddoc: String, view: String): Boolean = false + + def checkIfTableSupported(table: String): Unit = { + if (!supportedTables.contains(table)) { + throw UnsupportedView(table) + } + } + + protected def supportedTables: Set[String] +} + +/** + * Base class for handlers which do not perform joins for computing views + */ +abstract class SimpleHandler extends DocumentHandler { + override def transformViewResult( + ddoc: String, + view: String, + startKey: List[Any], + endKey: List[Any], + includeDocs: Boolean, + js: JsObject, + provider: DocumentProvider)(implicit transid: TransactionId, ec: ExecutionContext): Future[Seq[JsObject]] = { + //Query result from CouchDB have below object structure with actual result in `value` key + //So transform the result to confirm to that structure + val viewResult = JsObject( + "id" -> js.fields("_id"), + "key" -> createKey(ddoc, view, startKey, js), + "value" -> computeView(ddoc, view, js)) + + val result = if (includeDocs) JsObject(viewResult.fields + ("doc" -> js)) else viewResult + Future.successful(Seq(result)) + } + + /** + * Computes the view as per viewName. Its passed either the projected object or actual + * object + */ + def computeView(ddoc: String, view: String, js: JsObject): JsObject + + /** + * Key is an array which matches the view query key + */ + protected def createKey(ddoc: String, view: String, startKey: List[Any], js: JsObject): JsArray +} + +object ActivationHandler extends SimpleHandler { + val NS_PATH = "nspath" + private val commonFields = + Set("namespace", "name", "version", "publish", "annotations", "activationId", "start", "cause") + private val fieldsForView = commonFields ++ Seq("end", "response.statusCode") + + protected val supportedTables = + Set("activations/byDate", "whisks-filters.v2.1.0/activations", "whisks.v2.1.0/activations") + + override def computedFields(js: JsObject): JsObject = { + val path = js.fields.get("namespace") match { + case Some(JsString(namespace)) => JsString(namespace + PATHSEP + pathFilter(js)) + case _ => JsNull + } + val deleteLogs = annotationValue(js, "kind", { v => + v.convertTo[String] != "sequence" + }, true) + dropNull((NS_PATH, path), ("deleteLogs", JsBoolean(deleteLogs))) + } + + override def fieldsRequiredForView(ddoc: String, view: String): Set[String] = view match { + case "activations" => fieldsForView + case _ => throw UnsupportedView(s"$ddoc/$view") + } + + def computeView(ddoc: String, view: String, js: JsObject): JsObject = view match { + case "activations" => computeActivationView(js) + case _ => throw UnsupportedView(s"$ddoc/$view") + } + + def createKey(ddoc: String, view: String, startKey: List[Any], js: JsObject): JsArray = { + startKey match { + case (ns: String) :: Nil => JsArray(Vector(JsString(ns))) + case (ns: String) :: _ :: Nil => JsArray(Vector(JsString(ns), js.fields("start"))) + case _ => throw UnsupportedQueryKeys("$ddoc/$view -> ($startKey, $endKey)") + } + } + + private def computeActivationView(js: JsObject): JsObject = { + val common = js.fields.filterKeys(commonFields) + + val (endTime, duration) = js.getFields("end", "start") match { + case Seq(JsNumber(end), JsNumber(start)) if end != 0 => (JsNumber(end), JsNumber(end - start)) + case _ => (JsNull, JsNull) + } + + val statusCode = JsHelpers.getFieldPath(js, "response", "statusCode").getOrElse(JsNull) + + val result = common + ("end" -> endTime) + ("duration" -> duration) + ("statusCode" -> statusCode) + JsObject(result.filter(_._2 != JsNull)) + } + + protected[database] def pathFilter(js: JsObject): String = { + val name = js.fields("name").convertTo[String] + annotationValue(js, "path", { v => + val p = v.convertTo[String].split(PATHSEP) + if (p.length == 3) p(1) + PATHSEP + name else name + }, name) + } + + /** + * Finds and transforms annotation with matching key. + * + * @param js js object having annotations array + * @param key annotation key + * @param vtr transformer function to map annotation value + * @param default default value to use if no matching annotation found + * @return annotation value matching given key + */ + protected[database] def annotationValue[T](js: JsObject, key: String, vtr: JsValue => T, default: T): T = { + js.fields.get("annotations") match { + case Some(JsArray(e)) => + e.view + .map(_.asJsObject.getFields("key", "value")) + .collectFirst { + case Seq(JsString(`key`), v: JsValue) => vtr(v) //match annotation with given key + } + .getOrElse(default) + case _ => default + } + } + + private def dropNull(fields: JsField*) = JsObject(fields.filter(_._2 != JsNull): _*) +} + +object WhisksHandler extends SimpleHandler { + val ROOT_NS = "rootns" + private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") + private val actionFields = commonFields ++ Set("limits", "exec.binary") + private val packageFields = commonFields ++ Set("binding") + private val packagePublicFields = commonFields + private val ruleFields = commonFields + private val triggerFields = commonFields + + protected val supportedTables = Set( + "whisks.v2.1.0/actions", Review comment: You mean have a separate variable for version and refer to that in name? That can be done Here I wanted to explicitly list the actual version which current implementation is coded for. If any change in version is done in CouchDB views and config in application.conf updated this hardcoding would ensure that we take a conscious update here and ensure that implementations logic is adapted for newer version ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
