cbickel closed pull request #3436: move cleanup logic of cloudant/kafka trigger dbs to a different location URL: https://github.com/apache/incubator-openwhisk/pull/3436
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/tests/src/test/scala/whisk/core/database/test/CleanUpWhisksDbSkriptTests.scala b/tests/src/test/scala/whisk/core/database/test/CleanUpWhisksDbSkriptTests.scala deleted file mode 100644 index 1df1a485ec..0000000000 --- a/tests/src/test/scala/whisk/core/database/test/CleanUpWhisksDbSkriptTests.scala +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package whisk.core.database.test - -import java.io.File -import java.time.Instant -import java.time.temporal.ChronoUnit - -import common.{StreamLogging, TestUtils, WhiskProperties, WskActorSystem} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{FlatSpec, Matchers} -import pureconfig.loadConfigOrThrow -import spray.json._ -import spray.json.DefaultJsonProtocol._ -import whisk.core.database.CouchDbConfig -//import whisk.core.{ConfigKeys, WhiskConfig} -import whisk.core.ConfigKeys -import whisk.core.entity._ - -@RunWith(classOf[JUnitRunner]) -class CleanUpWhisksDbSkriptTests - extends FlatSpec - with Matchers - with DatabaseScriptTestUtils - with WskActorSystem - with StreamLogging { - - val cleanupScript = WhiskProperties.getFileRelativeToWhiskHome("tools/db/cleanUpWhisks.py").getAbsolutePath - val dbConfig = loadConfigOrThrow[CouchDbConfig](ConfigKeys.couchdb) - val authDBName = dbConfig.databaseFor[WhiskAuth] - - def runScript(dbUrl: String, whisksDbName: String, subjectsDbName: String) = { - println(s"Running script: $dbUrl, $whisksDbName, $subjectsDbName") - - val cmd = - Seq( - python, - cleanupScript, - "--dbUrl", - dbUrl, - "--dbNameWhisks", - whisksDbName, - "--dbNameSubjects", - subjectsDbName, - "--days", - "1", - "--docsPerRequest", - "1") - - val rr = TestUtils.runCmd(0, new File("."), cmd: _*) - - val Seq(marked, deleted, skipped, kept) = - Seq("marking: ", "deleting: ", "skipping: ", "keeping: ").map { linePrefix => - rr.stdout.lines.collect { - case line if line.startsWith(linePrefix) => line.replace(linePrefix, "") - }.toList - } - - println(s"marked: $marked") - println(s"deleted: $deleted") - println(s"skipped: $skipped") - println(s"kept: $kept") - - (marked, deleted, skipped, kept) - } - - behavior of "Cleanup whisksDb script" - - it should "mark documents for deletion if namespace does not exist" in { - // Create whisks db - val dbName = dbPrefix + "cleanup_whisks_test_mark_for_deletion" - val client = createDatabase(dbName, None) - - // Create document/action with random namespace - val documents = Map( - "whisksCleanTests/utils/actionName1" -> JsObject("namespace" -> JsString("whisksCleanTests/utils")), - "whisksCleanTests/utils/actionName2" -> JsObject("namespace" -> JsString("whisksCleanTests")), - "whisksCleanTests/actionName3" -> JsObject("namespace" -> JsString("whisksCleanTests"))) - - documents.foreach { - case (id, document) => - client.putDoc(id, document).futureValue - } - - // execute script - val (marked, _, _, _) = runScript(dbUrl, dbName, authDBName) - println(s"marked: $marked") - - // Check, that script marked document to be deleted: output + document from DB - val ids = documents.keys - println(s"ids: $ids") - marked should contain allElementsOf ids - - val databaseResponse = client.getAllDocs(includeDocs = Some(true)).futureValue - databaseResponse should be('right) - val databaseDocuments = databaseResponse.right.get.fields("rows").convertTo[List[JsObject]] - - databaseDocuments.foreach { doc => - doc.fields("doc").asJsObject.fields.keys should contain("markedForDeletion") - } - - // Delete database - client.deleteDb().futureValue - } - - it should "delete marked for deletion documents if namespace does not exists" in { - // Create whisks db - val dbName = dbPrefix + "cleanup_whisks_test_delete_mark_for_deletion" - val client = createDatabase(dbName, None) - - // Create document/action with random namespace and markedForDeletion field - val documents = Map( - "whisksCleanTests/utils/actionName1" -> JsObject( - "namespace" -> JsString("whisksCleanTests/utils"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli)), - "whisksCleanTests/utils/actionName2" -> JsObject( - "namespace" -> JsString("whisksCleanTests"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli)), - "whisksCleanTests/actionName3" -> JsObject( - "namespace" -> JsString("whisksCleanTests"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli))) - - documents.foreach { - case (id, document) => - client.putDoc(id, document).futureValue - } - - // execute script - val (marked, deleted, _, _) = runScript(dbUrl, dbName, authDBName) - println(s"marked: $marked") - println(s"deleted: $deleted") - - // Check, that script deleted already marked documents from DB - val ids = documents.keys - println(s"ids: $ids") - marked shouldBe empty - - val databaseResponse = client.getAllDocs(includeDocs = Some(true)).futureValue - databaseResponse should be('right) - - val databaseDocuments = databaseResponse.right.get.fields("rows").convertTo[List[JsObject]] - databaseDocuments shouldBe empty - - // Delete database - client.deleteDb().futureValue - } - - it should "not mark documents for deletion if namespace does exist" in { - // Create whisks db - val dbName = dbPrefix + "cleanup_whisks_test_not_mark_for_deletion" - val client = createDatabase(dbName, None) - - // Create document/action with whisk-system namespace - val documents = Map( - "whisk.system/utils" -> JsObject("namespace" -> JsString("whisk.system")), - "whisk.system/samples/helloWorld" -> JsObject("namespace" -> JsString("whisk.system/samples")), - "whisk.system/utils/namespace" -> JsObject("namespace" -> JsString("whisk.system/utils"))) - - documents.foreach { - case (id, document) => - client.putDoc(id, document).futureValue - } - - // execute script - val (_, _, _, kept) = runScript(dbUrl, dbName, authDBName) - println(s"kept: $kept") - - // Check, that script did not mark documents for deletion - val ids = documents.keys - println(s"ids: $ids") - kept should contain allElementsOf ids - - val databaseResponse = client.getAllDocs(includeDocs = Some(true)).futureValue - databaseResponse should be('right) - - val databaseDocuments = databaseResponse.right.get.fields("rows").convertTo[List[JsObject]] - val databaseDocumentIDs = databaseDocuments.map(_.fields("id").convertTo[String]) - databaseDocumentIDs should contain allElementsOf ids - - // Delete database - client.deleteDb().futureValue - } - - it should "skip design documents" in { - // Create whisks db - val dbName = dbPrefix + "cleanup_whisks_test_skip_design_documents" - val client = createDatabase(dbName, None) - - // Create design documents - val documents = Map( - "_design/all-whisks.v2.1.0" -> JsObject("language" -> JsString("javascript")), - "_design/snapshotFilters" -> JsObject("language" -> JsString("javascript")), - "_design/whisks.v2.1.0" -> JsObject("language" -> JsString("javascript"))) - - documents.foreach { - case (id, document) => - client.putDoc(id, document).futureValue - } - - // execute script - val (_, _, skipped, _) = runScript(dbUrl, dbName, authDBName) - println(s"skipped: $skipped") - - // Check, that script skipped design documents - val ids = documents.keys - println(s"ids: $ids") - skipped should contain allElementsOf ids - - val databaseResponse = client.getAllDocs(includeDocs = Some(true)).futureValue - databaseResponse should be('right) - - val databaseDocuments = databaseResponse.right.get.fields("rows").convertTo[List[JsObject]] - - val databaseDocumentIDs = databaseDocuments.map(_.fields("id").convertTo[String]) - databaseDocumentIDs should contain allElementsOf ids - - // Delete database - client.deleteDb().futureValue - } - - it should "not delete marked for deletion documents if namespace does exists" in { - // Create whisks db - val dbName = dbPrefix + "cleanup_whisks_test_not_delete_mark_for_deletion" - val client = createDatabase(dbName, None) - - // Create document/action with whisk-system namespace and markedForDeletion field - val documents = Map( - "whisk.system/utils" -> JsObject( - "namespace" -> JsString("whisk.system"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli)), - "whisk.system/samples/helloWorld" -> JsObject( - "namespace" -> JsString("whisk.system/samples"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli)), - "whisk.system/utils/namespace" -> JsObject( - "namespace" -> JsString("whisk.system/utils"), - "markedForDeletion" -> JsNumber(Instant.now().minus(8, ChronoUnit.DAYS).toEpochMilli))) - - documents.foreach { - case (id, document) => - client.putDoc(id, document).futureValue - } - - // execute script - val (_, _, _, kept) = runScript(dbUrl, dbName, authDBName) - println(s"kept: $kept") - - // Check, that script kept documents in DB - val ids = documents.keys - println(s"ids: $ids") - kept should contain allElementsOf ids - - val databaseResponse = client.getAllDocs(includeDocs = Some(true)).futureValue - databaseResponse should be('right) - - val databaseDocuments = databaseResponse.right.get.fields("rows").convertTo[List[JsObject]] - - val databaseDocumentIDs = databaseDocuments.map(_.fields("id").convertTo[String]) - databaseDocumentIDs should contain allElementsOf ids - - // Delete database - client.deleteDb().futureValue - } -} diff --git a/tools/db/cleanUpWhisks.py b/tools/db/cleanUpWhisks.py deleted file mode 100755 index 83b47bb6a0..0000000000 --- a/tools/db/cleanUpWhisks.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python -"""Python script to delete whisks entries having none existent ns. - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -""" - -import argparse -import time -import couchdb.client - -skipWhisks = 0 - -try: - long # Python 2 -except NameError: - long = int # Python 3 - -HOUR = 1000 * 60 * 60 -DAY = HOUR * 24 - -# -# simple ring buffer like list -# -class SimpleRingBuffer: - def __init__(self, size): - self.index = -1 - self.data = [] - self.maxsize = size * 2 - - def append(self, ns, bool): - - self.index=(self.index+2)%self.maxsize - - if len(self.data) < self.maxsize: - self.data.append(ns) - self.data.append(bool) - else: - self.data[self.index-1]=ns - self.data[(self.index)]=bool - - def getself(self): - return self.data - - def get(self, ns): - if ns in self.data: - return self.data[self.data.index(ns)+1] - else: - return None - -# -# mark whisks entry for deletion of delete if already marked -# -def deleteWhisk(dbWhisks, wdoc): - - global skipWhisks - - wdocd = dbWhisks[wdoc['id']] - if not 'markedForDeletion' in wdocd: - print('marking: {0}'.format(wdoc['id'])) - dts = int(time.time() * 1000) - wdocd['markedForDeletion'] = dts - dbWhisks.save(wdocd) - else: - dts = wdocd['markedForDeletion'] - now = int(time.time() * 1000) - elapsedh = int((now - dts) / HOUR) - elapsedd = int((now - dts) / DAY) - - if elapsedd >= args.days: - print('deleting: {0}'.format(wdoc['id'])) - dbWhisks.delete(wdocd) - skipWhisks-=1 - else: - print('marked: {0}, elapsed hours: {1}, elapsed days: {2}'.format(wdoc['id'], elapsedh, elapsedd)) - - -# -# check subjects db for existence of ns -# -def checkNamespace(dbSubjects, namespace): - - while True: - - allNamespaces = dbSubjects.view('subjects/identities', startkey=[namespace], endkey=[namespace]) - - if allNamespaces: - return True - else: - return False - - -# -# check whisks db for entries having none existent ns -# -def checkWhisks(args): - - dbWhisks = couchdb.client.Server(args.dbUrl)[args.dbNameWhisks] - dbSubjects = couchdb.client.Server(args.dbUrl)[args.dbNameSubjects] - - rb = SimpleRingBuffer(args.bufferLen) - - global skipWhisks - while True: - allWhisks = dbWhisks.view('_all_docs', limit=args.docsPerRequest, skip=skipWhisks) - skipWhisks += args.docsPerRequest - if allWhisks: - for wdoc in allWhisks: - if wdoc['id'].startswith('_design/'): - print('skipping: {0}'.format(wdoc['id'])) - continue - namespace = wdoc['id'][0:wdoc['id'].find('/')] - - exists = rb.get(namespace) - if exists == None: - exists = checkNamespace(dbSubjects, namespace) - rb.append(namespace, exists) - - if exists: - print('keeping: {0}'.format(wdoc['id'])) - else: - deleteWhisk(dbWhisks, wdoc) - else: - return - - -parser = argparse.ArgumentParser(description="Utility to mark/delete whisks entries where the ns does not exist in the subjects database.") -parser.add_argument("--dbUrl", required=True, help="Server URL of the database, that has to be cleaned of old activations. E.g. 'https://xxx:[email protected]:443'") -parser.add_argument("--dbNameWhisks", required=True, help="Name of the Whisks Database of the whisks entries to be marked for deletion or deleted if already marked.") -parser.add_argument("--dbNameSubjects", required=True, help="Name of the Subjects Database.") -parser.add_argument("--days", required=True, type=int, default=7, help="How many days whisks keep entries marked for deletion before deleting them.") -parser.add_argument("--docsPerRequest", type=int, default=200, help="Number of documents handled on each CouchDb Request. Default is 200.") -parser.add_argument("--bufferLen", type=int, default=100, help="Maximum buffer length to cache already checked ns. Default is 100.") -args = parser.parse_args() - -checkWhisks(args) ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
