markusthoemmes commented on a change in pull request #3421: Add ElasticSearch Log Store URL: https://github.com/apache/incubator-openwhisk/pull/3421#discussion_r177029555
########## File path: common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchLogStore.scala ########## @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package whisk.core.containerpool.logging + +import java.nio.file.{Path, Paths} + +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import akka.http.scaladsl.model._ + +import whisk.core.entity.{ActivationLogs, Identity, WhiskActivation} +import whisk.core.containerpool.logging.ElasticSearchJsonProtocol._ +import whisk.core.ConfigKeys + +import scala.concurrent.{Future, Promise} +import scala.util.Try + +import spray.json._ + +import pureconfig._ + +case class ElasticSearchLogStoreConfig(protocol: String, + host: String, + port: Int, + path: String, + logMessageField: String, + activationIdField: String, + streamField: String, + actionField: String, + requiredHeaders: String = "") + +/** + * ElasticSearch based implementation of a DockerToActivationFileLogStore. When using the JSON log driver, docker writes + * stdout/stderr to JSON formatted files. Those files can be processed by a backend service asynchronously to store + * user logs in ElasticSearch. This log store allows user logs then to be fetched from ElasticSearch. + */ +class ElasticSearchLogStore( + system: ActorSystem, + httpFlow: Option[Flow[(HttpRequest, Promise[HttpResponse]), (Try[HttpResponse], Promise[HttpResponse]), Any]] = None, + destinationDirectory: Path = Paths.get("logs"), + elasticSearchConfig: ElasticSearchLogStoreConfig = + loadConfigOrThrow[ElasticSearchLogStoreConfig](ConfigKeys.elasticSearch)) + extends DockerToActivationFileLogStore(system, destinationDirectory) { + + // Schema of logs in ES + case class UserLogEntry(message: String, stream: String, time: String, action: String) + + object UserLogEntry extends DefaultJsonProtocol { + implicit val serdes = + jsonFormat( + UserLogEntry.apply, + "message", + elasticSearchConfig.streamField, + "time_date", + elasticSearchConfig.actionField) + } + + implicit val actorSystem = system + + private val esClient = new ElasticSearchRestClient( + elasticSearchConfig.protocol, + elasticSearchConfig.host, + elasticSearchConfig.port, + httpFlow) + private val requiredHeaders = elasticSearchConfig.requiredHeaders match { + case headers if !headers.isEmpty => headers.split(",") + case _ => Array.empty[String] + } + private val logQuery = + s"_type: ${elasticSearchConfig.logMessageField} AND ${elasticSearchConfig.activationIdField}: %s" + + private def transcribeLogs(queryResult: EsSearchResult): ActivationLogs = { + val logs = queryResult.hits.hits.map(hit => { + val userLogEntry = hit.source.convertTo[UserLogEntry] + s"${userLogEntry.time} ${userLogEntry.stream}: ${userLogEntry.message.stripLineEnd}" + }) + + ActivationLogs(logs) + } + + private def extractRequiredHeaders(headers: Seq[HttpHeader]) = + headers.filter { + case header: HttpHeader if requiredHeaders.contains(header.lowercaseName) => true + case _ => false + }.toList + + private def generatePayload(activation: WhiskActivation) = { + val queryString = EsQueryString(logQuery.format(activation.activationId)) + val queryOrder = EsQueryOrder("time_date", EsOrderAsc) + + EsQuery(queryString, Some(queryOrder)).toJson.asJsObject + } + + private def generatePath(user: Identity) = { + Uri(elasticSearchConfig.path.replace("$UUID", user.uuid.asString)) + } Review comment: replace will create new Regex Matchers and run them each time this is called. Should we rather pass in an interpolated string and use the native `format` like ``` elasticSearchConfig.path.format(user.uuid.asString) ``` which can then be defined as ``` /test/index-%s-%s ``` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
