sven-lange-last commented on a change in pull request #2878: Streamingly read
user-logs.
URL:
https://github.com/apache/incubator-openwhisk/pull/2878#discussion_r152211268
##########
File path:
core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
##########
@@ -224,31 +230,73 @@ class DockerContainer(protected val id: ContainerId,
*
* @return a vector of Strings with log lines in our own JSON format
*/
- def logs(limit: ByteSize, waitForSentinel: Boolean)(implicit transid:
TransactionId): Future[Vector[String]] = {
-
- def readLogs(retries: Int): Future[Vector[String]] = {
- docker
- .rawContainerLogs(id, logFileOffset)
- .flatMap { rawLogBytes =>
- val rawLog =
- new String(rawLogBytes.array, rawLogBytes.arrayOffset,
rawLogBytes.position, StandardCharsets.UTF_8)
- val (isComplete, isTruncated, formattedLogs) =
processJsonDriverLogContents(rawLog, waitForSentinel, limit)
-
- if (retries > 0 && !isComplete && !isTruncated) {
- logging.info(this, s"log cursor advanced but missing sentinel,
trying $retries more times")
- akka.pattern.after(filePollInterval,
as.scheduler)(readLogs(retries - 1))
- } else {
- logFileOffset += rawLogBytes.position - rawLogBytes.arrayOffset
- Future.successful(formattedLogs)
- }
- }
- .andThen {
- case Failure(e) =>
- logging.error(this, s"Failed to obtain logs of ${id.asString}:
${e.getClass} - ${e.getMessage}")
- }
+ def logs(limit: ByteSize, waitForSentinel: Boolean)(implicit transid:
TransactionId): Source[ByteString, Any] = {
+ val source = docker
+ .rawContainerLogs(id, logFileOffset.get(), if (waitForSentinel)
Some(filePollInterval) else None)
+ .via(Framing.delimiter(delimiter, Int.MaxValue))
+ .limitWeighted(limit.toBytes) { obj =>
+ val size = obj.size + 1
+ logFileOffset.addAndGet(size)
+ size
+ }
+ .recover {
+ case _: StreamLimitReachedException =>
+ ByteString(LogLine(Instant.now.toString, "stderr",
Messages.truncateLogs(limit)).toJson.compactPrint)
+ }
+
+ // Only apply sentinel counting if it is needed
+ val specializedSource = if (waitForSentinel) {
+ source.via(new
WaitForOccurances(_.containsSlice(DockerContainer.ActivationSentinel), 2))
+ } else {
+ source
}
- readLogs((waitForLogs / filePollInterval).toInt)
+ specializedSource.takeWithin(waitForLogs)
}
+ /* Delimiter used to split logs */
+ private val delimiter = ByteString("\n")
+}
+
+/**
+ * Completes the stream once the given predicate is fulfilled by N events in
the stream.
+ *
+ * '''Emits when''' an upstream element arrives and does not fulfill the
predicate
+ *
+ * '''Backpressures when''' downstream backpressures
+ *
+ * '''Completes when''' upstream completes or predicate is fulfilled N times
+ *
+ * '''Cancels when''' downstream cancels
+ */
+class WaitForOccurances[T](isInEvent: T => Boolean, neededOccurances: Int)
extends GraphStage[FlowShape[T, T]] {
+ val in = Inlet[T]("WaitForOccurances.in")
+ val out = Outlet[T]("WaitForOccurances.out")
+ override val shape = FlowShape.of(in, out)
+
+ override def createLogic(inheritedAttributes: Attributes): GraphStageLogic =
+ new GraphStageLogic(shape) with InHandler with OutHandler {
+ private var occurancesFound = 0
+
+ override def onPull(): Unit = pull(in)
+
+ override def onPush(): Unit = {
+ val event = grab(in)
Review comment:
I suggest `element` instead of `event`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
