sven-lange-last commented on a change in pull request #2878: Streamingly read
user-logs.
URL:
https://github.com/apache/incubator-openwhisk/pull/2878#discussion_r152203173
##########
File path:
core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
##########
@@ -224,31 +230,73 @@ class DockerContainer(protected val id: ContainerId,
*
* @return a vector of Strings with log lines in our own JSON format
*/
- def logs(limit: ByteSize, waitForSentinel: Boolean)(implicit transid:
TransactionId): Future[Vector[String]] = {
-
- def readLogs(retries: Int): Future[Vector[String]] = {
- docker
- .rawContainerLogs(id, logFileOffset)
- .flatMap { rawLogBytes =>
- val rawLog =
- new String(rawLogBytes.array, rawLogBytes.arrayOffset,
rawLogBytes.position, StandardCharsets.UTF_8)
- val (isComplete, isTruncated, formattedLogs) =
processJsonDriverLogContents(rawLog, waitForSentinel, limit)
-
- if (retries > 0 && !isComplete && !isTruncated) {
- logging.info(this, s"log cursor advanced but missing sentinel,
trying $retries more times")
- akka.pattern.after(filePollInterval,
as.scheduler)(readLogs(retries - 1))
- } else {
- logFileOffset += rawLogBytes.position - rawLogBytes.arrayOffset
- Future.successful(formattedLogs)
- }
- }
- .andThen {
- case Failure(e) =>
- logging.error(this, s"Failed to obtain logs of ${id.asString}:
${e.getClass} - ${e.getMessage}")
- }
+ def logs(limit: ByteSize, waitForSentinel: Boolean)(implicit transid:
TransactionId): Source[ByteString, Any] = {
+ val source = docker
+ .rawContainerLogs(id, logFileOffset.get(), if (waitForSentinel)
Some(filePollInterval) else None)
+ .via(Framing.delimiter(delimiter, Int.MaxValue))
+ .limitWeighted(limit.toBytes) { obj =>
+ val size = obj.size + 1
+ logFileOffset.addAndGet(size)
+ size
+ }
+ .recover {
+ case _: StreamLimitReachedException =>
+ ByteString(LogLine(Instant.now.toString, "stderr",
Messages.truncateLogs(limit)).toJson.compactPrint)
+ }
+
+ // Only apply sentinel counting if it is needed
+ val specializedSource = if (waitForSentinel) {
+ source.via(new
WaitForOccurances(_.containsSlice(DockerContainer.ActivationSentinel), 2))
+ } else {
+ source
}
- readLogs((waitForLogs / filePollInterval).toInt)
+ specializedSource.takeWithin(waitForLogs)
}
+ /* Delimiter used to split logs */
+ private val delimiter = ByteString("\n")
+}
+
+/**
+ * Completes the stream once the given predicate is fulfilled by N events in
the stream.
+ *
+ * '''Emits when''' an upstream element arrives and does not fulfill the
predicate
+ *
+ * '''Backpressures when''' downstream backpressures
+ *
+ * '''Completes when''' upstream completes or predicate is fulfilled N times
+ *
+ * '''Cancels when''' downstream cancels
+ */
+class WaitForOccurances[T](isInEvent: T => Boolean, neededOccurances: Int)
extends GraphStage[FlowShape[T, T]] {
Review comment:
* Typo: _Occurrence_ instead of _Occurrance_. See
https://en.oxforddictionaries.com/definition/occurrence
* I wonder whether it's reasonable to use _wait_ in the flow's name. The
name's connotation is to expect that the flow waits until the predicate is
fulfilled and starts emitting elements. Maybe `completeOnOccurrence` expresses
the intended semantics better?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
