vjagadish1989 commented on a change in pull request #1086: SAMZA-2256:
Hotstandby fix for exception while reading side inputs
URL: https://github.com/apache/samza/pull/1086#discussion_r295425942
##########
File path:
samza-core/src/main/scala/org/apache/samza/storage/ContainerStorageManager.java
##########
@@ -704,44 +718,44 @@ public void run() {
// start the systemConsumers for consuming input
this.sideInputSystemConsumers.start();
- // create a thread for sideInput reads
- Thread readSideInputs = new Thread(() -> {
- while (!shutDownSideInputRead) {
- IncomingMessageEnvelope envelope =
sideInputSystemConsumers.choose(true);
- if (envelope != null) {
-
- if (!envelope.isEndOfStream())
-
sideInputStorageManagers.get(envelope.getSystemStreamPartition()).process(envelope);
- checkSideInputCaughtUp(envelope.getSystemStreamPartition(),
envelope.getOffset(),
- SystemStreamMetadata.OffsetType.NEWEST,
envelope.isEndOfStream());
+ try {
- } else {
- LOG.trace("No incoming message was available");
+ // submit the side input read runnable
+ sideInputsReadExecutor.submit(() -> {
+ try {
+ while (!shouldShutdown) {
+ IncomingMessageEnvelope envelope =
sideInputSystemConsumers.choose(true);
+
+ if (envelope != null) {
+ if (!envelope.isEndOfStream()) {
+
sideInputStorageManagers.get(envelope.getSystemStreamPartition()).process(envelope);
+ }
+
+ checkSideInputCaughtUp(envelope.getSystemStreamPartition(),
envelope.getOffset(),
+ SystemStreamMetadata.OffsetType.NEWEST,
envelope.isEndOfStream());
+ } else {
+ LOG.trace("No incoming message was available");
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Exception in reading side inputs", e);
+ sideInputException = e;
}
- }
- });
+ });
- readSideInputs.setDaemon(true);
- readSideInputs.setUncaughtExceptionHandler(new
Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- sideInputException = Optional.of(e);
- sideInputsCaughtUp.countDown();
+ // Make the main thread wait until all sideInputs have been caughtup or
an exception was thrown
+ while (sideInputException == null &&
!this.sideInputsCaughtUp.await(SIDE_INPUT_READ_THREAD_TIMEOUT_SECONDS,
+ TimeUnit.SECONDS)) {
+ LOG.debug("Checking for any side-input-exception occurrence");
}
- });
- try {
- readSideInputs.start();
- // Make the main thread wait until all sideInputs have been caughtup or
thrown an exception
- this.sideInputsCaughtUp.await();
-
- if (sideInputException.isPresent()) { // Throw exception if there was an
exception in catching-up sideInputs
- // TODO: SAMZA-2113 relay exception to main thread
- throw new SamzaException("Exception in restoring side inputs",
sideInputException.get());
+ if (sideInputException
Review comment:
remove the line-break
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
