m-trieu commented on code in PR #31133:
URL: https://github.com/apache/beam/pull/31133#discussion_r1593187770
##########
runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/config/StreamingEngineComputationConfigFetcher.java:
##########
@@ -102,49 +112,112 @@ public static StreamingEngineComputationConfigFetcher
forTesting(
hasReceivedGlobalConfig,
globalConfigRefreshPeriodMillis,
dataflowServiceClient,
- executorSupplier.apply(GLOBAL_PIPELINE_CONFIG_REFRESHER),
+ executorSupplier.apply(CONFIG_REFRESHER_THREAD_NAME),
onStreamingConfig);
}
- private static BackOff defaultConfigBackoff() {
- return FluentBackoff.DEFAULT
- .withInitialBackoff(Duration.millis(100))
- .withMaxBackoff(Duration.standardMinutes(1))
- .withMaxCumulativeBackoff(Duration.standardMinutes(5))
- .backoff();
- }
-
- private MapTask createMapTask(StreamingComputationConfig computationConfig) {
+ @VisibleForTesting
+ static MapTask createMapTask(StreamingComputationConfig computationConfig) {
return new MapTask()
.setSystemName(computationConfig.getSystemName())
.setStageName(computationConfig.getStageName())
.setInstructions(computationConfig.getInstructions());
}
+ private static Optional<StreamingConfigTask> fetchConfigWithRetry(
+ ThrowingFetchWorkItemFn fetchWorkItemFn) {
+ BackOff backoff = BACKOFF_FACTORY.backoff();
+ while (true) {
+ try {
+ return fetchWorkItemFn
+ .fetchWorkItem()
+ .map(
+ workItem -> {
+ StreamingConfigTask config =
workItem.getStreamingConfigTask();
+ Preconditions.checkState(
+ config != null,
+ "Received invalid WorkItem without StreamingConfigTask.
WorkItem={}",
+ workItem);
+ return config;
+ });
+ } catch (IllegalArgumentException | IOException e) {
+ LOG.warn("Error fetching config: ", e);
+ try {
+ if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) {
+ return Optional.empty();
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Error backing off, will not retry: ", ioe);
+ return Optional.empty();
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ return Optional.empty();
+ }
+ }
+ }
+ }
+
+ private static Optional<StreamingEnginePipelineConfig> createPipelineConfig(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
