Switch to using a FixedThreadPool by default Reduces the number of active threads, which should improve scheduling behavior.
Additionally improves caching behavior of ParDo evaluators due to increased reuse of DoFns. Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9a3fe343 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9a3fe343 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9a3fe343 Branch: refs/heads/master Commit: 9a3fe3433062052138c0e5fbfc4a081b3c8e3715 Parents: 2fbc0ea Author: Thomas Groh <[email protected]> Authored: Tue May 10 15:09:00 2016 -0700 Committer: Thomas Groh <[email protected]> Committed: Thu May 12 09:59:44 2016 -0700 ---------------------------------------------------------------------- .../CachedThreadPoolExecutorServiceFactory.java | 44 ------------------- .../FixedThreadPoolExecutorServiceFactory.java | 45 ++++++++++++++++++++ .../direct/InProcessPipelineOptions.java | 4 +- 3 files changed, 47 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9a3fe343/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CachedThreadPoolExecutorServiceFactory.java ---------------------------------------------------------------------- diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CachedThreadPoolExecutorServiceFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CachedThreadPoolExecutorServiceFactory.java deleted file mode 100644 index 5b8e5fc..0000000 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/CachedThreadPoolExecutorServiceFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.beam.runners.direct; - -import org.apache.beam.sdk.options.DefaultValueFactory; -import org.apache.beam.sdk.options.PipelineOptions; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * A {@link ExecutorServiceFactory} that produces cached thread pools via - * {@link Executors#newCachedThreadPool()}. - */ -class CachedThreadPoolExecutorServiceFactory - implements DefaultValueFactory<ExecutorServiceFactory>, ExecutorServiceFactory { - private static final CachedThreadPoolExecutorServiceFactory INSTANCE = - new CachedThreadPoolExecutorServiceFactory(); - - @Override - public ExecutorServiceFactory create(PipelineOptions options) { - return INSTANCE; - } - - @Override - public ExecutorService create() { - return Executors.newCachedThreadPool(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9a3fe343/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FixedThreadPoolExecutorServiceFactory.java ---------------------------------------------------------------------- diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FixedThreadPoolExecutorServiceFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FixedThreadPoolExecutorServiceFactory.java new file mode 100644 index 0000000..74c4292 --- /dev/null +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/FixedThreadPoolExecutorServiceFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.direct; + +import org.apache.beam.sdk.options.DefaultValueFactory; +import org.apache.beam.sdk.options.PipelineOptions; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * A {@link ExecutorServiceFactory} that produces fixed thread pools via + * {@link Executors#newFixedThreadPool(int)}, with the number of threads equal to the available + * processors as provided by {@link Runtime#availableProcessors()}. + */ +class FixedThreadPoolExecutorServiceFactory + implements DefaultValueFactory<ExecutorServiceFactory>, ExecutorServiceFactory { + private static final FixedThreadPoolExecutorServiceFactory INSTANCE = + new FixedThreadPoolExecutorServiceFactory(); + + @Override + public ExecutorServiceFactory create(PipelineOptions options) { + return INSTANCE; + } + + @Override + public ExecutorService create() { + return Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9a3fe343/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java ---------------------------------------------------------------------- diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java index 512b3bd..0498521 100644 --- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java +++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/InProcessPipelineOptions.java @@ -43,13 +43,13 @@ public interface InProcessPipelineOptions extends PipelineOptions, ApplicationNa * it cannot enter a state in which it will not schedule additional pending work unless currently * scheduled work completes, as this may cause the {@link Pipeline} to cease processing. * - * <p>Defaults to a {@link CachedThreadPoolExecutorServiceFactory}, which produces instances of + * <p>Defaults to a {@link FixedThreadPoolExecutorServiceFactory}, which produces instances of * {@link Executors#newCachedThreadPool()}. */ @JsonIgnore @Required @Hidden - @Default.InstanceFactory(CachedThreadPoolExecutorServiceFactory.class) + @Default.InstanceFactory(FixedThreadPoolExecutorServiceFactory.class) ExecutorServiceFactory getExecutorServiceFactory(); void setExecutorServiceFactory(ExecutorServiceFactory executorService);
