MAHOUT-1821: Use a mahout-flink-conf.yaml configuration file for Mahout specific Flink configuration
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/7c275f06 Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/7c275f06 Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/7c275f06 Branch: refs/heads/master Commit: 7c275f06e618624ac5bdc269b19d328a6c2b6983 Parents: 00a09dc Author: smarthi <[email protected]> Authored: Fri Apr 8 20:24:04 2016 -0400 Committer: smarthi <[email protected]> Committed: Fri Apr 8 20:24:04 2016 -0400 ---------------------------------------------------------------------- conf/flink-config.yaml | 67 ++++++++++++++++++++ .../flinkbindings/DistributedFlinkSuite.scala | 3 - 2 files changed, 67 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/7c275f06/conf/flink-config.yaml ---------------------------------------------------------------------- diff --git a/conf/flink-config.yaml b/conf/flink-config.yaml new file mode 100644 index 0000000..968cb04 --- /dev/null +++ b/conf/flink-config.yaml @@ -0,0 +1,67 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +#============================================================================== +# Common +#============================================================================== + +# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline. + +taskmanager.numberOfTaskSlots: 1 + +# The parallelism used for programs that did not specify and other parallelism. + +parallelism.default: 4 + +#============================================================================== +# Advanced +#============================================================================== + +# The number of buffers for the network stack. +# +# taskmanager.network.numberOfBuffers: 2048 + + +# Directories for temporary files. +# +# Add a delimited list for multiple directories, using the system directory +# delimiter (colon ':' on unix) or a comma, e.g.: +# /data1/tmp:/data2/tmp:/data3/tmp +# +# Note: Each directory entry is read from and written to by a different I/O +# thread. You can include the same directory multiple times in order to create +# multiple I/O threads against that directory. This is for example relevant for +# high-throughput RAIDs. +# +# If not specified, the system-specific Java temporary directory (java.io.tmpdir +# property) is taken. +# +taskmanager.tmp.dirs: /tmp/mahoutcache + + +# Path to the Hadoop configuration directory. +# +# This configuration is used when writing into HDFS. Unless specified otherwise, +# HDFS file creation will use HDFS default settings with respect to block-size, +# replication factor, etc. +# +# You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml +# via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'. +# +# fs.hdfs.hadoopconf: /path/to/hadoop/conf/ http://git-wip-us.apache.org/repos/asf/mahout/blob/7c275f06/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala ---------------------------------------------------------------------- diff --git a/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala b/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala index 480a256..094c45b 100644 --- a/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala +++ b/flink/src/test/scala/org/apache/mahout/flinkbindings/DistributedFlinkSuite.scala @@ -28,7 +28,6 @@ import org.scalatest.{ConfigMap, Suite} import scala.concurrent.duration.FiniteDuration - trait DistributedFlinkSuite extends DistributedMahoutSuite { this: Suite => protected implicit var mahoutCtx: DistributedContext = _ @@ -49,12 +48,10 @@ trait DistributedFlinkSuite extends DistributedMahoutSuite { this: Suite => override def afterEach() { super.afterEach() -// env.execute("Mahout Flink Binding Test Suite") } override protected def afterAll(configMap: ConfigMap): Unit = { super.afterAll(configMap) -// resetContext() cluster.foreach(c => TestBaseUtils.stopCluster(c, DEFAULT_TIMEOUT)) }
