Repository: mahout Updated Branches: refs/heads/master e73fdb869 -> 2f6399f3f
MAHOUT-1881:flink-config.yaml is not copied to /conf in Binary Distro, this closes apache/mahout#255 Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/2f6399f3 Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/2f6399f3 Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/2f6399f3 Branch: refs/heads/master Commit: 2f6399f3fd36f5074c2f4892407ed9d4f889fa07 Parents: e73fdb8 Author: smarthi <[email protected]> Authored: Thu Sep 8 10:55:04 2016 -0400 Committer: smarthi <[email protected]> Committed: Thu Sep 8 10:55:04 2016 -0400 ---------------------------------------------------------------------- conf/flink-config.yaml | 67 -------------------- .../flinkbindings/FlinkDistributedContext.scala | 2 +- .../mahout/flinkbindings/blas/FlinkOpAtA.scala | 4 +- .../blas/FlinkOpTimesRightMatrix.scala | 2 +- src/conf/flink-config.yaml | 67 ++++++++++++++++++++ 5 files changed, 71 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/2f6399f3/conf/flink-config.yaml ---------------------------------------------------------------------- diff --git a/conf/flink-config.yaml b/conf/flink-config.yaml deleted file mode 100644 index 968cb04..0000000 --- a/conf/flink-config.yaml +++ /dev/null @@ -1,67 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - - -#============================================================================== -# Common -#============================================================================== - -# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline. - -taskmanager.numberOfTaskSlots: 1 - -# The parallelism used for programs that did not specify and other parallelism. - -parallelism.default: 4 - -#============================================================================== -# Advanced -#============================================================================== - -# The number of buffers for the network stack. -# -# taskmanager.network.numberOfBuffers: 2048 - - -# Directories for temporary files. -# -# Add a delimited list for multiple directories, using the system directory -# delimiter (colon ':' on unix) or a comma, e.g.: -# /data1/tmp:/data2/tmp:/data3/tmp -# -# Note: Each directory entry is read from and written to by a different I/O -# thread. You can include the same directory multiple times in order to create -# multiple I/O threads against that directory. This is for example relevant for -# high-throughput RAIDs. -# -# If not specified, the system-specific Java temporary directory (java.io.tmpdir -# property) is taken. -# -taskmanager.tmp.dirs: /tmp/mahoutcache - - -# Path to the Hadoop configuration directory. -# -# This configuration is used when writing into HDFS. Unless specified otherwise, -# HDFS file creation will use HDFS default settings with respect to block-size, -# replication factor, etc. -# -# You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml -# via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'. -# -# fs.hdfs.hadoopconf: /path/to/hadoop/conf/ http://git-wip-us.apache.org/repos/asf/mahout/blob/2f6399f3/flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkDistributedContext.scala ---------------------------------------------------------------------- diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkDistributedContext.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkDistributedContext.scala index 8338080..7a61ee6 100644 --- a/flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkDistributedContext.scala +++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/FlinkDistributedContext.scala @@ -27,7 +27,7 @@ class FlinkDistributedContext(val env: ExecutionEnvironment) extends Distributed val mahoutHome = getMahoutHome() - GlobalConfiguration.loadConfiguration(mahoutHome + "/conf/flink-config.yaml") + GlobalConfiguration.loadConfiguration(mahoutHome + "/src/conf/flink-config.yaml") val conf = GlobalConfiguration.getConfiguration http://git-wip-us.apache.org/repos/asf/mahout/blob/2f6399f3/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala ---------------------------------------------------------------------- diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala index 4dc265f..10f1b92 100644 --- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala +++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpAtA.scala @@ -134,7 +134,7 @@ object FlinkOpAtA { val subresults: DataSet[(Int, Matrix)] = ds.flatMap(new RichFlatMapFunction[(Array[K], Matrix), (Int, Matrix)] { - var ranges: Array[Range] = null + var ranges: Array[Range] = _ override def open(params: Configuration): Unit = { val runtime = this.getRuntimeContext @@ -157,7 +157,7 @@ object FlinkOpAtA { val res = subresults.groupBy(0) .reduceGroup(new RichGroupReduceFunction[(Int, Matrix), BlockifiedDrmTuple[Int]] { - var ranges: Array[Range] = null + var ranges: Array[Range] = _ override def open(params: Configuration): Unit = { val runtime = this.getRuntimeContext http://git-wip-us.apache.org/repos/asf/mahout/blob/2f6399f3/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala ---------------------------------------------------------------------- diff --git a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala index 2ad7bd6..a4c5373 100644 --- a/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala +++ b/flink/src/main/scala/org/apache/mahout/flinkbindings/blas/FlinkOpTimesRightMatrix.scala @@ -50,7 +50,7 @@ object FlinkOpTimesRightMatrix { val singletonDataSetB = ctx.env.fromCollection(rows) val res = A.asBlockified.ds.map(new RichMapFunction[(Array[K], Matrix), (Array[K], Matrix)] { - var inCoreB: Matrix = null + var inCoreB: Matrix = _ override def open(params: Configuration): Unit = { val runtime = this.getRuntimeContext http://git-wip-us.apache.org/repos/asf/mahout/blob/2f6399f3/src/conf/flink-config.yaml ---------------------------------------------------------------------- diff --git a/src/conf/flink-config.yaml b/src/conf/flink-config.yaml new file mode 100644 index 0000000..968cb04 --- /dev/null +++ b/src/conf/flink-config.yaml @@ -0,0 +1,67 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +#============================================================================== +# Common +#============================================================================== + +# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline. + +taskmanager.numberOfTaskSlots: 1 + +# The parallelism used for programs that did not specify and other parallelism. + +parallelism.default: 4 + +#============================================================================== +# Advanced +#============================================================================== + +# The number of buffers for the network stack. +# +# taskmanager.network.numberOfBuffers: 2048 + + +# Directories for temporary files. +# +# Add a delimited list for multiple directories, using the system directory +# delimiter (colon ':' on unix) or a comma, e.g.: +# /data1/tmp:/data2/tmp:/data3/tmp +# +# Note: Each directory entry is read from and written to by a different I/O +# thread. You can include the same directory multiple times in order to create +# multiple I/O threads against that directory. This is for example relevant for +# high-throughput RAIDs. +# +# If not specified, the system-specific Java temporary directory (java.io.tmpdir +# property) is taken. +# +taskmanager.tmp.dirs: /tmp/mahoutcache + + +# Path to the Hadoop configuration directory. +# +# This configuration is used when writing into HDFS. Unless specified otherwise, +# HDFS file creation will use HDFS default settings with respect to block-size, +# replication factor, etc. +# +# You can also directly specify the paths to hdfs-default.xml and hdfs-site.xml +# via keys 'fs.hdfs.hdfsdefault' and 'fs.hdfs.hdfssite'. +# +# fs.hdfs.hadoopconf: /path/to/hadoop/conf/
