[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201932815
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201931079
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201931079
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201927632
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201926779
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
Review comment:
need doc for these member functions
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201927974
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201926433
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
Review comment:
check for data_inited_ before freeing memory
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r201927556
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,592 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization, please set the same hash key
for same layer, "
+"Block.prefix is typically used as in
:class:`gluon.nn.contrib.SyncBatchNorm`.");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int num_devices_;
+ T mean_;
+ T *data_;
+ bool *flag_;
+ bool mean_ready_ = false;
+ bool data_inited_ = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :num_devices_(ndev) {
+flag_ = new bool[ndev];
+data_ = new T[ndev];
+memset(flag_, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+mshadow::FreeSpace(&mean_);
+delete [] flag_;
+delete [] data_;
+ }
+
+ void Init(mshadow::Shape<1> shape) {
+std::lock_guard lock(mutex_);
+if (!data_inited_) {
+ for (int i = 0; i < num_devices_; i++) {
+data_[i] = mshadow::NewTensor(shape, 0.0f);
+ }
+ mean_ = mshadow::NewTensor(shape, 0.0f);
+ data_inited_ = true;
+}
+ }
+
+ T* Retrieve(mshadow::Shape<1> shape, int index) {
+if (!data_inited_) {
+ Init(shape);
+}
+if (flag_[index] == false) {
+ return &data_[index];
+} else {
+ return nullptr;
+}
+ }
+
+ bool SetReady(int index) {
+if (flag_[index] == false) {
+ flag_[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag_[index] = false;
+T tmp = mean_;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (mean_ready_) {
+ return true;
+}
+for (int i = 0; i < num_devices_; i++) {
+ if (!flag_[i]) {
+return false;
+ }
+}
+for (int i = 1; i < num_devices_; i++) {
+ data_[0] += data_[i];
+}
+mean_ = data_[0] * 1.0f / num_devices_;
+mean_ready_ = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < num_devices_; i++) {
+ if (flag_[i]) return;
+}
+mean_ready_ = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r199570230
##
File path: python/mxnet/gluon/contrib/nn/basic_layers.py
##
@@ -151,3 +152,75 @@ def __repr__(self):
s = '{block_name}({input_dim} -> {output_dim}, {dtype})'
return s.format(block_name=self.__class__.__name__,
**self._kwargs)
+
+class SyncBatchNorm(BatchNorm):
+"""Cross-GPU Synchronized Batch normalization (SyncBN)
+Standard BN [1]_ implementation only normalize the data within each device.
+SyncBN normalizes the input within the whole mini-batch.
+We follow the sync-onece implmentation described in the paper [2]_ .
+Parameters
+--
+momentum: float, default 0.9
+Momentum for the moving average.
+epsilon: float, default 1e-5
+Small float added to variance to avoid dividing by zero.
+center: bool, default True
+If True, add offset of `beta` to normalized tensor.
+If False, `beta` is ignored.
+scale: bool, default True
+If True, multiply by `gamma`. If False, `gamma` is not used.
+When the next layer is linear (also e.g. `nn.relu`),
+this can be disabled since the scaling
+will be done by the next layer.
+use_global_stats: bool, default False
+If True, use global moving statistics instead of local batch-norm.
This will force
+change batch-norm into a scale shift operator.
+If False, use local batch-norm.
+beta_initializer: str or `Initializer`, default 'zeros'
+Initializer for the beta weight.
+gamma_initializer: str or `Initializer`, default 'ones'
+Initializer for the gamma weight.
+moving_mean_initializer: str or `Initializer`, default 'zeros'
+Initializer for the moving mean.
+moving_variance_initializer: str or `Initializer`, default 'ones'
+Initializer for the moving variance.
+in_channels : int, default 0
+Number of channels (feature maps) in input data. If not specified,
+initialization will be deferred to the first time `forward` is called
+and `in_channels` will be inferred from the shape of input data.
+num_devices : int, default number of visible GPUs
+
+
+Inputs:
+- **data**: input tensor with arbitrary shape.
+Outputs:
+- **out**: output tensor with the same shape as `data`.
+
+Reference:
+.. [1] Ioffe, Sergey, and Christian Szegedy. "Batch normalization:
Accelerating
+deep network training by reducing internal covariate shift." *ICML
2015*
+.. [2] Hang Zhang, Kristin Dana, Jianping Shi, Zhongyue Zhang,
Xiaogang Wang,
+Ambrish Tyagi, and Amit Agrawal. "Context Encoding for Semantic
Segmentation." *CVPR 2018*
+"""
+def __init__(self, in_channels=0, num_devices=None, momentum=0.9,
epsilon=1e-5,
+ center=True, scale=True, use_global_stats=False,
beta_initializer='zeros',
+ gamma_initializer='ones', running_mean_initializer='zeros',
+ running_variance_initializer='ones', **kwargs):
+super(SyncBatchNorm, self).__init__(1, momentum, epsilon, center,
scale, use_global_stats,
+beta_initializer,
gamma_initializer,
+running_mean_initializer,
running_variance_initializer,
+in_channels, **kwargs)
+num_devices = self._get_num_devices() if num_devices is None else
num_devices
+self._kwargs = {'eps': epsilon, 'momentum': momentum,
+'fix_gamma': not scale, 'use_global_stats':
use_global_stats,
+'ndev': num_devices, 'key': self.prefix}
+
+def _get_num_devices(self):
+# Caution: if not using all the GPUs, please mannually set num_devices
Review comment:
add the warning to docstring rather than showing a comment here
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r199570776
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+# include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int nDev;
Review comment:
convention for variables is `xxx_` for private members
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r199570350 ## File path: src/operator/contrib/sync_batch_norm-inl.h ## @@ -0,0 +1,562 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/*! + * Copyright (c) 2018 by Contributors + * \file sync_batch_norm-inl.h + * \brief Synchronized BatchNorm modified from BatchNormV1 + * \author Hang Zhang +*/ +#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_ +#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_ + +#include +#include +#include +# include Review comment: space between # and include? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r199572413
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+# include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int nDev;
+ T mean;
+ T *data;
+ bool *flag;
+ bool meanReady = false;
+ bool meanInited = false;
+ std::mutex mutex_;
+
+ public:
+ explicit SharedND(int ndev) :nDev(ndev) {
+ flag = new bool[ndev];
+ data = new T[ndev];
+ memset(flag, false, ndev * sizeof(bool));
+ }
+
+ ~SharedND() {
+delete [] flag;
+delete [] data;
+ }
+
+ bool Push(T input, int index) {
+if (flag[index] == false) {
+ data[index] = input;
+ flag[index] = true;
+ return true;
+} else {
+ return false;
+}
+ }
+
+ T Pop(int index) {
+std::lock_guard lock(mutex_);
+while (!MeanReady()) {}
+flag[index] = false;
+T tmp = mean;
+ResetMean();
+return tmp;
+ }
+
+ bool MeanReady() {
+if (meanReady) {
+ return true;
+}
+for (int i = 0; i < nDev; i++) {
+ if (!flag[i]) {
+return false;
+ }
+}
+for (int i = 1; i < nDev; i++) {
+ data[0] += data[i];
+}
+if (!meanInited) {
+ mean = mshadow::NewTensor(data[0].shape_, 0.0f);
+ meanInited = true;
+}
+mean = data[0] * 1.0f / nDev;
+meanReady = true;
+return true;
+ }
+
+ void ResetMean() {
+for (int i = 0; i < nDev; i++) {
+ if (flag[i]) return;
+}
+meanReady = false;
+ }
+};
+
+template
+class GlobalShared {
+ public:
+ T* Register(const std::string &key, int ndev) {
+std::lock_guard lock(mutex_);
+auto it = registry_.find(key);
+if (it != registry_.end()) return it->second;
+T *newT = new T(ndev);
Review comment:
memory is not released pointed by these raw pointers
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhreshold commented on a change in pull request #11502: [MXNET-614] Adding Synchronized Batch Normalization
zhreshold commented on a change in pull request #11502: [MXNET-614] Adding
Synchronized Batch Normalization
URL: https://github.com/apache/incubator-mxnet/pull/11502#discussion_r199571599
##
File path: src/operator/contrib/sync_batch_norm-inl.h
##
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file sync_batch_norm-inl.h
+ * \brief Synchronized BatchNorm modified from BatchNormV1
+ * \author Hang Zhang
+*/
+#ifndef MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+#define MXNET_OPERATOR_CONTRIB_SYNC_BATCH_NORM_INL_H_
+
+#include
+#include
+#include
+# include
+#include
+#include
+#include
+#include
+#include "../operator_common.h"
+#include "../mshadow_op.h"
+
+namespace mxnet {
+namespace op {
+
+namespace syncbatchnorm {
+enum BatchNormOpInputs {kData, kGamma, kBeta};
+enum BatchNormOpOutputs {kOut, kMean, kVar};
+enum BatchNormOpAuxiliary {kMovingMean, kMovingVar};
+enum BatchNormBackResource {kTempSpace};
+} // namespace syncbatchnorm
+
+struct SyncBatchNormParam : public dmlc::Parameter {
+ float eps;
+ float momentum;
+ bool fix_gamma;
+ bool use_global_stats;
+ bool output_mean_var;
+ int ndev;
+ std::string key;
+ DMLC_DECLARE_PARAMETER(SyncBatchNormParam) {
+DMLC_DECLARE_FIELD(eps).set_default(1e-3f)
+.describe("Epsilon to prevent div 0");
+DMLC_DECLARE_FIELD(momentum).set_default(0.9f)
+.describe("Momentum for moving average");
+DMLC_DECLARE_FIELD(fix_gamma).set_default(true)
+.describe("Fix gamma while training");
+DMLC_DECLARE_FIELD(use_global_stats).set_default(false)
+.describe("Whether use global moving statistics instead of local
batch-norm. "
+ "This will force change batch-norm into a scale shift
operator.");
+DMLC_DECLARE_FIELD(output_mean_var).set_default(false)
+.describe("Output All,normal mean and var");
+DMLC_DECLARE_FIELD(ndev).set_default(1)
+ .describe("The count of GPU devices");
+DMLC_DECLARE_FIELD(key)
+ .set_default("")
+ .describe("Hash key for synchronization");
+ }
+};
+
+// Modified from https://github.com/brucechin/SharedTensor
+template
+class SharedND {
+ private:
+ int nDev;
Review comment:
and camel for functions, which is correct right now
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
