This is an automated email from the ASF dual-hosted git repository.
jevans pushed a commit to branch v1.9.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.9.x by this push:
new cfbcfb1a77 revert cleanup stream (#20998)
cfbcfb1a77 is described below
commit cfbcfb1a773c27f25ab0d7bbc44abb7b2ad2bad8
Author: waytrue17 <[email protected]>
AuthorDate: Mon Apr 11 20:26:14 2022 -0700
revert cleanup stream (#20998)
Co-authored-by: Wei Chu <[email protected]>
---
src/engine/naive_engine.cc | 3 +++
src/engine/threaded_engine_perdevice.cc | 4 ++++
2 files changed, 7 insertions(+)
diff --git a/src/engine/naive_engine.cc b/src/engine/naive_engine.cc
index 2cc28a354d..e7412fa6f4 100644
--- a/src/engine/naive_engine.cc
+++ b/src/engine/naive_engine.cc
@@ -73,11 +73,14 @@ class NaiveEngine final : public Engine {
LOG(INFO) << "Engine shutdown";
for (size_t i = 0; i < streams_.size(); ++i) {
if (streams_[i] != nullptr) {
+ // Catch exception for CUDA driver shutdown
+ MSHADOW_CATCH_ERROR(mshadow::DeleteStream(streams_[i]));
streams_[i] = nullptr;
}
}
for (size_t i = 0; i < aux_streams_.size(); ++i) {
if (aux_streams_[i] != nullptr) {
+ delete aux_streams_[i];
aux_streams_[i] = nullptr;
}
}
diff --git a/src/engine/threaded_engine_perdevice.cc
b/src/engine/threaded_engine_perdevice.cc
index dfdb6d1da6..c59a06b568 100644
--- a/src/engine/threaded_engine_perdevice.cc
+++ b/src/engine/threaded_engine_perdevice.cc
@@ -270,6 +270,10 @@ class ThreadedEnginePerDevice : public ThreadedEngine {
while (task_queue->Pop(&opr_block)) {
this->ExecuteOprBlock(run_ctx, opr_block);
}
+ // Catch exception for CUDA driver shutdown
+ MSHADOW_CATCH_ERROR(mshadow::DeleteStream<gpu>(stream));
+ if (aux_stream != nullptr)
+ delete aux_stream;
#else
ready_event->signal();
#endif
