Andreas Sandberg has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/50408 )
Change subject: sim: Fix fork for multithreaded simulations
......................................................................
sim: Fix fork for multithreaded simulations
It is currently not possible to call m5.fork when the simulator is
running in with multiple parallel event queues. The POSIX standard
have very weak guarantees when forking a process with multiple
threads. In order to use fork correctly, we need to ensure that all
helper threads servicing event queues have terminated before the fork
system call is invoked.
There are two ways this could be implemented: 1) Always terminate
helper threads when taking a global simulator exit event, or 2)
terminate helper threads just before fork is called from Python.
This change implements the second strategy since the KVM-based CPUs
currently assume that TIDs don't change unless there is a fork event.
Change-Id: I22feaecd49f7f81689b43185d63a8f14428bed63
Signed-off-by: Andreas Sandberg <andreas.sandb...@arm.com>
---
M src/python/m5/simulate.py
M src/python/pybind11/event.cc
M src/sim/simulate.cc
M src/sim/simulate.hh
4 files changed, 90 insertions(+), 14 deletions(-)
diff --git a/src/python/m5/simulate.py b/src/python/m5/simulate.py
index 66e6a08..b5b8c78 100644
--- a/src/python/m5/simulate.py
+++ b/src/python/m5/simulate.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2012,2019 ARM Limited
+# Copyright (c) 2012, 2019, 2021 Arm Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
@@ -348,6 +348,9 @@
drain()
+ # Terminate helper threads that service parallel event queues.
+ _m5.event.terminateEventQueueThreads()
+
try:
pid = os.fork()
except OSError as e:
diff --git a/src/python/pybind11/event.cc b/src/python/pybind11/event.cc
index aefe50a..7a02221 100644
--- a/src/python/pybind11/event.cc
+++ b/src/python/pybind11/event.cc
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017 ARM Limited
+ * Copyright (c) 2017, 2021 Arm Limited
* All rights reserved
*
* The license below extends only to copyright in the software and shall
@@ -107,6 +107,7 @@
m.def("simulate", &simulate,
py::arg("ticks") = MaxTick);
+ m.def("terminateEventQueueThreads", &terminateEventQueueThreads);
m.def("exitSimLoop", &exitSimLoop);
m.def("getEventQueue", []() { return curEventQueue(); },
py::return_value_policy::reference);
diff --git a/src/sim/simulate.cc b/src/sim/simulate.cc
index 4a00869..a87bd04 100644
--- a/src/sim/simulate.cc
+++ b/src/sim/simulate.cc
@@ -1,4 +1,16 @@
/*
+ * Copyright (c) 2021 Arm Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder. You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
* Copyright (c) 2006 The Regents of The University of Michigan
* Copyright (c) 2013 Advanced Micro Devices, Inc.
* Copyright (c) 2013 Mark D. Hill and David A. Wood
@@ -30,6 +42,7 @@
#include "sim/simulate.hh"
+#include <atomic>
#include <mutex>
#include <thread>
@@ -46,11 +59,15 @@
{
//! Mutex for handling async events.
-std::mutex asyncEventMutex;
+static std::mutex asyncEventMutex;
//! Global barrier for synchronizing threads entering/exiting the
//! simulation loop.
-Barrier *threadBarrier;
+static Barrier *threadBarrier;
+
+static std::atomic<bool> terminateEqThreads = false;
+
+static std::vector<std::thread *> eventThreads;
//! forward declaration
Event *doSimLoop(EventQueue *);
@@ -66,9 +83,12 @@
static void
thread_loop(EventQueue *queue)
{
- while (true) {
- threadBarrier->wait();
+ /* Wait for all initialisation to complete */
+ threadBarrier->wait();
+
+ while (!terminateEqThreads) {
doSimLoop(queue);
+ threadBarrier->wait();
}
}
@@ -86,18 +106,14 @@
// create a thread for each of event queues referenced by the
// instantiated sim objects.
static bool threads_initialized = false;
- static std::vector<std::thread *> threads;
+
+ /* terminateEqThreads is initialised to false and should only be
+ * set to true temporarily in terminateEventQueueThreads. */
+ assert(!terminateEqThreads);
if (!threads_initialized) {
threadBarrier = new Barrier(numMainEventQueues);
- // the main thread (the one we're currently running on)
- // handles queue 0, so we only need to allocate new threads
- // for queues 1..N-1. We'll call these the "subordinate" threads.
- for (uint32_t i = 1; i < numMainEventQueues; i++) {
- threads.push_back(new std::thread(thread_loop,
mainEventQueue[i]));
- }
-
threads_initialized = true;
simulate_limit_event =
new GlobalSimLoopExitEvent(mainEventQueue[0]->getCurTick(),
@@ -125,6 +141,16 @@
inParallelMode = true;
}
+ if (eventThreads.empty()) {
+ // the main thread (the one we're currently running on)
+ // handles queue 0, so we only need to allocate new threads
+ // for queues 1..N-1. We'll call these the "subordinate" threads.
+ for (uint32_t i = 1; i < numMainEventQueues; i++) {
+ eventThreads.push_back(
+ new std::thread(thread_loop, mainEventQueue[i]));
+ }
+ }
+
// all subordinate (created) threads should be waiting on the
// barrier; the arrival of the main thread here will satisfy the
// barrier, and all threads will enter doSimLoop in parallel
@@ -151,6 +177,31 @@
return global_exit_event;
}
+void
+terminateEventQueueThreads()
+{
+ assert(!terminateEqThreads);
+
+ /* This function should only be called when the simulator is
+ * handling a global exit event (typically from Python). This
+ * means that the helper threads will be waiting on the
+ * barrier. Tell the helper threads to exit and release them from
+ * their barrier. */
+ terminateEqThreads = true;
+ threadBarrier->wait();
+
+ /* Wait for all of the threads to terminate */
+ for (auto &thread : eventThreads) {
+ thread->join();
+ delete thread;
+ thread = nullptr;
+ }
+
+ terminateEqThreads = false;
+ eventThreads.clear();
+}
+
+
/**
* Test and clear the global async_event flag, such that each time the
* flag is cleared, only one thread returns true (and thus is assigned
diff --git a/src/sim/simulate.hh b/src/sim/simulate.hh
index 0817bbd..5ef4995 100644
--- a/src/sim/simulate.hh
+++ b/src/sim/simulate.hh
@@ -1,4 +1,16 @@
/*
+ * Copyright (c) 2021 Arm Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder. You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
* Copyright (c) 2006 The Regents of The University of Michigan
* All rights reserved.
*
@@ -34,6 +46,15 @@
class GlobalSimLoopExitEvent;
GlobalSimLoopExitEvent *simulate(Tick num_cycles = MaxTick);
+
+/**
+ * Terminate helper threads when running in parallel mode.
+ *
+ * @pre Simulator must have returned from simulate() to service a
+ * GlobalExitEvent prior to calling this function.
+ */
+void terminateEventQueueThreads();
+
extern GlobalSimLoopExitEvent *simulate_limit_event;
} // namespace gem5
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/50408
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: I22feaecd49f7f81689b43185d63a8f14428bed63
Gerrit-Change-Number: 50408
Gerrit-PatchSet: 1
Gerrit-Owner: Andreas Sandberg <andreas.sandb...@arm.com>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- gem5-dev@gem5.org
To unsubscribe send an email to gem5-dev-le...@gem5.org
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
