Re: [edk2-devel] [Patch] BaseTools: Improve the method of checking queue empty
Reviewed-by: Liming Gao
-Original Message-
From: devel@edk2.groups.io On Behalf Of Bob Feng
Sent: 2020年8月3日 10:04
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming ;
Chen, Christine ; Lucy Yan
Subject: [edk2-devel] [Patch] BaseTools: Improve the method of checking queue
empty
From: "Feng, Bob C"
Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807
The Queue.empty() method is not reliable in the multiple process runtime
environment. This patch uses a new method to check if all modules are processed
and workers need to be stopped. That is to add a None item at the bottom of the
queue. Worker check if it gets that None item to know if all the module is
processed.
Signed-off-by: Bob Feng
Cc: Liming Gao
Cc: Yuwei Chen
Cc: Lucy Yan
---
.../Source/Python/AutoGen/AutoGenWorker.py| 26 ++-
BaseTools/Source/Python/build/build.py| 3 ++-
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 563d91b421..017f676399 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -22,10 +22,11 @@ except:
from Queue import Empty
import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe import logging
+import time
def clearQ(q):
try:
while True:
q.get_nowait()
@@ -109,11 +110,15 @@ class AutoGenManager(threading.Thread):
badnews = self.feedback_q.get()
if badnews is None:
break
if badnews == "Done":
fin_num += 1
+elif badnews == "QueueEmpty":
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), badnews))
+self.TerminateWorkers()
else:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s"
+ % (os.getpid(), badnews))
self.Status = False
self.TerminateWorkers()
if fin_num == len(self.autogen_workers):
self.clearQueue()
for w in self.autogen_workers:
@@ -225,16 +230,25 @@ class AutoGenWorkerInProcess(mp.Process):
FfsCmd = {}
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile =
self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
-if self.module_queue.empty():
-break
if self.error_event.is_set():
break
module_count += 1
-
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib
= self.module_queue.get_nowait()
+try:
+
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib
= self.module_queue.get_nowait()
+except Empty:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), "Fake Empty."))
+time.sleep(0.01)
+continue
+if module_file is None:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), "Worker get the last item in the queue."))
+self.feedback_q.put("QueueEmpty")
+time.sleep(0.01)
+continue
+
modulefullpath = os.path.join(module_root,module_file)
taskname = " : ".join((modulefullpath,module_arch))
module_metafile = PathClass(module_file,module_root)
if module_path:
module_metafile.Path = module_path @@ -278,15 +292,15 @@
class AutoGenWorkerInProcess(mp.Process):
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch,
"MakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch,
"MakeCache", False))
-except Empty:
-pass
-except:
+except Exception as e:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
+ (os.getpid(), str(e)))
self.feedback_q.put(taskname)
finally:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
+ (os.getpid(), "Done"))
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " %
(os.getpid(),len(AutoGen.
[edk2-devel] [Patch] BaseTools: Improve the method of checking queue empty
From: "Feng, Bob C"
Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807
The Queue.empty() method is not reliable in the multiple
process runtime environment. This patch uses a new method
to check if all modules are processed and workers need
to be stopped. That is to add a None item at the bottom
of the queue. Worker check if it gets that None item to
know if all the module is processed.
Signed-off-by: Bob Feng
Cc: Liming Gao
Cc: Yuwei Chen
Cc: Lucy Yan
---
.../Source/Python/AutoGen/AutoGenWorker.py| 26 ++-
BaseTools/Source/Python/build/build.py| 3 ++-
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 563d91b421..017f676399 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -22,10 +22,11 @@ except:
from Queue import Empty
import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe
import logging
+import time
def clearQ(q):
try:
while True:
q.get_nowait()
@@ -109,11 +110,15 @@ class AutoGenManager(threading.Thread):
badnews = self.feedback_q.get()
if badnews is None:
break
if badnews == "Done":
fin_num += 1
+elif badnews == "QueueEmpty":
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), badnews))
+self.TerminateWorkers()
else:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), badnews))
self.Status = False
self.TerminateWorkers()
if fin_num == len(self.autogen_workers):
self.clearQueue()
for w in self.autogen_workers:
@@ -225,16 +230,25 @@ class AutoGenWorkerInProcess(mp.Process):
FfsCmd = {}
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile =
self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
-if self.module_queue.empty():
-break
if self.error_event.is_set():
break
module_count += 1
-
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib
= self.module_queue.get_nowait()
+try:
+
module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib
= self.module_queue.get_nowait()
+except Empty:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), "Fake Empty."))
+time.sleep(0.01)
+continue
+if module_file is None:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
(os.getpid(), "Worker get the last item in the queue."))
+self.feedback_q.put("QueueEmpty")
+time.sleep(0.01)
+continue
+
modulefullpath = os.path.join(module_root,module_file)
taskname = " : ".join((modulefullpath,module_arch))
module_metafile = PathClass(module_file,module_root)
if module_path:
module_metafile.Path = module_path
@@ -278,15 +292,15 @@ class AutoGenWorkerInProcess(mp.Process):
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch,
"MakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch,
"MakeCache", False))
-except Empty:
-pass
-except:
+except Exception as e:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(),
str(e)))
self.feedback_q.put(taskname)
finally:
+EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(),
"Done"))
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " %
(os.getpid(),len(AutoGen.Cache(
diff --git a/BaseTools/Source/Python/build/build.py
b/BaseTools/Source/Python/build/build.py
index 1ab1e60a64..59ceacfed0 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1215,11 +1215,11 @@ class Build():
if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
mqueue = mp.Queue()
for m in AutoGenObject.GetAllModuleInfo:
mqueue.put(m)
-
+
