Please review. Bump. >-----Original Message----- >From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of >Christian Rodriguez >Sent: Friday, May 24, 2019 7:40 AM >To: devel@edk2.groups.io >Cc: Feng, Bob C <bob.c.f...@intel.com>; Gao, Liming ><liming....@intel.com>; Zhu, Yonghong <yonghong....@intel.com> >Subject: [edk2-devel] [Patch V3 2/2] BaseTools: Add a checking for Sources >section in INF file [Part 2/2] > >BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804 > >In V3: Seperate checker and hashing into individual patches In V2: Enable >check for all builds, move conditional to hash invalidation In the Edk2 INF >spec >3.9, it states, All HII Unicode format files must be listed in [Sources] >section. >Add a check to see if [Sources] section lists all the "source" type files of a >module. Performance impact should be minimal with this patch since >information is already being fetched for Makefile purposes. All other >information is already cached in memory. No extra IO time is needed. Part 2 is >hashing update. > >Signed-off-by: Christian Rodriguez <christian.rodrig...@intel.com> >Cc: Bob Feng <bob.c.f...@intel.com> >Cc: Liming Gao <liming....@intel.com> >Cc: Yonghong Zhu <yonghong....@intel.com> >--- > BaseTools/Source/Python/AutoGen/AutoGen.py | 6 +- > BaseTools/Source/Python/AutoGen/GenMake.py | 6 ++ > BaseTools/Source/Python/Common/GlobalData.py | 3 +- > BaseTools/Source/Python/build/build.py | 65 ++++++++++++-------- > 4 files changed, 53 insertions(+), 27 deletions(-) > >diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py >b/BaseTools/Source/Python/AutoGen/AutoGen.py >index a843f294b9..0bc27fb2b3 100644 >--- a/BaseTools/Source/Python/AutoGen/AutoGen.py >+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py >@@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen): > for LibraryAutoGen in self.LibraryAutoGenList: > LibraryAutoGen.CreateMakeFile() > >- if self.CanSkip(): >+ # Don't enable if hash feature enabled, CanSkip uses timestamps to >determine build skipping >+ if not GlobalData.gUseHashCache and self.CanSkip(): > return > > if len(self.CustomMakefile) == 0: >@@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen): > for LibraryAutoGen in self.LibraryAutoGenList: > LibraryAutoGen.CreateCodeFile() > >- if self.CanSkip(): >+ # Don't enable if hash feature enabled, CanSkip uses timestamps to >determine build skipping >+ if not GlobalData.gUseHashCache and self.CanSkip(): > return > > AutoGenList = [] >diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py >b/BaseTools/Source/Python/AutoGen/GenMake.py >index 5c992d7c26..212ca0fa7f 100644 >--- a/BaseTools/Source/Python/AutoGen/GenMake.py >+++ b/BaseTools/Source/Python/AutoGen/GenMake.py >@@ -935,10 +935,16 @@ cleanlib: > continue > headerFileDependencySet.add(aFileName) > >+ # Ensure that gModuleBuildTracking has been initialized per >architecture >+ if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking: >+ GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] = >+ dict() >+ > # Check if a module dependency header file is missing from the > module's >MetaFile > for aFile in headerFileDependencySet: > if aFile in headerFilesInMetaFileSet: > continue >+ if GlobalData.gUseHashCache: >+ >GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGen >Object] = 'FAIL_METAFILE' > EdkLogger.warn("build","Module MetaFile [Sources] is missing local >header!", > ExtraData = "Local Header: " + aFile + " not found in > " + >self._AutoGenObject.MetaFile.Path > ) >diff --git a/BaseTools/Source/Python/Common/GlobalData.py >b/BaseTools/Source/Python/Common/GlobalData.py >index 95e28a988f..bd45a43728 100644 >--- a/BaseTools/Source/Python/Common/GlobalData.py >+++ b/BaseTools/Source/Python/Common/GlobalData.py >@@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False >gSikpAutoGenCache = set() > > # Dictionary for tracking Module build status as success or failure -# False > -> >Fail : True -> Success >+# Top Dict: Key: Arch Type Value: Dictionary >+# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE' > gModuleBuildTracking = dict() > > # Dictionary of booleans that dictate whether a module or diff --git >a/BaseTools/Source/Python/build/build.py >b/BaseTools/Source/Python/build/build.py >index 80ceb98310..0855d4561c 100644 >--- a/BaseTools/Source/Python/build/build.py >+++ b/BaseTools/Source/Python/build/build.py >@@ -625,8 +625,16 @@ class BuildTask: > BuildTask._ErrorFlag.set() > BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ > (threading.currentThread().getName(), > Command, >WorkingDir) >- if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and >not >BuildTask._ErrorFlag.isSet(): >- GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True >+ >+ # Set the value used by hash invalidation flow in >GlobalData.gModuleBuildTracking to 'SUCCESS' >+ # If Module or Lib is being tracked, it did not fail header check >test, and >built successfully >+ if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking >and >+ self.BuildItem.BuildObject in >GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and >+ >GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildI >tem.BuildObject] != 'FAIL_METAFILE' and >+ not BuildTask._ErrorFlag.isSet() >+ ): >+ >GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildI >tem.BuildObject] = 'SUCCESS' >+ > # indicate there's a thread is available for another build task > BuildTask._RunningQueueLock.acquire() > BuildTask._RunningQueue.pop(self.BuildItem) >@@ -1154,27 +1162,30 @@ class Build(): > # > # > def invalidateHash(self): >- # GlobalData.gModuleBuildTracking contains only modules that cannot >be skipped by hash >- for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): >- # False == FAIL : True == Success >- # Skip invalidating for Successful module builds >- if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True: >- continue >+ # Only for hashing feature >+ if not GlobalData.gUseHashCache: >+ return >+ >+ # GlobalData.gModuleBuildTracking contains only modules or libs that >cannot be skipped by hash >+ for moduleAutoGenObjArch in GlobalData.gModuleBuildTracking.keys(): >+ for moduleAutoGenObj in >GlobalData.gModuleBuildTracking[moduleAutoGenObjArch].keys(): >+ # Skip invalidating for Successful Module/Lib builds >+ if >GlobalData.gModuleBuildTracking[moduleAutoGenObjArch][moduleAutoGe >nObj] == 'SUCCESS': >+ continue > >- # The module failed to build or failed to start building, from >this point >on >+ # The module failed to build, failed to start building, >+ or failed the header check test from this point on > >- # Remove .hash from build >- if GlobalData.gUseHashCache: >- ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, >moduleAutoGenObj.Name + ".hash") >+ # Remove .hash from build >+ ModuleHashFile = >+ os.path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + >+ ".hash") > if os.path.exists(ModuleHashFile): > os.remove(ModuleHashFile) > >- # Remove .hash file from cache >- if GlobalData.gBinCacheDest: >- FileDir = path.join(GlobalData.gBinCacheDest, >moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, >moduleAutoGenObj.MetaFile.BaseName) >- HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash') >- if os.path.exists(HashFile): >- os.remove(HashFile) >+ # Remove .hash file from cache >+ if GlobalData.gBinCacheDest: >+ FileDir = os.path.join(GlobalData.gBinCacheDest, >moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, >moduleAutoGenObj.MetaFile.BaseName) >+ HashFile = os.path.join(FileDir, moduleAutoGenObj.Name + >'.hash') >+ if os.path.exists(HashFile): >+ os.remove(HashFile) > > ## Build a module or platform > # >@@ -1825,9 +1836,11 @@ class Build(): > if self.Target == "genmake": > return True > self.BuildModules.append(Ma) >- # Initialize all modules in tracking to False >(FAIL) >- if Ma not in GlobalData.gModuleBuildTracking: >- GlobalData.gModuleBuildTracking[Ma] = False >+ # Initialize all modules in tracking to 'FAIL' >+ if Ma.Arch not in GlobalData.gModuleBuildTracking: >+ GlobalData.gModuleBuildTracking[Ma.Arch] = >dict() >+ if Ma not in >GlobalData.gModuleBuildTracking[Ma.Arch]: >+ GlobalData.gModuleBuildTracking[Ma.Arch][Ma] >= 'FAIL' > self.AutoGenTime += int(round((time.time() - > AutoGenStart))) > MakeStart = time.time() > for Ma in self.BuildModules: >@@ -1911,6 +1924,7 @@ class Build(): > # Save MAP buffer into MAP file. > # > self._SaveMapFile (MapBuffer, Wa) >+ self.invalidateHash() > > def _GenFfsCmd(self,ArchList): > # convert dictionary of Cmd:(Inf,Arch) @@ -2009,9 +2023,11 @@ class >Build(): > if self.Target == "genmake": > continue > self.BuildModules.append(Ma) >- # Initialize all modules in tracking to False (FAIL) >- if Ma not in GlobalData.gModuleBuildTracking: >- GlobalData.gModuleBuildTracking[Ma] = False >+ # Initialize all modules in tracking to 'FAIL' >+ if Ma.Arch not in GlobalData.gModuleBuildTracking: >+ GlobalData.gModuleBuildTracking[Ma.Arch] = dict() >+ if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: >+ GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = >'FAIL' > self.Progress.Stop("done!") > self.AutoGenTime += int(round((time.time() - > AutoGenStart))) > MakeStart = time.time() @@ -2099,6 +2115,7 @@ class > Build(): > # Save MAP buffer into MAP file. > # > self._SaveMapFile(MapBuffer, Wa) >+ self.invalidateHash() > > ## Generate GuidedSectionTools.txt in the FV directories. > # >-- >2.19.1.windows.1 > > >
-=-=-=-=-=-=-=-=-=-=-=- Groups.io Links: You receive all messages sent to this group. View/Reply Online (#41615): https://edk2.groups.io/g/devel/message/41615 Mute This Topic: https://groups.io/mt/31833972/21656 Group Owner: devel+ow...@edk2.groups.io Unsubscribe: https://edk2.groups.io/g/devel/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
