BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1680
Consider modules with .inc source files as Binary Modules
and do not Skip by hash.
Re-order hashing operations so we don't do redundant hashes.
Respect artifact location within directory structure.
Re-use libraries, since they have already been hashed.
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Christian Rodriguez <christian.rodrig...@intel.com>
Cc: Bob Feng <bob.c.f...@intel.com>
Cc: Liming Gao <liming....@intel.com>
Cc: Yonghong Zhu <yonghong....@intel.com>
---
BaseTools/Source/Python/AutoGen/AutoGen.py | 47
++++++++++++++++++++++++++++++++++++-----------
BaseTools/Source/Python/AutoGen/GenMake.py | 6 +++---
BaseTools/Source/Python/build/build.py | 15 ++++++++++++++-
3 files changed, 53 insertions(+), 15 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
b/BaseTools/Source/Python/AutoGen/AutoGen.py
index 8c7c20a386..d087ca5e0e 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -661,7 +661,7 @@ class WorkspaceAutoGen(AutoGen):
#
# Generate Package level hash value
#
- GlobalData.gPackageHash[Arch] = {}
+ GlobalData.gPackageHash = {}
if GlobalData.gUseHashCache:
for Pkg in Pkgs:
self._GenPkgLevelHash(Pkg)
@@ -747,7 +747,7 @@ class WorkspaceAutoGen(AutoGen):
return True
def _GenPkgLevelHash(self, Pkg):
- if Pkg.PackageName in GlobalData.gPackageHash[Pkg.Arch]:
+ if Pkg.PackageName in GlobalData.gPackageHash:
return
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
@@ -770,7 +770,7 @@ class WorkspaceAutoGen(AutoGen):
f.close()
m.update(Content)
SaveFileOnChange(HashFile, m.hexdigest(), False)
- GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = m.hexdigest()
+ GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()
def _GetMetaFiles(self, Target, Toolchain, Arch):
AllWorkSpaceMetaFiles = set()
@@ -3906,9 +3906,11 @@ class ModuleAutoGen(AutoGen):
FileDir = path.join(GlobalData.gBinCacheDest, self.Arch,
self.SourceDir, self.MetaFile.BaseName)
CreateDirectory (FileDir)
HashFile = path.join(self.BuildDir, self.Name + '.hash')
- ModuleFile = path.join(self.OutputDir, self.Name + '.inf')
if os.path.exists(HashFile):
shutil.copy2(HashFile, FileDir)
+ if self.IsLibrary:
+ return
+ ModuleFile = path.join(self.OutputDir, self.Name + '.inf')
if os.path.exists(ModuleFile):
shutil.copy2(ModuleFile, FileDir)
if not self.OutputFile:
@@ -3920,11 +3922,20 @@ class ModuleAutoGen(AutoGen):
if not os.path.isabs(File):
File = os.path.join(self.OutputDir, File)
if os.path.exists(File):
- shutil.copy2(File, FileDir)
+ sub_dir = os.path.relpath(File, self.OutputDir)
+ destination_file = os.path.join(FileDir, sub_dir)
+ destination_dir = os.path.dirname(destination_file)
+ CreateDirectory(destination_dir)
+ shutil.copy2(File, destination_dir)
def AttemptModuleCacheCopy(self):
+ # If library or Module is binary do not skip by hash
if self.IsBinaryModule:
return False
+ # .inc is contains binary information so do not skip by hash as well
+ for f_ext in self.SourceFileList:
+ if '.inc' in str(f_ext):
+ return False
FileDir = path.join(GlobalData.gBinCacheSource, self.Arch,
self.SourceDir, self.MetaFile.BaseName)
HashFile = path.join(FileDir, self.Name + '.hash')
if os.path.exists(HashFile):
@@ -3939,7 +3950,11 @@ class ModuleAutoGen(AutoGen):
shutil.copy2(HashFile, self.BuildDir)
else:
File = path.join(root, f)
- shutil.copy2(File, self.OutputDir)
+ sub_dir = os.path.relpath(File, FileDir)
+ destination_file =
os.path.join(self.OutputDir, sub_dir)
+ destination_dir =
os.path.dirname(destination_file)
+ CreateDirectory(destination_dir)
+ shutil.copy2(File, destination_dir)
if self.Name == "PcdPeim" or self.Name == "PcdDxe":
CreatePcdDatabaseCode(self, TemplateString(),
TemplateString())
return True
@@ -4087,14 +4102,16 @@ class ModuleAutoGen(AutoGen):
def GenModuleHash(self):
if self.Arch not in GlobalData.gModuleHash:
GlobalData.gModuleHash[self.Arch] = {}
+ if self.Name in GlobalData.gModuleHash[self.Arch] and
GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():
+ return False
m = hashlib.md5()
# Add Platform level hash
m.update(GlobalData.gPlatformHash.encode('utf-8'))
# Add Package level hash
if self.DependentPackageList:
for Pkg in sorted(self.DependentPackageList, key=lambda x:
x.PackageName):
- if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]:
-
m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName].encode('utf-8'))
+ if Pkg.PackageName in GlobalData.gPackageHash:
+
m.update(GlobalData.gPackageHash[Pkg.PackageName].encode('utf-8'))
# Add Library hash
if self.LibraryAutoGenList:
@@ -4119,14 +4136,22 @@ class ModuleAutoGen(AutoGen):
ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")
if self.Name not in GlobalData.gModuleHash[self.Arch]:
GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()
- if GlobalData.gBinCacheSource:
- if self.AttemptModuleCacheCopy():
- return False
+ if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy():
+ return False
return SaveFileOnChange(ModuleHashFile, m.hexdigest(), False)
## Decide whether we can skip the ModuleAutoGen process
def CanSkipbyHash(self):
+ # If library or Module is binary do not skip by hash
+ if self.IsBinaryModule:
+ return False
+ # .inc is contains binary information so do not skip by hash as well
+ for f_ext in self.SourceFileList:
+ if '.inc' in str(f_ext):
+ return False
if GlobalData.gUseHashCache:
+ # If there is a valid hash or function generated a valid hash;
function will return False
+ # and the statement below will return True
return not self.GenModuleHash()
return False
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
b/BaseTools/Source/Python/AutoGen/GenMake.py
index b441817b52..537314e33b 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -983,7 +983,7 @@ cleanlib:
## For creating makefile targets for dependent libraries
def ProcessDependentLibrary(self):
for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
- if not LibraryAutoGen.IsBinaryModule:
+ if not LibraryAutoGen.IsBinaryModule and not
LibraryAutoGen.CanSkipbyHash():
self.LibraryBuildDirectoryList.append(self.PlaceMacro(LibraryAutoGen.BuildDir,
self.Macros))
## Return a list containing source file's dependencies
@@ -1486,7 +1486,7 @@ cleanlib:
def GetLibraryBuildDirectoryList(self):
DirList = []
for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
- if not LibraryAutoGen.IsBinaryModule:
+ if not LibraryAutoGen.IsBinaryModule and not
LibraryAutoGen.CanSkipbyHash():
DirList.append(os.path.join(self._AutoGenObject.BuildDir,
LibraryAutoGen.BuildDir))
return DirList
@@ -1622,7 +1622,7 @@ class TopLevelMakefile(BuildFile):
def GetLibraryBuildDirectoryList(self):
DirList = []
for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList:
- if not LibraryAutoGen.IsBinaryModule:
+ if not LibraryAutoGen.IsBinaryModule and not
LibraryAutoGen.CanSkipbyHash():
DirList.append(os.path.join(self._AutoGenObject.BuildDir,
LibraryAutoGen.BuildDir))
return DirList
diff --git a/BaseTools/Source/Python/build/build.py
b/BaseTools/Source/Python/build/build.py
index de641fb452..7938a3dd56 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1775,7 +1775,8 @@ class Build():
for Module in Pa.Platform.Modules:
if self.ModuleFile.Dir == Module.Dir and
self.ModuleFile.Name == Module.Name:
Ma = ModuleAutoGen(Wa, Module, BuildTarget,
ToolChain, Arch, self.PlatformFile)
- if Ma is None: continue
+ if Ma is None:
+ continue
MaList.append(Ma)
if Ma.CanSkipbyHash():
self.HashSkipModules.append(Ma)
@@ -2149,12 +2150,24 @@ class Build():
RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)
def CreateAsBuiltInf(self):
+ all_lib_set = set()
+ all_mod_set = set()
for Module in self.BuildModules:
Module.CreateAsBuiltInf()
+ all_mod_set.add(Module)
for Module in self.HashSkipModules:
Module.CreateAsBuiltInf(True)
+ all_mod_set.add(Module)
+ for Module in all_mod_set:
+ for lib in Module.LibraryAutoGenList:
+ all_lib_set.add(lib)
+ for lib in all_lib_set:
+ lib.CreateAsBuiltInf(True)
+ all_lib_set.clear()
+ all_mod_set.clear()
self.BuildModules = []
self.HashSkipModules = []
+
## Do some clean-up works when error occurred
def Relinquish(self):
OldLogLevel = EdkLogger.GetLevel()
--
2.19.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
