https://bugzilla.tianocore.org/show_bug.cgi?id=1288
Currently, AutoGen and GenFds run in different python interpreters. The
parser are duplicated. This patch is going to create new API for GenFds
and have the build to call that API instead of executing GenFds.py. As
such, the GenFds and build can share the parser data.
This patch is expected to save the time of GenFds about 2~3 seconds.
More details will be logged in BZ.
This is the summary measure data generated from python cProfile for
building Ovmf.
Currently:
8379147 function calls (8135450 primitive calls) in 12.580 seconds
After applying this patch:
3428712 function calls (3418881 primitive calls) in 8.944 seconds
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: ZhiqiangX Zhao <zhiqiangx.z...@intel.com>
Cc: Liming Gao <liming....@intel.com>
Cc: Carsey, Jaben <jaben.car...@intel.com>
Cc: Bob Feng <bob.c.f...@intel.com>
---
BaseTools/Source/Python/AutoGen/AutoGen.py | 4 +
BaseTools/Source/Python/GenFds/GenFds.py | 266 ++++++++++++++++++++++++++++-
BaseTools/Source/Python/build/build.py | 4 +-
3 files changed, 271 insertions(+), 3 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
b/BaseTools/Source/Python/AutoGen/AutoGen.py
index f3560bfc78..10ce7eb962 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -935,6 +935,10 @@ class WorkspaceAutoGen(AutoGen):
def GenFdsCommand(self):
return
(GenMake.TopLevelMakefile(self)._TEMPLATE_.Replace(GenMake.TopLevelMakefile(self)._TemplateDict)).strip()
+ @property
+ def GenFdsCommandDict(self):
+ return GenMake.TopLevelMakefile(self)._TemplateDict
+
## Create makefile for the platform and modules in it
#
# @param CreateDepsMakeFile Flag indicating if the makefile for
diff --git a/BaseTools/Source/Python/GenFds/GenFds.py
b/BaseTools/Source/Python/GenFds/GenFds.py
index 0c8091b798..96158568b0 100644
--- a/BaseTools/Source/Python/GenFds/GenFds.py
+++ b/BaseTools/Source/Python/GenFds/GenFds.py
@@ -35,7 +35,7 @@ from Common.Misc import DirCache, PathClass,
GuidStructureStringToGuidString
from Common.Misc import SaveFileOnChange, ClearDuplicatedInf
from Common.BuildVersion import gBUILD_VERSION
from Common.MultipleWorkspace import MultipleWorkspace as mws
-from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR,
FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING,
FORMAT_NOT_SUPPORTED,OPTION_VALUE_INVALID
+from Common.BuildToolError import FatalError, GENFDS_ERROR, CODE_ERROR,
FORMAT_INVALID, RESOURCE_NOT_AVAILABLE, FILE_NOT_FOUND, OPTION_MISSING,
FORMAT_NOT_SUPPORTED, OPTION_VALUE_INVALID, PARAMETER_INVALID
from Workspace.WorkspaceDatabase import WorkspaceDatabase
from .FdfParser import FdfParser, Warning
@@ -716,6 +716,270 @@ class GenFds(object):
os.remove(GuidXRefFileName)
GuidXRefFile.close()
+ @staticmethod
+ def GenFdsApi(FdsCommandDict, WorkSpaceDataBase):
+ global Workspace
+ Workspace = os.environ.get('WORKSPACE')
+ ArchList = None
+ ReturnCode = 0
+ try:
+ if not os.path.exists(Workspace):
+ EdkLogger.error("GenFds", PARAMETER_INVALID, "WORKSPACE is
invalid",
+ ExtraData="Please use '-w' switch to pass it
or set the WORKSPACE environment variable.")
+ else:
+ Workspace = os.path.normcase(Workspace)
+ GenFdsGlobalVariable.WorkSpaceDir = Workspace
+ if 'EDK_SOURCE' in os.environ:
+ GenFdsGlobalVariable.EdkSourceDir =
os.path.normcase(os.environ['EDK_SOURCE'])
+ os.chdir(GenFdsGlobalVariable.WorkSpaceDir)
+
+ # set multiple workspace
+ PackagesPath = os.getenv("PACKAGES_PATH")
+ mws.setWs(GenFdsGlobalVariable.WorkSpaceDir, PackagesPath)
+
+ if FdsCommandDict["fdf_file"]:
+ FdfFilename = FdsCommandDict["fdf_file"][0]
+ FdfFilename =
GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdfFilename.Path)
+
+ if FdfFilename[0:2] == '..':
+ FdfFilename = os.path.realpath(FdfFilename)
+ if not os.path.isabs(FdfFilename):
+ FdfFilename = mws.join(GenFdsGlobalVariable.WorkSpaceDir,
FdfFilename)
+ if not os.path.exists(FdfFilename):
+ EdkLogger.error("GenFds", FILE_NOT_FOUND,
ExtraData=FdfFilename)
+
+ GenFdsGlobalVariable.FdfFile = FdfFilename
+ GenFdsGlobalVariable.FdfFileTimeStamp =
os.path.getmtime(FdfFilename)
+ else:
+ EdkLogger.error("GenFds", OPTION_MISSING, "Missing FDF
filename")
+
+ if FdsCommandDict["build_target"]:
+ GenFdsGlobalVariable.TargetName =
FdsCommandDict["build_target"]
+
+ if FdsCommandDict["toolchain_tag"]:
+ GenFdsGlobalVariable.ToolChainTag =
FdsCommandDict["toolchain_tag"]
+
+ if FdsCommandDict["active_platform"]:
+ ActivePlatform = FdsCommandDict["active_platform"]
+ ActivePlatform =
GenFdsGlobalVariable.ReplaceWorkspaceMacro(ActivePlatform)
+
+ if ActivePlatform[0:2] == '..':
+ ActivePlatform = os.path.realpath(ActivePlatform)
+
+ if not os.path.isabs (ActivePlatform):
+ ActivePlatform =
mws.join(GenFdsGlobalVariable.WorkSpaceDir, ActivePlatform)
+
+ if not os.path.exists(ActivePlatform):
+ EdkLogger.error("GenFds", FILE_NOT_FOUND, "ActivePlatform
doesn't exist!")
+ else:
+ EdkLogger.error("GenFds", OPTION_MISSING, "Missing active
platform")
+
+ GenFdsGlobalVariable.ActivePlatform =
PathClass(NormPath(ActivePlatform))
+
+ if FdsCommandDict["conf_directory"]:
+ # Get alternate Conf location, if it is absolute, then just
use the absolute directory name
+ ConfDirectoryPath =
os.path.normpath(FdsCommandDict["conf_directory"])
+ if ConfDirectoryPath.startswith('"'):
+ ConfDirectoryPath = ConfDirectoryPath[1:]
+ if ConfDirectoryPath.endswith('"'):
+ ConfDirectoryPath = ConfDirectoryPath[:-1]
+ if not os.path.isabs(ConfDirectoryPath):
+ # Since alternate directory name is not absolute, the
alternate directory is located within the WORKSPACE
+ # This also handles someone specifying the Conf directory
in the workspace. Using --conf=Conf
+ ConfDirectoryPath =
os.path.join(GenFdsGlobalVariable.WorkSpaceDir, ConfDirectoryPath)
+ else:
+ if "CONF_PATH" in os.environ:
+ ConfDirectoryPath =
os.path.normcase(os.environ["CONF_PATH"])
+ else:
+ # Get standard WORKSPACE/Conf, use the absolute path to
the WORKSPACE/Conf
+ ConfDirectoryPath =
mws.join(GenFdsGlobalVariable.WorkSpaceDir, 'Conf')
+ GenFdsGlobalVariable.ConfDir = ConfDirectoryPath
+ if not GlobalData.gConfDirectory:
+ GlobalData.gConfDirectory = GenFdsGlobalVariable.ConfDir
+ BuildConfigurationFile =
os.path.normpath(os.path.join(ConfDirectoryPath, "target.txt"))
+ if os.path.isfile(BuildConfigurationFile) == True:
+ TargetTxt = TargetTxtClassObject()
+ TargetTxt.LoadTargetTxtFile(BuildConfigurationFile)
+ # if no build target given in command line, get it from
target.txt
+ if not GenFdsGlobalVariable.TargetName:
+ BuildTargetList =
TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TARGET]
+ if len(BuildTargetList) != 1:
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
ExtraData="Only allows one instance for Target.")
+ GenFdsGlobalVariable.TargetName = BuildTargetList[0]
+
+ # if no tool chain given in command line, get it from
target.txt
+ if not GenFdsGlobalVariable.ToolChainTag:
+ ToolChainList =
TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_TOOL_CHAIN_TAG]
+ if ToolChainList is None or len(ToolChainList) == 0:
+ EdkLogger.error("GenFds", RESOURCE_NOT_AVAILABLE,
ExtraData="No toolchain given. Don't know how to build.")
+ if len(ToolChainList) != 1:
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
ExtraData="Only allows one instance for ToolChain.")
+ GenFdsGlobalVariable.ToolChainTag = ToolChainList[0]
+ else:
+ EdkLogger.error("GenFds", FILE_NOT_FOUND,
ExtraData=BuildConfigurationFile)
+
+ if FdsCommandDict["macro"]:
+ for Pair in FdsCommandDict["macro"]:
+ if Pair.startswith('"'):
+ Pair = Pair[1:]
+ if Pair.endswith('"'):
+ Pair = Pair[:-1]
+ List = Pair.split('=')
+ if len(List) == 2:
+ if not List[1].strip():
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
ExtraData="No Value given for Macro %s" %List[0])
+ if List[0].strip() == "EFI_SOURCE":
+ GlobalData.gEfiSource = List[1].strip()
+ GlobalData.gGlobalDefines["EFI_SOURCE"] =
GlobalData.gEfiSource
+ continue
+ elif List[0].strip() == "EDK_SOURCE":
+ GlobalData.gEdkSource = List[1].strip()
+ GlobalData.gGlobalDefines["EDK_SOURCE"] =
GlobalData.gEdkSource
+ continue
+ elif List[0].strip() in ["WORKSPACE", "TARGET",
"TOOLCHAIN"]:
+ GlobalData.gGlobalDefines[List[0].strip()] =
List[1].strip()
+ else:
+ GlobalData.gCommandLineDefines[List[0].strip()] =
List[1].strip()
+ else:
+ GlobalData.gCommandLineDefines[List[0].strip()] =
"TRUE"
+ os.environ["WORKSPACE"] = Workspace
+
+ # Use the -t and -b option as gGlobalDefines's TOOLCHAIN and
TARGET if they are not defined
+ if "TARGET" not in GlobalData.gGlobalDefines:
+ GlobalData.gGlobalDefines["TARGET"] =
GenFdsGlobalVariable.TargetName
+ if "TOOLCHAIN" not in GlobalData.gGlobalDefines:
+ GlobalData.gGlobalDefines["TOOLCHAIN"] =
GenFdsGlobalVariable.ToolChainTag
+ if "TOOL_CHAIN_TAG" not in GlobalData.gGlobalDefines:
+ GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] =
GenFdsGlobalVariable.ToolChainTag
+
+ """call Workspace build create database"""
+ BuildWorkSpace = WorkSpaceDataBase
+
+ #
+ # Get files real name in workspace dir
+ #
+ GlobalData.gAllFiles = DirCache(Workspace)
+ GlobalData.gWorkspace = Workspace
+
+ if FdsCommandDict["build_architecture_list"]:
+ ArchList = FdsCommandDict["build_architecture_list"].split(',')
+ else:
+ ArchList =
BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON,
FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList
+
+ TargetArchList =
set(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, TAB_COMMON,
FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].SupArchList) &
set(ArchList)
+ if len(TargetArchList) == 0:
+ EdkLogger.error("GenFds", GENFDS_ERROR, "Target ARCH %s not in
platform supported ARCH %s" % (str(ArchList),
str(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform,
TAB_COMMON].SupArchList)))
+
+ for Arch in ArchList:
+ GenFdsGlobalVariable.OutputDirFromDscDict[Arch] =
NormPath(BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, Arch,
FdsCommandDict["build_target"],
FdsCommandDict["toolchain_tag"]].OutputDirectory)
+
+ # assign platform name based on last entry in ArchList
+ GenFdsGlobalVariable.PlatformName =
BuildWorkSpace.BuildObject[GenFdsGlobalVariable.ActivePlatform, ArchList[-1],
FdsCommandDict["build_target"], FdsCommandDict["toolchain_tag"]].PlatformName
+
+ if FdsCommandDict["platform_build_directory"]:
+ OutputDirFromCommandLine =
GenFdsGlobalVariable.ReplaceWorkspaceMacro(FdsCommandDict["platform_build_directory"])
+ if not os.path.isabs (OutputDirFromCommandLine):
+ OutputDirFromCommandLine =
os.path.join(GenFdsGlobalVariable.WorkSpaceDir, OutputDirFromCommandLine)
+ for Arch in ArchList:
+ GenFdsGlobalVariable.OutputDirDict[Arch] =
OutputDirFromCommandLine
+ else:
+ for Arch in ArchList:
+ GenFdsGlobalVariable.OutputDirDict[Arch] =
os.path.join(GenFdsGlobalVariable.OutputDirFromDscDict[Arch],
GenFdsGlobalVariable.TargetName + '_' + GenFdsGlobalVariable.ToolChainTag)
+
+ for Key in GenFdsGlobalVariable.OutputDirDict:
+ OutputDir = GenFdsGlobalVariable.OutputDirDict[Key]
+ if OutputDir[0:2] == '..':
+ OutputDir = os.path.realpath(OutputDir)
+
+ if OutputDir[1] != ':':
+ OutputDir = os.path.join
(GenFdsGlobalVariable.WorkSpaceDir, OutputDir)
+
+ if not os.path.exists(OutputDir):
+ EdkLogger.error("GenFds", FILE_NOT_FOUND,
ExtraData=OutputDir)
+ GenFdsGlobalVariable.OutputDirDict[Key] = OutputDir
+
+ """ Parse Fdf file, has to place after build Workspace as FDF may
contain macros from DSC file """
+ FdfParserObj = GlobalData.gFdfParser
+
+ if FdfParserObj.CycleReferenceCheck():
+ EdkLogger.error("GenFds", FORMAT_NOT_SUPPORTED, "Cycle
Reference Detected in FDF file")
+
+ if FdsCommandDict["fd"]:
+ if FdsCommandDict["fd"][0].upper() in
FdfParserObj.Profile.FdDict:
+ GenFds.OnlyGenerateThisFd = FdsCommandDict["fd"][0]
+ else:
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+ "No such an FD in FDF file: %s" %
FdsCommandDict["fd"][0])
+
+ if FdsCommandDict["fv"]:
+ if FdsCommandDict["fv"][0].upper() in
FdfParserObj.Profile.FvDict:
+ GenFds.OnlyGenerateThisFv = FdsCommandDict["fv"][0]
+ else:
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+ "No such an FV in FDF file: %s" %
FdsCommandDict["fv"][0])
+
+ if FdsCommandDict["cap"]:
+ if FdsCommandDict["cap"][0].upper() in
FdfParserObj.Profile.CapsuleDict:
+ GenFds.OnlyGenerateThisCap = FdsCommandDict["cap"][0]
+ else:
+ EdkLogger.error("GenFds", OPTION_VALUE_INVALID,
+ "No such a Capsule in FDF file: %s" %
FdsCommandDict["cap"][0])
+
+ GenFdsGlobalVariable.WorkSpace = BuildWorkSpace
+ if ArchList:
+ GenFdsGlobalVariable.ArchList = ArchList
+
+ # Dsc Build Data will handle Pcd Settings from CommandLine.
+
+ """Modify images from build output if the feature of loading
driver at fixed address is on."""
+ if GenFdsGlobalVariable.FixedLoadAddress:
+ GenFds.PreprocessImage(BuildWorkSpace,
GenFdsGlobalVariable.ActivePlatform)
+
+ # Record the FV Region info that may specific in the FD
+ if FdfParserObj.Profile.FvDict and FdfParserObj.Profile.FdDict:
+ for FvObj in FdfParserObj.Profile.FvDict.values():
+ for FdObj in FdfParserObj.Profile.FdDict.values():
+ for RegionObj in FdObj.RegionList:
+ if RegionObj.RegionType != BINARY_FILE_TYPE_FV:
+ continue
+ for RegionData in RegionObj.RegionDataList:
+ if FvObj.UiFvName.upper() ==
RegionData.upper():
+ if FvObj.FvRegionInFD:
+ if FvObj.FvRegionInFD !=
RegionObj.Size:
+ EdkLogger.error("GenFds",
FORMAT_INVALID, "The FV %s's region is specified in multiple FD with different
value." %FvObj.UiFvName)
+ else:
+ FvObj.FvRegionInFD = RegionObj.Size
+
RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, FvObj)
+
+ """Call GenFds"""
+ GenFds.GenFd('', FdfParserObj, BuildWorkSpace, ArchList)
+
+ """Generate GUID cross reference file"""
+ GenFds.GenerateGuidXRefFile(BuildWorkSpace, ArchList, FdfParserObj)
+
+ """Display FV space info."""
+ GenFds.DisplayFvSpaceInfo(FdfParserObj)
+
+ except Warning as X:
+ EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName,
Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)
+ ReturnCode = FORMAT_INVALID
+ except FatalError as X:
+ ReturnCode = X.args[0]
+ except:
+ import traceback
+ EdkLogger.error(
+ "\nPython",
+ CODE_ERROR,
+ "Tools code failure",
+ ExtraData="Please send email to
edk2-devel@lists.01.org for help, attaching following call stack trace!\n",
+ RaiseError=False
+ )
+ EdkLogger.quiet(traceback.format_exc())
+ ReturnCode = CODE_ERROR
+ finally:
+ ClearDuplicatedInf()
+ return ReturnCode
+
if __name__ == '__main__':
r = main()
## 0-127 is a safe return range, and 1 is a standard default error
diff --git a/BaseTools/Source/Python/build/build.py
b/BaseTools/Source/Python/build/build.py
index d74082fc26..71940b37e3 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1392,7 +1392,7 @@ class Build():
# genfds
if Target == 'fds':
- LaunchCommand(AutoGenObject.GenFdsCommand,
AutoGenObject.MakeFileDir)
+ GenFds.GenFdsApi(AutoGenObject.GenFdsCommandDict, self.Db)
return True
# run
@@ -2136,7 +2136,7 @@ class Build():
# Generate FD image if there's a FDF file found
#
GenFdsStart = time.time()
- LaunchCommand(Wa.GenFdsCommand, os.getcwd())
+ GenFds.GenFdsApi(Wa.GenFdsCommandDict, self.Db)
#
# Create MAP file for all platform FVs after GenFds.
--
2.14.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
