[edk2-devel] [Patch 1/2] BaseTools: Fixed the build fail on Linux with --genfds-multi-thread

[Bob Feng]

If GenSec input file not exist, the related command will fail and make
will stop on Linux. GenSec input file is allow to be non-existent.

This patch is to let "make" continue if gensec input file not exist.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py 
b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
index 037828ea1cca..0f691ae52420 100644
--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
@@ -487,11 +487,11 @@ class GenFdsGlobalVariable:
 SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
 if IsMakefile:
 if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
 Cmd = ['if', 'exist', Input[0]] + Cmd
 else:
-Cmd = ['test', '-e', Input[0], "&&"] + Cmd
+Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
 if ' '.join(Cmd).strip() not in 
GenFdsGlobalVariable.SecCmdList:
 GenFdsGlobalVariable.SecCmdList.append(' 
'.join(Cmd).strip())
 elif GenFdsGlobalVariable.NeedsUpdate(Output, list(Input) + 
[CommandFile]):
 GenFdsGlobalVariable.DebugLogger(EdkLogger.DEBUG_5, "%s needs 
update because of newer %s" % (Output, Input))
 GenFdsGlobalVariable.CallExternalTool(Cmd, "Failed to generate 
section")
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46862): https://edk2.groups.io/g/devel/message/46862
Mute This Topic: https://groups.io/mt/33146588/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 0/2] Enable --genfds-multi-thread to default build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302

Enable --genfds-multi-thread as default build behavior can improve 
build performance but will not bring nagtive impact.

Feng, Bob C (2):
  BaseTools: Fixed the build fail on Linux with --genfds-multi-thread
  BaseTools: Enable --genfds-multi-thread to default build

 BaseTools/Source/Python/AutoGen/GenMake.py | 4 ++--
 BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py| 3 +--
 BaseTools/Source/Python/Common/GlobalData.py   | 2 +-
 BaseTools/Source/Python/Common/buildoptions.py | 3 ++-
 BaseTools/Source/Python/GenFds/GenFds.py   | 9 ++---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 4 ++--
 BaseTools/Source/Python/build/build.py | 2 +-
 7 files changed, 15 insertions(+), 12 deletions(-)

-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46861): https://edk2.groups.io/g/devel/message/46861
Mute This Topic: https://groups.io/mt/33146585/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 2/2] BaseTools: Enable --genfds-multi-thread to default build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302

This patch enable --genfds-multi-thread to default build.
This patch keep --genfds-multi-thread build option for
compatibility and also add a new build option to disable
genfds-multi-thread as --no-genfds-multi-thread.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 4 ++--
 BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py| 3 +--
 BaseTools/Source/Python/Common/GlobalData.py   | 2 +-
 BaseTools/Source/Python/Common/buildoptions.py | 3 ++-
 BaseTools/Source/Python/GenFds/GenFds.py   | 9 ++---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
 BaseTools/Source/Python/build/build.py | 2 +-
 7 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 47dae82e1aeb..4f85a93055ab 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1578,12 +1578,12 @@ class TopLevelMakefile(BuildFile):
 elif LogLevel == EdkLogger.QUIET:
 ExtraOption += " -q"
 
 if GlobalData.gCaseInsensitive:
 ExtraOption += " -c"
-if GlobalData.gEnableGenfdsMultiThread:
-ExtraOption += " --genfds-multi-thread"
+if not GlobalData.gEnableGenfdsMultiThread:
+ExtraOption += " --no-genfds-multi-thread"
 if GlobalData.gIgnoreSource:
 ExtraOption += " --ignore-sources"
 
 for pcd in GlobalData.BuildOptionPcd:
 if pcd[2]:
diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py 
b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 24942674721f..365cfdefeaa1 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -832,12 +832,11 @@ class WorkspaceAutoGen(AutoGen):
 elif LogLevel <= EdkLogger.DEBUG_9:
 FdsCommandDict["debug"] = LogLevel - 1
 elif LogLevel == EdkLogger.QUIET:
 FdsCommandDict["quiet"] = True
 
-if GlobalData.gEnableGenfdsMultiThread:
-FdsCommandDict["GenfdsMultiThread"] = True
+FdsCommandDict["GenfdsMultiThread"] = 
GlobalData.gEnableGenfdsMultiThread
 if GlobalData.gIgnoreSource:
 FdsCommandDict["IgnoreSources"] = True
 
 FdsCommandDict["OptionPcd"] = []
 for pcd in GlobalData.BuildOptionPcd:
diff --git a/BaseTools/Source/Python/Common/GlobalData.py 
b/BaseTools/Source/Python/Common/GlobalData.py
index 61327ad8f10e..8eb72aa1d6f5 100755
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -104,11 +104,11 @@ gUseHashCache = None
 gBinCacheDest = None
 gBinCacheSource = None
 gPlatformHash = None
 gPackageHash = {}
 gModuleHash = {}
-gEnableGenfdsMultiThread = False
+gEnableGenfdsMultiThread = True
 gSikpAutoGenCache = set()
 
 # Dictionary for tracking Module build status as success or failure
 # Top Dict: Key: Arch Type  Value: Dictionary
 # Second Dict:  Key: AutoGen ObjValue: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
diff --git a/BaseTools/Source/Python/Common/buildoptions.py 
b/BaseTools/Source/Python/Common/buildoptions.py
index 7161aa66f23e..a717c58d8c9a 100644
--- a/BaseTools/Source/Python/Common/buildoptions.py
+++ b/BaseTools/Source/Python/Common/buildoptions.py
@@ -82,11 +82,12 @@ def MyOptionParser():
 Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set 
PCD value by command line. Format: \"PcdName=Value\" ")
 Parser.add_option("-l", "--cmd-len", action="store", type="int", 
dest="CommandLength", help="Specify the maximum line length of build command. 
Default is 4096.")
 Parser.add_option("--hash", action="store_true", dest="UseHashCache", 
default=False, help="Enable hash-based caching during build process.")
 Parser.add_option("--binary-destination", action="store", type="string", 
dest="BinCacheDest", help="Generate a cache of binary files in the specified 
directory.")
 Parser.add_option("--binary-source", action="store", type="string", 
dest="BinCacheSource", help="Consume a cache of binary files from the specified 
directory.")
-Parser.add_option("--genfds-multi-thread", action="store_true", 
dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to 
generate ffs file.")
+Parser.add_option("--genfds-multi-thread", action="

Re: [edk2-devel] [edk2] [Patch 32/33] BaseTools: ECC tool Python3 adaption

[Bob Feng]

odule_as_main
> > > "__main__", mod_spec)
> > >   File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
> > > exec(code, run_globals)
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/EccMain.py", line 410, 
> > > in 
> > > Ecc = Ecc()
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/EccMain.py", line 94, 
> > > in __init__
> > > self.DetectOnlyScanDirs()
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/EccMain.py", line 130, 
> > > in DetectOnlyScanDirs
> > > self.BuildDatabase()
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/EccMain.py", line 150, 
> > > in BuildDatabase
> > > c.CollectSourceCodeDataIntoDB(EccGlobalData.gTarget)
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/c.py", line 526, in 
> > > CollectSourceCodeDataIntoDB
> > > collector.ParseFile()
> > >   File 
> > > "/work/git/edk2/BaseTools/Source/Python/Ecc/CodeFragmentCollector.py", 
> > > line 517, in ParseFile
> > > lexer = CLexer(cStream)
> > >   File "/work/git/edk2/BaseTools/Source/Python/Ecc/CParser3/CLexer.py", 
> > > line 147, in __init__
> > > Lexer.__init__(self, input)
> > >   File "/usr/lib/python3/dist-packages/antlr3/recognizers.py", line 1039, 
> > > in __init__
> > > BaseRecognizer.__init__(self, state)
> > >   File "/usr/lib/python3/dist-packages/antlr3/recognizers.py", line 169, 
> > > in __init__
> > > .format(self.api_version))
> > > RuntimeError: ANTLR version mismatch: The recognizer has been generated 
> > > with API V0, but this runtime does not support this.
> > > ---
> > > 
> > > Any idea?
> > > 
> > > Best Regards,
> > > 
> > > Leif
> > > 
> > > On Tue, Jan 29, 2019 at 10:06:09AM +0800, Feng, Bob C wrote:
> > > > v2:
> > > > The python files under CParser4 are generated by antlr4 and for
> > > > python3 usage. They have python3 specific syntax, for example 
> > > > the data type declaration for the arguments of a function. That 
> > > > is not compitable with python2. this patch is to remove these syntax.
> > > > 
> > > > ECC tool Python3 adaption.
> > > > 
> > > > Contributed-under: TianoCore Contribution Agreement 1.1
> > > > Signed-off-by: Bob Feng 
> > > > Cc: Liming Gao 
> > > > ---
> > > >  BaseTools/Source/Python/Ecc/{ => CParser3}/CLexer.py  |0
> > > >  BaseTools/Source/Python/Ecc/{ => CParser3}/CParser.py |0
> > > >  BaseTools/Source/Python/Ecc/CParser3/__init__.py  |0
> > > >  BaseTools/Source/Python/Ecc/CParser4/C.g4 |  637 
> > > > +++
> > > >  BaseTools/Source/Python/Ecc/CParser4/CLexer.py|  632 
> > > > +++
> > > >  BaseTools/Source/Python/Ecc/CParser4/CListener.py |  815 
> > > > ++
> > > >  BaseTools/Source/Python/Ecc/CParser4/CParser.py   | 6279 
> > > > 
> > > >  +
> > > >  BaseTools/Source/Python/Ecc/CParser4/__init__.py  |0
> > > >  BaseTools/Source/Python/Ecc/Check.py  |4 +-
> > > >  BaseTools/Source/Python/Ecc/CodeFragmentCollector.py  |   20 +--
> > > >  BaseTools/Source/Python/Ecc/Configuration.py  |3 -
> > > >  BaseTools/Source/Python/Ecc/EccMain.py|2 +-
> > > >  BaseTools/Source/Python/Ecc/EccToolError.py   |4 +-
> > > >  BaseTools/Source/Python/Ecc/FileProfile.py|2 +-
> > > >  BaseTools/Source/Python/Ecc/MetaDataParser.py |2 +-
> > > >  BaseTools/Source/Python/Ecc/c.py  |6 +-
> > > >  BaseTools/Source/Python/Ecc/config.ini|2 -
> > > >  17 files changed, 8385 insertions(+), 23 deletions(-)
> > 
> > 
> > 

-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46870): https://edk2.groups.io/g/devel/message/46870
Mute This Topic: https://groups.io/mt/33115040/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

[Bob Feng]

https://bugzilla.tianocore.org/show_bug.cgi?id=2080

This patch is to improve build -u option to re-use
GlobalVar__.bin file which is
introduced by multiple-process-autogen feature.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 .../Source/Python/AutoGen/AutoGenWorker.py|   2 +-
 .../Source/Python/AutoGen/ModuleAutoGen.py|   4 +-
 .../Python/AutoGen/ModuleAutoGenHelper.py |  29 ++
 .../Source/Python/AutoGen/PlatformAutoGen.py  |   2 +
 .../Source/Python/AutoGen/WorkspaceAutoGen.py |   1 -
 BaseTools/Source/Python/build/build.py| 355 +++---
 6 files changed, 255 insertions(+), 138 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py 
b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 2e68538b1cb4..f488ae9d5f80 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
 Ma.ReferenceModules = 
Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
 if GlobalData.gBinCacheSource and CommandTarget in [None, "", 
"all"]:
 Ma.GenModuleFilesHash(GlobalData.gCacheIR)
 Ma.GenPreMakefileHash(GlobalData.gCacheIR)
 if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
-   continue
+continue
 
 Ma.CreateCodeFile(False)
 
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
 
 if GlobalData.gBinCacheSource and CommandTarget in [None, "", 
"all"]:
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 
b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index 3bb7e91154ac..5f28681e3146 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
 self.GenModuleFilesHash(gDict)
 
 if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
-   EdkLogger.quiet("[cache warning]: Cannot generate 
ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
-   return
+EdkLogger.quiet("[cache warning]: Cannot generate 
ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch))
+return
 
 # Initialze hash object
 m = hashlib.md5()
 
 # Add Platform level hash
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py 
b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
index c7591253debd..9dd93b9beb12 100644
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
@@ -234,22 +234,51 @@ class AutoGenInfo(object):
 #
 #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in multiple thread
 #
 class WorkSpaceInfo(AutoGenInfo):
 def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
+if not hasattr(self, "_Init"):
+self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
+self._Init = True
+def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
 self._SrcTimeStamp = 0
 self.Db = BuildDB
 self.BuildDatabase = self.Db.BuildObject
 self.Target = Target
 self.ToolChain = ToolChain
 self.WorkspaceDir = Workspace
 self.ActivePlatform = MetaFile
 self.ArchList = Arch
+self.AutoGenObjectList = []
+@property
+def BuildDir(self):
+return self.AutoGenObjectList[0].BuildDir
 
+@property
+def Name(self):
+return self.AutoGenObjectList[0].Platform.PlatformName
+
+@property
+def FlashDefinition(self):
+return self.AutoGenObjectList[0].Platform.FlashDefinition
+@property
+def GenFdsCommandDict(self):
+FdsCommandDict = 
self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
+if FdsCommandDict:
+return FdsCommandDict
+return {}
+
+@cached_property
+def FvDir(self):
+return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
 
 class PlatformInfo(AutoGenInfo):
 def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
+if not hasattr(self, "_Init"):
+self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
+self._Init = True
+def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
 self.Wa = Workspace
 self.WorkspaceDir = self.Wa.WorkspaceDir
 self.MetaFile = MetaFile
 self.Arch = Arch
 self.Target = Target
diff --g

[edk2-devel] [Patch 1/1] BaseTools: Fix a bug for Hii Pcd override

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2157

Hii Pcd links to a efi variable. The Variable
default value is evaluated by related Hii Pcds
setting. If multiple Hii Pcds links to one variable,
and the offset overlap, the later Hii Pcds setting
should be effective. There is a tool bug that is if
the Pcds are in different dsc file which are included
into the platform dsc file, build tool does not get
the Pcds relative position correctly. That means
build tool does not know which Pcd is the later one. As
the result, the variable default value will be incorrect.

This patch is to fix this bug.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/Workspace/DscBuildData.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py 
b/BaseTools/Source/Python/Workspace/DscBuildData.py
index d2b5ccbb7a30..9192077f9064 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -3012,11 +3012,11 @@ class DscBuildData(PlatformBuildClassObject):
 PcdList.append((PcdCName, TokenSpaceGuid, SkuName, 
DefaultStore, Dummy5))
 PcdDict[Arch, SkuName, PcdCName, TokenSpaceGuid, DefaultStore] = 
Setting
 
 
 # Remove redundant PCD candidates, per the ARCH and SKU
-for PcdCName, TokenSpaceGuid, SkuName, DefaultStore, Dummy4 in PcdList:
+for index,(PcdCName, TokenSpaceGuid, SkuName, DefaultStore, Dummy4) in 
enumerate(PcdList):
 
 Setting = PcdDict[self._Arch, SkuName, PcdCName, TokenSpaceGuid, 
DefaultStore]
 if Setting is None:
 continue
 VariableName, VariableGuid, VariableOffset, DefaultValue, 
VarAttribute = self._ValidatePcd(PcdCName, TokenSpaceGuid, Setting, Type, 
Dummy4)
@@ -3081,11 +3081,11 @@ class DscBuildData(PlatformBuildClassObject):
 IsDsc=True)
 if (PcdCName, TokenSpaceGuid) in UserDefinedDefaultStores:
 PcdClassObj.UserDefinedDefaultStoresFlag = True
 Pcds[PcdCName, TokenSpaceGuid] = PcdClassObj
 
-Pcds[PcdCName, TokenSpaceGuid].CustomAttribute['DscPosition'] 
= int(Dummy4)
+Pcds[PcdCName, TokenSpaceGuid].CustomAttribute['DscPosition'] 
= index
 if SkuName not in Pcds[PcdCName, TokenSpaceGuid].DscRawValue:
 Pcds[PcdCName, TokenSpaceGuid].DscRawValue[SkuName] = {}
 Pcds[PcdCName, TokenSpaceGuid].DscRawValueInfo[SkuName] = {}
 Pcds[PcdCName, TokenSpaceGuid].DscRawValue[SkuName][DefaultStore] 
= DefaultValue
 Pcds[PcdCName, 
TokenSpaceGuid].DscRawValueInfo[SkuName][DefaultStore] = 
(self.MetaFile.File,Dummy4)
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46890): https://edk2.groups.io/g/devel/message/46890
Mute This Topic: https://groups.io/mt/33151625/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 0/2] Enable --genfds-multi-thread to default build

[Bob Feng]

Emulator
Ovmf
MinPlatforms
And Intel Server Platform.

-Bob

-Original Message-
From: Gao, Liming 
Sent: Thursday, September 5, 2019 9:09 PM
To: devel@edk2.groups.io; Feng, Bob C 
Subject: RE: [edk2-devel] [Patch 0/2] Enable --genfds-multi-thread to default 
build

Bob:
  Can you list the platform list have been verified with this change?

Thanks
Liming> -Original Message-
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of 
> Bob Feng
> Sent: Thursday, September 5, 2019 11:11 AM
> To: devel@edk2.groups.io
> Subject: [edk2-devel] [Patch 0/2] Enable --genfds-multi-thread to 
> default build
> 
> BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302
> 
> Enable --genfds-multi-thread as default build behavior can improve 
> build performance but will not bring nagtive impact.
> 
> Feng, Bob C (2):
>   BaseTools: Fixed the build fail on Linux with --genfds-multi-thread
>   BaseTools: Enable --genfds-multi-thread to default build
> 
>  BaseTools/Source/Python/AutoGen/GenMake.py | 4 ++--
>  BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py| 3 +--
>  BaseTools/Source/Python/Common/GlobalData.py   | 2 +-
>  BaseTools/Source/Python/Common/buildoptions.py | 3 ++-
>  BaseTools/Source/Python/GenFds/GenFds.py   | 9 ++---
>  BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 4 ++--
>  BaseTools/Source/Python/build/build.py | 2 +-
>  7 files changed, 15 insertions(+), 12 deletions(-)
> 
> --
> 2.20.1.windows.1
> 
> 
> 


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46898): https://edk2.groups.io/g/devel/message/46898
Mute This Topic: https://groups.io/mt/33146585/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

[Bob Feng]

Liming,

Yes, this change only impact -u option. 

I separate _MultiThreadBuildPlatform function into two functions and create 2 
new functions specially for IgnoreAutoGen, so the there looks  big change for 
build.py in this patch. I verified this patch on Ovmf, E

-Bob

-Original Message-
From: Gao, Liming 
Sent: Thursday, September 5, 2019 9:12 PM
To: devel@edk2.groups.io; Feng, Bob C 
Cc: Shi, Steven 
Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

Bob: 
 Does this change only impact -u option behavior? 

Thanks
Liming
> -Original Message-
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of 
> Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel@edk2.groups.io
> Cc: Gao, Liming ; Shi, Steven 
> ; Feng, Bob C 
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for 
> IgnoreAutoGen
> 
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
> 
> This patch is to improve build -u option to re-use 
> GlobalVar__.bin file which is introduced by 
> multiple-process-autogen feature.
> 
> Cc: Liming Gao 
> Cc: Steven Shi 
> Signed-off-by: Bob Feng 
> ---
>  .../Source/Python/AutoGen/AutoGenWorker.py|   2 +-
>  .../Source/Python/AutoGen/ModuleAutoGen.py|   4 +-
>  .../Python/AutoGen/ModuleAutoGenHelper.py |  29 ++
>  .../Source/Python/AutoGen/PlatformAutoGen.py  |   2 +
>  .../Source/Python/AutoGen/WorkspaceAutoGen.py |   1 -
>  BaseTools/Source/Python/build/build.py| 355 +++---
>  6 files changed, 255 insertions(+), 138 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py 
> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
>  Ma.ReferenceModules = 
> Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
>  if GlobalData.gBinCacheSource and CommandTarget in [None, 
> "", "all"]:
>  Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>  Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>  if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> -   continue
> +continue
> 
>  Ma.CreateCodeFile(False)
>
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, 
> Ma.Arch),[]))
> 
>  if GlobalData.gBinCacheSource and CommandTarget in [None, 
> "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>  self.GenModuleFilesHash(gDict)
> 
>  if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> -   EdkLogger.quiet("[cache warning]: Cannot generate 
> ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> -   return
> +EdkLogger.quiet("[cache warning]: Cannot generate 
> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> +return
> 
>  # Initialze hash object
>  m = hashlib.md5()
> 
>  # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py 
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
>  #
>  #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in 
> multiple thread  #  class WorkSpaceInfo(AutoGenInfo):
>  def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> +if not hasattr(self, "_Init"):
> +self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
> +self._Init = True
> +def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
>  self._SrcTimeStamp = 0
>  self.Db = BuildDB
>  self.BuildDatabase = self.Db.BuildObject
>  self.Target = Target
>  self.ToolChain = ToolChain
>   

Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

[Bob Feng]

Sorry for the incomplete mail.

I verified this patch on Ovmf, Emulator, MinPlatforms and Intel Server Platform.

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Bob Feng
Sent: Thursday, September 5, 2019 10:04 PM
To: Gao, Liming ; devel@edk2.groups.io
Cc: Shi, Steven 
Subject: Re: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

Liming,

Yes, this change only impact -u option. 

I separate _MultiThreadBuildPlatform function into two functions and create 2 
new functions specially for IgnoreAutoGen, so the there looks  big change for 
build.py in this patch. I verified this patch on Ovmf, E

-Bob

-Original Message-
From: Gao, Liming
Sent: Thursday, September 5, 2019 9:12 PM
To: devel@edk2.groups.io; Feng, Bob C 
Cc: Shi, Steven 
Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen

Bob: 
 Does this change only impact -u option behavior? 

Thanks
Liming
> -Original Message-
> From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of 
> Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel@edk2.groups.io
> Cc: Gao, Liming ; Shi, Steven 
> ; Feng, Bob C 
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for 
> IgnoreAutoGen
> 
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
> 
> This patch is to improve build -u option to re-use 
> GlobalVar__.bin file which is introduced by 
> multiple-process-autogen feature.
> 
> Cc: Liming Gao 
> Cc: Steven Shi 
> Signed-off-by: Bob Feng 
> ---
>  .../Source/Python/AutoGen/AutoGenWorker.py|   2 +-
>  .../Source/Python/AutoGen/ModuleAutoGen.py|   4 +-
>  .../Python/AutoGen/ModuleAutoGenHelper.py |  29 ++
>  .../Source/Python/AutoGen/PlatformAutoGen.py  |   2 +
>  .../Source/Python/AutoGen/WorkspaceAutoGen.py |   1 -
>  BaseTools/Source/Python/build/build.py| 355 +++---
>  6 files changed, 255 insertions(+), 138 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
>  Ma.ReferenceModules = 
> Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
>  if GlobalData.gBinCacheSource and CommandTarget in [None, 
> "", "all"]:
>  Ma.GenModuleFilesHash(GlobalData.gCacheIR)
>  Ma.GenPreMakefileHash(GlobalData.gCacheIR)
>  if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> -   continue
> +continue
> 
>  Ma.CreateCodeFile(False)
>
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
> Ma.Arch),[]))
> 
>  if GlobalData.gBinCacheSource and CommandTarget in [None, 
> "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
>  self.GenModuleFilesHash(gDict)
> 
>  if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> -   EdkLogger.quiet("[cache warning]: Cannot generate 
> ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> -   return
> +EdkLogger.quiet("[cache warning]: Cannot generate 
> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> +return
> 
>  # Initialze hash object
>  m = hashlib.md5()
> 
>  # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
>  #
>  #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in 
> multiple thread  #  class WorkSpaceInfo(AutoGenInfo):
>  def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> +if not hasattr(self, "_Init"):
> +self

[edk2-devel] [Patch 2/3] BaseTools: Enable --genfds-multi-thread to default build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302

This patch enable --genfds-multi-thread to default build.
This patch keep --genfds-multi-thread build option for
compatibility and also add a new build option to disable
genfds-multi-thread as --no-genfds-multi-thread.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 4 ++--
 BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py| 3 +--
 BaseTools/Source/Python/Common/GlobalData.py   | 2 +-
 BaseTools/Source/Python/Common/buildoptions.py | 3 ++-
 BaseTools/Source/Python/GenFds/GenFds.py   | 9 ++---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
 BaseTools/Source/Python/build/build.py | 2 +-
 7 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 47dae82e1aeb..4f85a93055ab 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1578,12 +1578,12 @@ class TopLevelMakefile(BuildFile):
 elif LogLevel == EdkLogger.QUIET:
 ExtraOption += " -q"
 
 if GlobalData.gCaseInsensitive:
 ExtraOption += " -c"
-if GlobalData.gEnableGenfdsMultiThread:
-ExtraOption += " --genfds-multi-thread"
+if not GlobalData.gEnableGenfdsMultiThread:
+ExtraOption += " --no-genfds-multi-thread"
 if GlobalData.gIgnoreSource:
 ExtraOption += " --ignore-sources"
 
 for pcd in GlobalData.BuildOptionPcd:
 if pcd[2]:
diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py 
b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 24942674721f..365cfdefeaa1 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -832,12 +832,11 @@ class WorkspaceAutoGen(AutoGen):
 elif LogLevel <= EdkLogger.DEBUG_9:
 FdsCommandDict["debug"] = LogLevel - 1
 elif LogLevel == EdkLogger.QUIET:
 FdsCommandDict["quiet"] = True
 
-if GlobalData.gEnableGenfdsMultiThread:
-FdsCommandDict["GenfdsMultiThread"] = True
+FdsCommandDict["GenfdsMultiThread"] = 
GlobalData.gEnableGenfdsMultiThread
 if GlobalData.gIgnoreSource:
 FdsCommandDict["IgnoreSources"] = True
 
 FdsCommandDict["OptionPcd"] = []
 for pcd in GlobalData.BuildOptionPcd:
diff --git a/BaseTools/Source/Python/Common/GlobalData.py 
b/BaseTools/Source/Python/Common/GlobalData.py
index 61327ad8f10e..8eb72aa1d6f5 100755
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -104,11 +104,11 @@ gUseHashCache = None
 gBinCacheDest = None
 gBinCacheSource = None
 gPlatformHash = None
 gPackageHash = {}
 gModuleHash = {}
-gEnableGenfdsMultiThread = False
+gEnableGenfdsMultiThread = True
 gSikpAutoGenCache = set()
 
 # Dictionary for tracking Module build status as success or failure
 # Top Dict: Key: Arch Type  Value: Dictionary
 # Second Dict:  Key: AutoGen ObjValue: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
diff --git a/BaseTools/Source/Python/Common/buildoptions.py 
b/BaseTools/Source/Python/Common/buildoptions.py
index 7161aa66f23e..a717c58d8c9a 100644
--- a/BaseTools/Source/Python/Common/buildoptions.py
+++ b/BaseTools/Source/Python/Common/buildoptions.py
@@ -82,11 +82,12 @@ def MyOptionParser():
 Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set 
PCD value by command line. Format: \"PcdName=Value\" ")
 Parser.add_option("-l", "--cmd-len", action="store", type="int", 
dest="CommandLength", help="Specify the maximum line length of build command. 
Default is 4096.")
 Parser.add_option("--hash", action="store_true", dest="UseHashCache", 
default=False, help="Enable hash-based caching during build process.")
 Parser.add_option("--binary-destination", action="store", type="string", 
dest="BinCacheDest", help="Generate a cache of binary files in the specified 
directory.")
 Parser.add_option("--binary-source", action="store", type="string", 
dest="BinCacheSource", help="Consume a cache of binary files from the specified 
directory.")
-Parser.add_option("--genfds-multi-thread", action="store_true", 
dest="GenfdsMultiThread", default=False, help="Enable GenFds multi thread to 
generate ffs file.")
+Parser.add_option("--genfds-multi-thread", action="

[edk2-devel] [Patch 0/3 V2]Enable --genfds-multi-thread to default build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302

Enable --genfds-multi-thread as default build behavior can improve 
build performance but will not bring nagtive impact.
Feng, Bob C (3):
  BaseTools: Fixed the build fail on Linux with --genfds-multi-thread
  BaseTools: Enable --genfds-multi-thread to default build
  BaseTools: Fixed the bug of multi-thread genffs for override inf

 BaseTools/Source/Python/AutoGen/AutoGenWorker.py   |  2 +-
 BaseTools/Source/Python/AutoGen/GenMake.py |  4 ++--
 BaseTools/Source/Python/AutoGen/ModuleAutoGen.py   |  2 +-
 BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py|  3 +--
 BaseTools/Source/Python/Common/GlobalData.py   |  2 +-
 BaseTools/Source/Python/Common/buildoptions.py |  3 ++-
 BaseTools/Source/Python/GenFds/FfsInfStatement.py  |  5 -
 BaseTools/Source/Python/GenFds/GenFds.py   |  9 ++---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py |  4 ++--
 BaseTools/Source/Python/build/build.py | 10 +-
 10 files changed, 25 insertions(+), 19 deletions(-)

-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46976): https://edk2.groups.io/g/devel/message/46976
Mute This Topic: https://groups.io/mt/33166298/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/3] BaseTools: Fixed the build fail on Linux with --genfds-multi-thread

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302

If GenSec input file not exist, the related command will fail and make
will stop on Linux. GenSec input file is allow to be non-existent.

This patch is to let "make" continue if gensec input file not exist.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py 
b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
index 037828ea1cca..0f691ae52420 100644
--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
@@ -487,11 +487,11 @@ class GenFdsGlobalVariable:
 SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
 if IsMakefile:
 if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
 Cmd = ['if', 'exist', Input[0]] + Cmd
 else:
-Cmd = ['test', '-e', Input[0], "&&"] + Cmd
+Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
 if ' '.join(Cmd).strip() not in 
GenFdsGlobalVariable.SecCmdList:
 GenFdsGlobalVariable.SecCmdList.append(' 
'.join(Cmd).strip())
 elif GenFdsGlobalVariable.NeedsUpdate(Output, list(Input) + 
[CommandFile]):
 GenFdsGlobalVariable.DebugLogger(EdkLogger.DEBUG_5, "%s needs 
update because of newer %s" % (Output, Input))
 GenFdsGlobalVariable.CallExternalTool(Cmd, "Failed to generate 
section")
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#46977): https://edk2.groups.io/g/devel/message/46977
Mute This Topic: https://groups.io/mt/33166299/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 3/3] BaseTools: Fixed the bug of multi-thread genffs for override inf

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302
If there is a inf override, and multi-thread genffs is enabled, build
will fail.

This patch is going to fix this bug.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/AutoGenWorker.py  | 2 +-
 BaseTools/Source/Python/AutoGen/ModuleAutoGen.py  | 2 +-
 BaseTools/Source/Python/GenFds/FfsInfStatement.py | 5 -
 BaseTools/Source/Python/build/build.py| 8 
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py 
b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 2e68538b1cb4..0d0ee319393c 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -239,11 +239,11 @@ class AutoGenWorkerInProcess(mp.Process):
 Ma.GenPreMakefileHash(GlobalData.gCacheIR)
 if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
continue
 
 Ma.CreateCodeFile(False)
-
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))
+
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
 
 if GlobalData.gBinCacheSource and CommandTarget in [None, "", 
"all"]:
 Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)
 Ma.GenMakeHash(GlobalData.gCacheIR)
 if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 
b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index 3bb7e91154ac..8250b93fd440 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -2121,11 +2121,11 @@ class ModuleAutoGen(AutoGen):
 if 
(self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path) in 
GlobalData.Refes:
 self.ReferenceModules = 
GlobalData.Refes[(self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path)]
 self.CreateCodeFile()
 if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone:
-
self.CreateMakeFile(GenFfsList=GlobalData.FfsCmd.get((self.MetaFile.File, 
self.Arch),[]))
+
self.CreateMakeFile(GenFfsList=GlobalData.FfsCmd.get((self.MetaFile.Path, 
self.Arch),[]))
 
 if not (self.MetaFile.Path, self.Arch) in gDict or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateCodeFileDone or \
not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone:
EdkLogger.quiet("[cache warning]: Cannot create CodeFile or 
Makefile for module %s[%s]" %(self.MetaFile.Path, self.Arch))
diff --git a/BaseTools/Source/Python/GenFds/FfsInfStatement.py 
b/BaseTools/Source/Python/GenFds/FfsInfStatement.py
index ebb93b14c4ff..43dbd0102f5e 100644
--- a/BaseTools/Source/Python/GenFds/FfsInfStatement.py
+++ b/BaseTools/Source/Python/GenFds/FfsInfStatement.py
@@ -500,11 +500,14 @@ class FfsInfStatement(FfsInfStatementClassObject):
 #
 MakefilePath = None
 if self.IsBinaryModule:
 IsMakefile = False
 if IsMakefile:
-MakefilePath = self.InfFileName, Arch
+PathClassObj = PathClass(self.InfFileName, 
GenFdsGlobalVariable.WorkSpaceDir)
+if self.OverrideGuid:
+PathClassObj = ProcessDuplicatedInf(PathClassObj, 
self.OverrideGuid, GenFdsGlobalVariable.WorkSpaceDir)
+MakefilePath = PathClassObj.Path, Arch
 if isinstance (Rule, RuleSimpleFile.RuleSimpleFile):
 SectionOutputList = self.__GenSimpleFileSection__(Rule, 
IsMakefile=IsMakefile)
 FfsOutput = self.__GenSimpleFileFfs__(Rule, SectionOutputList, 
MakefilePath=MakefilePath)
 return FfsOutput
 #
diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index 2a10f99a1e70..94318420196f 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -840,11 +840,11 @@ class Build():
 PcdMa.GenPreMakefileHash(share_data)
 if PcdMa.CanSkipbyPreMakefileCache(share_data):
continue
 
 PcdMa.CreateCodeFile(False)
-PcdMa.CreateMakeFile(False,GenFfsList = 
DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
+PcdMa.CreateMakeFile(False,GenFfsList = 
DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
 
 if GlobalData.gBinCacheSource and self.Target in [None, 
"", "all"]:
 PcdMa.GenMakeHeaderFilesHash(share_data)
 PcdMa.GenMakeHash(share

[edk2-devel] [Patch 1/1] BaseTools: Fixed build clean regression issue

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2166

file_lock and cache_lock are used as global data,
so move file_lock and cache_lock initialization in Build
object __init__ function.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/build/build.py | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index 0406ac314b65..cf82c29b7eb9 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -810,27 +810,25 @@ class Build():
 self.AutoGenMgr = None
 EdkLogger.info("")
 os.chdir(self.WorkspaceDir)
 GlobalData.gCacheIR = Manager().dict()
 self.log_q = log_q
+GlobalData.file_lock =  mp.Lock()
+GlobalData.cache_lock = mp.Lock()
 def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):
 try:
 if SkipAutoGen:
 return True,0
 feedback_q = mp.Queue()
-file_lock = mp.Lock()
 error_event = mp.Event()
-GlobalData.file_lock = file_lock
-cache_lock = mp.Lock()
-GlobalData.cache_lock = cache_lock
 FfsCmd = DataPipe.Get("FfsCommand")
 if FfsCmd is None:
 FfsCmd = {}
 GlobalData.FfsCmd = FfsCmd
 GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")
 GlobalData.Refes = DataPipe.Get("REFS")
-auto_workers = 
[AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,cache_lock,share_data,self.log_q,error_event)
 for _ in range(self.ThreadNumber)]
+auto_workers = 
[AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,GlobalData.cache_lock,share_data,self.log_q,error_event)
 for _ in range(self.ThreadNumber)]
 self.AutoGenMgr = 
AutoGenManager(auto_workers,feedback_q,error_event)
 self.AutoGenMgr.start()
 for w in auto_workers:
 w.start()
 if PcdMaList is not None:
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47014): https://edk2.groups.io/g/devel/message/47014
Mute This Topic: https://groups.io/mt/34074137/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] commit "BaseTools: Improve the cache hit in the edk2 build cache" causes build.py unknown fatal error

[Bob Feng]

Hi Vladimir and Steven,

I checked code and think when the build target are 'clean', 'cleanlib', 
'cleanall', 'run' and 'fds', build will fail with the error like you mentioned.
I filed a Bugzilla https://bugzilla.tianocore.org/show_bug.cgi?id=2166
And provide a fix in https://edk2.groups.io/g/devel/message/47014

Thanks,
Bob

From: Shi, Steven
Sent: Saturday, September 7, 2019 10:16 AM
To: Vladimir Olovyannikov ; 
devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C ; 
Rodriguez, Christian ; Johnson, Michael 

Subject: RE: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Vladimir,
Did you enable the build cache? Could you let me know your build steps and 
commands?


Thanks
Steven

From: Vladimir Olovyannikov [mailto:vladimir.olovyanni...@broadcom.com]
Sent: Saturday, September 7, 2019 4:45 AM
To: devel@edk2.groups.io; Shi, Steven 
mailto:steven@intel.com>>
Cc: Gao, Liming mailto:liming@intel.com>>; Feng, Bob 
C mailto:bob.c.f...@intel.com>>; Rodriguez, Christian 
mailto:christian.rodrig...@intel.com>>; Johnson, 
Michael mailto:michael.john...@intel.com>>
Subject: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Steven,

The commit
0e7e7a264cd80ab71ea0f9e9da2d0617d4b539c4
BaseTools: Improve the cache hit in the edk2 build cache
Author: Shi, Steven mailto:steven@intel.com>>  
2019-08-15 07:26:17
Committer: Feng, Bob C mailto:bob.c.f...@intel.com>>  
2019-08-19 23:06:05
Parent: 811328172841ce3f7ec1fe7033c130726ad189b7 
(ShellPkg/UefiShellDriver1CommandsLib: Make array big enough)
Child:  56c786b0a9fd4f08b1f0bc10b7718a7577cb0146 (BaseTools: Print first cache 
missing file for build cachle)
Branches: master, remotes/origin/master
Follows: edk2-stable201905
Precedes: edk2-stable201908

causes "unknown fatal error" in build.py when building our platform.

Here is the log:
...
build.py...
 : error C0DE: Unknown fatal error when processing 
[/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/ArmPkg/Library/ArmGicArchSecLib/ArmGicArchSecLib.inf]

(Please send email to devel@edk2.groups.io for 
help, attaching following call stack trace!)

(Python 3.6.8 on linux) Traceback (most recent call last):
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 2493, in Main
MyBuild.Launch()
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 2287, in Launch
self._BuildPlatform()
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 1742, in _BuildPlatform
self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 1236, in _BuildPa
AutoGenObject.CreateCodeFile(True)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/Common/caching.py",
 line 18, in CallMeHere
Value = self._function(obj, *args,**kwargs)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py",
 line 171, in CreateCodeFile
Ma.CreateCodeFile(CreateModuleCodeFile)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py",
 line 1841, in CreateCodeFile
LibraryAutoGen.CreateCodeFile()
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py",
 line 1886, in CreateCodeFile
with GlobalData.cache_lock:
AttributeError: __enter__


- Failed -
Build end time: 13:31:30, Sep.06 2019
Build total time: 00:00:04

Builds before the above mentioned commit was pushed, were fine.

Thank you,
Vladimir

-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47016): https://edk2.groups.io/g/devel/message/47016
Mute This Topic: https://groups.io/mt/33171550/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:Ecc handle another copyright format

[Bob Feng]

Reviewed-by:  Bob Feng  

-Original Message-
From: Fan, ZhijuX 
Sent: Friday, September 6, 2019 11:30 AM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:Ecc handle another copyright format

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2057

Ecc can not handle the copyright format like

(C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP

This will cause Ecc to report wrong information.

This patch is going to handle this format

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/Ecc/c.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/Ecc/c.py b/BaseTools/Source/Python/Ecc/c.py
index a99b40a701..a30122a45f 100644
--- a/BaseTools/Source/Python/Ecc/c.py
+++ b/BaseTools/Source/Python/Ecc/c.py
@@ -2388,7 +2388,7 @@ def CheckFileHeaderDoxygenComments(FullFileName):
 PrintErrorMsg(ERROR_HEADER_CHECK_FILE, 'File header 
comment content should start with two spaces at each line', FileTable, ID)
 
 CommentLine = CommentLine.strip()
-if CommentLine.startswith('Copyright'):
+if CommentLine.startswith('Copyright') or ('Copyright' in 
CommentLine and CommentLine.lower().startswith('(c)')):
 NoCopyrightFlag = False
 if CommentLine.find('All rights reserved') == -1:
 for Copyright in EccGlobalData.gConfig.Copyright:
-- 
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47018): https://edk2.groups.io/g/devel/message/47018
Mute This Topic: https://groups.io/mt/33161998/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:ECC need to handle lower case 'static'

[Bob Feng]

Reviewed-by: Bob Feng  

-Original Message-
From: Fan, ZhijuX 
Sent: Wednesday, September 4, 2019 5:54 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:ECC need to handle lower case 'static'

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1778

We are going to add keyword (lower case) 'static' for functions and global 
variables which are not referenced outside their current C file.

However, the ECC tool only recognizes upper case 'STATIC' at this moment.
This will lead to issue reports for new codes that follow the above coding 
style.

This patch is going to handle lower case 'static'

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/Ecc/Check.py | 2 +-
 BaseTools/Source/Python/Ecc/Configuration.py | 2 +-  
BaseTools/Source/Python/Ecc/EccToolError.py  | 2 +-
 BaseTools/Source/Python/Ecc/c.py | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/Ecc/Check.py 
b/BaseTools/Source/Python/Ecc/Check.py
index 86bb8562ba..2180818609 100644
--- a/BaseTools/Source/Python/Ecc/Check.py
+++ b/BaseTools/Source/Python/Ecc/Check.py
@@ -441,7 +441,7 @@ class Check(object):
 self.DeclCheckUnionType()
 
 
-# Check whether no use of int, unsigned, char, void, static, long in any 
.c, .h or .asl files.
+# Check whether no use of int, unsigned, char, void, long in any .c, .h or 
.asl files.
 def DeclCheckNoUseCType(self):
 if EccGlobalData.gConfig.DeclarationDataTypeCheckNoUseCType == '1' or 
EccGlobalData.gConfig.DeclarationDataTypeCheckAll == '1' or 
EccGlobalData.gConfig.CheckAll == '1':
 EdkLogger.quiet("Checking Declaration No use C type ...") diff 
--git a/BaseTools/Source/Python/Ecc/Configuration.py 
b/BaseTools/Source/Python/Ecc/Configuration.py
index 57ae932ed2..9d9feaca5e 100644
--- a/BaseTools/Source/Python/Ecc/Configuration.py
+++ b/BaseTools/Source/Python/Ecc/Configuration.py
@@ -256,7 +256,7 @@ class Configuration(object):
 ## Declarations and Data Types Checking
 self.DeclarationDataTypeCheckAll = 0
 
-# Check whether no use of int, unsigned, char, void, static, long in 
any .c, .h or .asl files.
+# Check whether no use of int, unsigned, char, void, long in any .c, 
.h or .asl files.
 self.DeclarationDataTypeCheckNoUseCType = 1
 # Check whether the modifiers IN, OUT, OPTIONAL, and UNALIGNED are 
used only to qualify arguments to a function and should not appear in a data 
type declaration
 self.DeclarationDataTypeCheckInOutModifier = 1 diff --git 
a/BaseTools/Source/Python/Ecc/EccToolError.py 
b/BaseTools/Source/Python/Ecc/EccToolError.py
index 4b1bef6a3e..0ff3b42674 100644
--- a/BaseTools/Source/Python/Ecc/EccToolError.py
+++ b/BaseTools/Source/Python/Ecc/EccToolError.py
@@ -149,7 +149,7 @@ gEccErrorMessage = {
 ERROR_INCLUDE_FILE_CHECK_NAME : "No permission for the include file with 
same names",
 
 ERROR_DECLARATION_DATA_TYPE_CHECK_ALL : "",
-ERROR_DECLARATION_DATA_TYPE_CHECK_NO_USE_C_TYPE : "There should be no use 
of int, unsigned, char, void, static, long in any .c, .h or .asl files",
+ERROR_DECLARATION_DATA_TYPE_CHECK_NO_USE_C_TYPE : "There should be 
+ no use of int, unsigned, char, void, long in any .c, .h or .asl 
+ files",
 ERROR_DECLARATION_DATA_TYPE_CHECK_IN_OUT_MODIFIER : """The modifiers IN, 
OUT, OPTIONAL, and UNALIGNED should be used only to qualify arguments to a 
function and should not appear in a data type declaration""",
 ERROR_DECLARATION_DATA_TYPE_CHECK_EFI_API_MODIFIER : "The EFIAPI modifier 
should be used at the entry of drivers, events, and member functions of 
protocols",
 ERROR_DECLARATION_DATA_TYPE_CHECK_ENUMERATED_TYPE : "Enumerated Type 
should have a 'typedef' and the name must be in capital letters", diff --git 
a/BaseTools/Source/Python/Ecc/c.py b/BaseTools/Source/Python/Ecc/c.py
index 2a73da7cb8..a99b40a701 100644
--- a/BaseTools/Source/Python/Ecc/c.py
+++ b/BaseTools/Source/Python/Ecc/c.py
@@ -1859,7 +1859,7 @@ def CheckDeclNoUseCType(FullFileName):
where Model = %d
""" % (FileTable, DataClass.MODEL_IDENTIFIER_VARIABLE)
 ResultSet = Db.TblFile.Exec(SqlStatement)
-CTypeTuple = ('int', 'unsigned', 'char', 'void', 'static', 'long')
+CTypeTuple = ('int', 'unsigned', 'char', 'void', 'long')
 for Result in ResultSet:
 for Type in CTypeTuple:
 if PatternInModifier(Result[0], Type):
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47019): https://edk2.groups.io/g/devel/message/47019
Mute This Topic: https://groups.io/mt/33136540/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/3] BaseTools: Fixed the build fail on Linux with --genfds-multi-thread

[Bob Feng]

For the case that the Non-Hii Driver does not generate .offset file for uni 
string offset.

-Original Message-
From: Gao, Liming 
Sent: Monday, September 9, 2019 3:46 PM
To: devel@edk2.groups.io; Feng, Bob C 
Subject: RE: [edk2-devel] [Patch 1/3] BaseTools: Fixed the build fail on Linux 
with --genfds-multi-thread

Bob:
  What case will call GenSec without the exist file? 

Thanks
Liming
>-Original Message-
>From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of 
>Bob Feng
>Sent: Friday, September 06, 2019 9:23 PM
>To: devel@edk2.groups.io
>Cc: Gao, Liming ; Feng, Bob C 
>
>Subject: [edk2-devel] [Patch 1/3] BaseTools: Fixed the build fail on 
>Linux with - -genfds-multi-thread
>
>BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1302
>
>If GenSec input file not exist, the related command will fail and make 
>will stop on Linux. GenSec input file is allow to be non-existent.
>
>This patch is to let "make" continue if gensec input file not exist.
>
>Cc: Liming Gao 
>Signed-off-by: Bob Feng 
>---
> BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
>diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>index 037828ea1cca..0f691ae52420 100644
>--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>@@ -487,11 +487,11 @@ class GenFdsGlobalVariable:
> SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
> if IsMakefile:
> if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
> Cmd = ['if', 'exist', Input[0]] + Cmd
> else:
>-Cmd = ['test', '-e', Input[0], "&&"] + Cmd
>+Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
> if ' '.join(Cmd).strip() not in 
> GenFdsGlobalVariable.SecCmdList:
> GenFdsGlobalVariable.SecCmdList.append(' 
> '.join(Cmd).strip())
> elif GenFdsGlobalVariable.NeedsUpdate(Output, list(Input) 
>+
>[CommandFile]):
> GenFdsGlobalVariable.DebugLogger(EdkLogger.DEBUG_5, 
>"%s needs update because of newer %s" % (Output, Input))
> GenFdsGlobalVariable.CallExternalTool(Cmd, "Failed to 
>generate
>section")
>--
>2.20.1.windows.1
>
>
>


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47029): https://edk2.groups.io/g/devel/message/47029
Mute This Topic: https://groups.io/mt/33166299/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] commit "BaseTools: Improve the cache hit in the edk2 build cache" causes build.py unknown fatal error

[Bob Feng]

Pushed at 53e2eaba7a19cda3183b1bb2a2619a2494538e52

Thanks,
Bob
From: Vladimir Olovyannikov [mailto:vladimir.olovyanni...@broadcom.com]
Sent: Tuesday, September 10, 2019 4:15 AM
To: Feng, Bob C ; Shi, Steven ; 
devel@edk2.groups.io
Cc: Gao, Liming ; Rodriguez, Christian 
; Johnson, Michael 
Subject: RE: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Bob, Steven,

Thank you for update.
Yes, it fails with “build clean”.
Example:
. ./edksetup.sh
build -a AARCH64 -p BroadcomPlatformPkg/StingrayPkg/StingrayPkg.dsc -t GCC5 -b 
RELEASE -D PLAT=Stingray -D TARGET=RELEASE -D EDK2_STAMP="dev/bcm958742k 
59b754c9f697" cleanall

Bob, with your patch it builds fine.
Steven, in case I am missing something, how do you enable the build cache?

Thank you,
Vladimir
From: Feng, Bob C [mailto:bob.c.f...@intel.com]
Sent: Sunday, September 8, 2019 6:23 PM
To: Shi, Steven mailto:steven@intel.com>>; Vladimir 
Olovyannikov 
mailto:vladimir.olovyanni...@broadcom.com>>;
 devel@edk2.groups.io
Cc: Gao, Liming mailto:liming@intel.com>>; Rodriguez, 
Christian 
mailto:christian.rodrig...@intel.com>>; Johnson, 
Michael mailto:michael.john...@intel.com>>
Subject: RE: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Vladimir and Steven,

I checked code and think when the build target are 'clean', 'cleanlib', 
'cleanall', 'run' and 'fds', build will fail with the error like you mentioned.
I filed a Bugzilla https://bugzilla.tianocore.org/show_bug.cgi?id=2166
And provide a fix in https://edk2.groups.io/g/devel/message/47014

Thanks,
Bob

From: Shi, Steven
Sent: Saturday, September 7, 2019 10:16 AM
To: Vladimir Olovyannikov 
mailto:vladimir.olovyanni...@broadcom.com>>;
 devel@edk2.groups.io
Cc: Gao, Liming mailto:liming@intel.com>>; Feng, Bob 
C mailto:bob.c.f...@intel.com>>; Rodriguez, Christian 
mailto:christian.rodrig...@intel.com>>; Johnson, 
Michael mailto:michael.john...@intel.com>>
Subject: RE: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Vladimir,
Did you enable the build cache? Could you let me know your build steps and 
commands?


Thanks
Steven

From: Vladimir Olovyannikov [mailto:vladimir.olovyanni...@broadcom.com]
Sent: Saturday, September 7, 2019 4:45 AM
To: devel@edk2.groups.io; Shi, Steven 
mailto:steven@intel.com>>
Cc: Gao, Liming mailto:liming@intel.com>>; Feng, Bob 
C mailto:bob.c.f...@intel.com>>; Rodriguez, Christian 
mailto:christian.rodrig...@intel.com>>; Johnson, 
Michael mailto:michael.john...@intel.com>>
Subject: commit "BaseTools: Improve the cache hit in the edk2 build cache" 
causes build.py unknown fatal error

Hi Steven,

The commit
0e7e7a264cd80ab71ea0f9e9da2d0617d4b539c4
BaseTools: Improve the cache hit in the edk2 build cache
Author: Shi, Steven mailto:steven@intel.com>>  
2019-08-15 07:26:17
Committer: Feng, Bob C mailto:bob.c.f...@intel.com>>  
2019-08-19 23:06:05
Parent: 811328172841ce3f7ec1fe7033c130726ad189b7 
(ShellPkg/UefiShellDriver1CommandsLib: Make array big enough)
Child:  56c786b0a9fd4f08b1f0bc10b7718a7577cb0146 (BaseTools: Print first cache 
missing file for build cachle)
Branches: master, remotes/origin/master
Follows: edk2-stable201905
Precedes: edk2-stable201908

causes "unknown fatal error" in build.py when building our platform.

Here is the log:
...
build.py...
 : error C0DE: Unknown fatal error when processing 
[/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/ArmPkg/Library/ArmGicArchSecLib/ArmGicArchSecLib.inf]

(Please send email to devel@edk2.groups.io for 
help, attaching following call stack trace!)

(Python 3.6.8 on linux) Traceback (most recent call last):
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 2493, in Main
MyBuild.Launch()
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 2287, in Launch
self._BuildPlatform()
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 1742, in _BuildPlatform
self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/BinWrappers/PosixLike/../../Source/Python/build/build.py",
 line 1236, in _BuildPa
AutoGenObject.CreateCodeFile(True)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/Common/caching.py",
 line 18, in CallMeHere
Value = self._function(obj, *args,**kwargs)
  File 
"/home/volovyan/otherwork/sandbox/uefi-related/uefi-orig/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py",
 l

[edk2-devel] [Patch 2/3] BaseTools: remove unnecessary calls of os.exist

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2101

This patch is going to remove unnecessary calls
of os.exist()

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 .../Source/Python/AutoGen/AutoGenWorker.py|  7 +-
 BaseTools/Source/Python/AutoGen/GenC.py   | 97 ++-
 BaseTools/Source/Python/AutoGen/GenMake.py|  6 +-
 .../Source/Python/AutoGen/PlatformAutoGen.py  |  8 +-
 .../Source/Python/AutoGen/UniClassObject.py   |  5 +-
 5 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py 
b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index f488ae9d5f80..79069d603eb9 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -153,14 +153,15 @@ class AutoGenWorkerInProcess(mp.Process):
 return self.PlatformMetaFileSet[(filepath,root)]
 def run(self):
 try:
 taskname = "Init"
 with self.file_lock:
-if not os.path.exists(self.data_pipe_file_path):
+try:
+self.data_pipe = MemoryDataPipe()
+self.data_pipe.load(self.data_pipe_file_path)
+except:
 self.feedback_q.put(taskname + ":" + "load data pipe %s 
failed." % self.data_pipe_file_path)
-self.data_pipe = MemoryDataPipe()
-self.data_pipe.load(self.data_pipe_file_path)
 EdkLogger.LogClientInitialize(self.log_q)
 loglevel = self.data_pipe.Get("LogLevel")
 if not loglevel:
 loglevel = EdkLogger.INFO
 EdkLogger.SetLevel(loglevel)
diff --git a/BaseTools/Source/Python/AutoGen/GenC.py 
b/BaseTools/Source/Python/AutoGen/GenC.py
index 910c8fe3706c..5e0d11e165a3 100755
--- a/BaseTools/Source/Python/AutoGen/GenC.py
+++ b/BaseTools/Source/Python/AutoGen/GenC.py
@@ -1746,63 +1746,64 @@ def CreateIdfFileCode(Info, AutoGenC, StringH, 
IdfGenCFlag, IdfGenBinBuffer):
 EdkLogger.error("build", AUTOGEN_ERROR, "The %s in 
%s is not defined in the driver's [Sources] section" % (FileObj.FileName, Idf), 
ExtraData="[%s]" % str(Info))
 
 for FileObj in ImageFiles.ImageFilesDict[Idf]:
 ID = FileObj.ImageID
 File = FileObj.File
-if not os.path.exists(File.Path) or not 
os.path.isfile(File.Path):
-EdkLogger.error("build", FILE_NOT_FOUND, 
ExtraData=File.Path)
-SearchImageID (FileObj, FileList)
-if FileObj.Referenced:
-if (ValueStartPtr - len(DEFINE_STR + ID)) <= 0:
-Line = DEFINE_STR + ' ' + ID + ' ' + 
DecToHexStr(Index, 4) + '\n'
-else:
-Line = DEFINE_STR + ' ' + ID + ' ' * 
(ValueStartPtr - len(DEFINE_STR + ID)) + DecToHexStr(Index, 4) + '\n'
+try:
+SearchImageID (FileObj, FileList)
+if FileObj.Referenced:
+if (ValueStartPtr - len(DEFINE_STR + ID)) <= 0:
+Line = DEFINE_STR + ' ' + ID + ' ' + 
DecToHexStr(Index, 4) + '\n'
+else:
+Line = DEFINE_STR + ' ' + ID + ' ' * 
(ValueStartPtr - len(DEFINE_STR + ID)) + DecToHexStr(Index, 4) + '\n'
 
-if File not in FileDict:
-FileDict[File] = Index
-else:
-DuplicateBlock = pack('B', 
EFI_HII_IIBT_DUPLICATE)
-DuplicateBlock += pack('H', FileDict[File])
-ImageBuffer += DuplicateBlock
+if File not in FileDict:
+FileDict[File] = Index
+else:
+DuplicateBlock = pack('B', 
EFI_HII_IIBT_DUPLICATE)
+DuplicateBlock += pack('H', FileDict[File])
+ImageBuffer += DuplicateBlock
+BufferStr = WriteLine(BufferStr, '// %s: 
%s: %s' % (DecToHexStr(Index, 4), ID, DecToHexStr(Index, 4)))
+TempBufferList = 
AscToHexList(DuplicateBlock)
+BufferStr = WriteLine(BufferStr, 
CreateArrayItem(TempBufferList, 16) + '\n')
+StringH.Append(Line)
+  

[edk2-devel] [Patch 1/3] BaseTools: Improve GetDependencyList function

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2102

GetDependencyList get the header file via
re.findall in the whole header file.

This patch is to pre-process the header file and
to feed the shorter string to re.findall.

This patch is to improve GetDependencyList() efficiency

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 29 --
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 47dae82e1aeb..5d02d9a05694 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1690,26 +1690,29 @@ def GetDependencyList(AutoGenObject, FileCache, File, 
ForceList, SearchPathList)
 CurrentFileDependencyList = []
 if F in DepDb:
 CurrentFileDependencyList = DepDb[F]
 else:
 try:
-Fd = open(F.Path, 'rb')
-FileContent = Fd.read()
-Fd.close()
+with open(F.Path, 'rb') as Fd:
+FileContent = Fd.read(1)
+Fd.seek(0)
+if not FileContent:
+continue
+if FileContent[0] == 0xff or FileContent[0] == 0xfe:
+FileContent2 = Fd.read()
+FileContent2 = FileContent2.decode('utf-16')
+IncludedFileList = 
gIncludePattern.findall(FileContent2)
+else:
+FileLines = Fd.readlines()
+FileContent2 = [line for line in FileLines if 
str(line).lstrip("#\t ")[:8] == "include "]
+simpleFileContent="".join(FileContent2)
+
+IncludedFileList = 
gIncludePattern.findall(simpleFileContent)
 except BaseException as X:
 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F.Path + 
"\n\t" + str(X))
-if len(FileContent) == 0:
+if not FileContent:
 continue
-try:
-if FileContent[0] == 0xff or FileContent[0] == 0xfe:
-FileContent = FileContent.decode('utf-16')
-else:
-FileContent = FileContent.decode()
-except:
-# The file is not txt file. for example .mcb file
-continue
-IncludedFileList = gIncludePattern.findall(FileContent)
 
 for Inc in IncludedFileList:
 Inc = Inc.strip()
 # if there's macro used to reference header file, expand it
 HeaderList = gMacroPattern.findall(Inc)
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47070): https://edk2.groups.io/g/devel/message/47070
Mute This Topic: https://groups.io/mt/34090062/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 3/3] BaseTools:Remove the unnecessary operation of renaming a file.

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2183

This patch is going to remove rename call to reduce unnecessary
io operation so that saving build time.

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/Common/Misc.py | 23 +++
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py 
b/BaseTools/Source/Python/Common/Misc.py
index 15ae6a9e408f..714eb840ea66 100755
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -490,17 +490,13 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True, 
FileLock=None):
 if FileLock:
 FileLock.acquire()
 
 
 if GlobalData.gIsWindows and not os.path.exists(File):
-# write temp file, then rename the temp file to the real file
-# to make sure the file be immediate saved to disk
-with tempfile.NamedTemporaryFile(OpenMode, dir=os.path.dirname(File), 
delete=False) as tf:
-tf.write(Content)
-tempname = tf.name
 try:
-os.rename(tempname, File)
+with open(File, OpenMode) as tf:
+tf.write(Content)
 except IOError as X:
 if GlobalData.gBinCacheSource:
 EdkLogger.quiet("[cache error]:fails to save file with error: 
%s" % (X))
 else:
 EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError 
%s' % X)
@@ -564,25 +560,12 @@ def CopyFileOnChange(SrcFile, Dst, FileLock=None):
 if not FileLock:
 FileLock = GlobalData.file_lock
 if FileLock:
 FileLock.acquire()
 
-# os.replace and os.rename are the atomic operations in python 3 and 2.
-# we use these two atomic operations to ensure the file copy is atomic:
-# copy the src to a temp file in the dst same folder firstly, then
-# replace or rename the temp file to the destination file.
-with tempfile.NamedTemporaryFile(dir=DirName, delete=False) as tf:
-CopyLong(SrcFile, tf.name)
-tempname = tf.name
 try:
-if hasattr(os, 'replace'):
-os.replace(tempname, DstFile)
-else:
-# os.rename reqire to remove the dst on Windows, otherwise OSError 
will be raised.
-if GlobalData.gIsWindows and os.path.exists(DstFile):
-os.remove(DstFile)
-os.rename(tempname, DstFile)
+CopyLong(SrcFile, DstFile)
 except IOError as X:
 if GlobalData.gBinCacheSource:
 EdkLogger.quiet("[cache error]:fails to copy file with error: %s" 
% (X))
 else:
 EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % 
X)
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47072): https://edk2.groups.io/g/devel/message/47072
Mute This Topic: https://groups.io/mt/34090067/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 0/3] Clean BaseTools code for build performance

[Bob Feng]

REF:
1 https://bugzilla.tianocore.org/show_bug.cgi?id=2102
2 https://bugzilla.tianocore.org/show_bug.cgi?id=2101
3 https://bugzilla.tianocore.org/show_bug.cgi?id=2183

By profiling build Ovmf with cProfile:

1. re.findall time reduced:
after patch:
0.017  {method 'findall' of '_sre.SRE_Pattern' objects}
original:
0.212  {method 'findall' of '_sre.SRE_Pattern' objects}

2. times of call of os.stat is reduced
after patch:
33307{built-in method nt.stat}
orignial:
33774{built-in method nt.stat}

3. rename call is removed.
original:
2.165 {built-in method nt.rename}

Feng, Bob C (3):
  BaseTools: Improve GetDependencyList function
  BaseTools: remove unnecessary calls of os.exist
  BaseTools:Remove the unnecessary operation of renaming a file.

 .../Source/Python/AutoGen/AutoGenWorker.py|  7 +-
 BaseTools/Source/Python/AutoGen/GenC.py   | 97 ++-
 BaseTools/Source/Python/AutoGen/GenMake.py| 35 ---
 .../Source/Python/AutoGen/PlatformAutoGen.py  |  8 +-
 .../Source/Python/AutoGen/UniClassObject.py   |  5 +-
 BaseTools/Source/Python/Common/Misc.py| 23 +
 6 files changed, 82 insertions(+), 93 deletions(-)

-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47073): https://edk2.groups.io/g/devel/message/47073
Mute This Topic: https://groups.io/mt/34090163/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fixed a bug of IgnoreAutoGen

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2080
After checking that if the build can't
ignore Autogen due to there is no compelet autogen files, 
the build tool need to do a completely Autogen.

This patch is to fix a bug that if AutoGen
can't be skiped, the SkipAutoGen flag
need to set to False

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/build/build.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index 13be6c33ecd5..a034664dc388 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -2202,10 +2202,11 @@ class Build():
 ExitFlag = threading.Event()
 ExitFlag.clear()
 if self.SkipAutoGen:
 Wa = self.VerifyAutoGenFiles()
 if Wa is None:
+self.SkipAutoGen = False
 Wa, self.BuildModules = 
self.PerformAutoGen(BuildTarget,ToolChain)
 else:
 GlobalData.gAutoGenPhase = True
 self.BuildModules = self.SetupMakeSetting(Wa)
 else:
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47119): https://edk2.groups.io/g/devel/message/47119
Mute This Topic: https://groups.io/mt/34101745/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:Replace PlatformInfo with PlatformAutoGen for Moudle

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Thursday, September 12, 2019 4:19 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:Replace PlatformInfo with PlatformAutoGen for Moudle

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2188

build -p MdeModulePkg\MdeModulePkg.dsc -a IA32 -m 
MdeModulePkg\Universal\PCD\Pei\Pcd.inf

Error:
AttributeError: 'PlatformInfo' object has no attribute 'DynamicPcdList'

The DSC data object used to build a separate module today is PlatformInfo 
rather than PlatformAutoGen 'PlatformAutoGen' object has attribute 
'DynamicPcdList'

This patch is going to fixed this issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/build/build.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index e81d3d6486..e845c139c9 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1860,6 +1860,9 @@ class Build():
 Ma = ModuleAutoGen(Wa, Module, BuildTarget, 
ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
 if Ma is None:
 continue
+if Ma.PcdIsDriver:
+Ma.PlatformInfo = Pa
+Ma.Workspace = Wa
 MaList.append(Ma)
 
 if GlobalData.gBinCacheSource and self.Target in 
[None, "", "all"]:
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47244): https://edk2.groups.io/g/devel/message/47244
Mute This Topic: https://groups.io/mt/34112948/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:change some incorrect parameter defaults

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Thursday, September 12, 2019 4:18 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:change some incorrect parameter defaults

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1858

for Dict={},There are pitfalls in the way this default parameter is set and 
Dict is not used in functions, other functions have these two cases, I will 
change some incorrect parameter defaults

This patch is going to fix this issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/AutoGen/BuildEngine.py |  4 +++-
 BaseTools/Source/Python/AutoGen/GenMake.py |  6 --
 BaseTools/Source/Python/Common/RangeExpression.py  |  4 +++-
 BaseTools/Source/Python/Common/StringUtils.py  | 10 +++---
 BaseTools/Source/Python/GenFds/AprioriSection.py   |  4 +++-
 BaseTools/Source/Python/GenFds/CompressSection.py  |  4 +++-
 BaseTools/Source/Python/GenFds/DataSection.py  |  4 +++-
 BaseTools/Source/Python/GenFds/EfiSection.py   |  4 +++-
 BaseTools/Source/Python/GenFds/FfsFileStatement.py |  5 -
 BaseTools/Source/Python/GenFds/FfsInfStatement.py  |  5 +++--
 BaseTools/Source/Python/GenFds/Fv.py   |  4 +++-
 BaseTools/Source/Python/GenFds/FvImageSection.py   |  4 +++-
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py |  2 +-
 BaseTools/Source/Python/GenFds/GuidSection.py  |  4 +++-
 BaseTools/Source/Python/GenFds/OptRomFileStatement.py  |  5 -
 BaseTools/Source/Python/GenFds/Region.py   |  4 +++-
 BaseTools/Source/Python/GenFds/Section.py  |  2 +-
 BaseTools/Source/Python/GenFds/UiSection.py|  4 +++-
 BaseTools/Source/Python/GenFds/VerSection.py   |  4 +++-
 19 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py 
b/BaseTools/Source/Python/AutoGen/BuildEngine.py
index bb91534477..069a3a1c9d 100644
--- a/BaseTools/Source/Python/AutoGen/BuildEngine.py
+++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py
@@ -176,7 +176,9 @@ class FileBuildRule:
 CommandString = "\n\t".join(self.CommandList)
 return "%s : %s\n\t%s" % (DestString, SourceString, CommandString)
 
-def Instantiate(self, Macros={}):
+def Instantiate(self, Macros = None):
+if Macros is None:
+Macros = {}
 NewRuleObject = copy.copy(self)
 NewRuleObject.BuildTargets = {}
 NewRuleObject.DestFileList = [] diff --git 
a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 4f85a93055..3185ebe368 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -205,10 +205,12 @@ class BuildFile(object):
 def GetRemoveDirectoryCommand(self, DirList):
 return [self._RD_TEMPLATE_[self._FileType] % {'dir':Dir} for Dir in 
DirList]
 
-def PlaceMacro(self, Path, MacroDefinitions={}):
+def PlaceMacro(self, Path, MacroDefinitions=None):
 if Path.startswith("$("):
 return Path
 else:
+if MacroDefinitions is None:
+MacroDefinitions = {}
 PathLength = len(Path)
 for MacroName in MacroDefinitions:
 MacroValue = MacroDefinitions[MacroName] @@ -1762,4 +1764,4 @@ 
def GetDependencyList(AutoGenObject, FileCache, File, ForceList, SearchPathList)
 
 # This acts like the main() function for the script, unless it is 'import'ed 
into another script.
 if __name__ == '__main__':
-pass
\ No newline at end of file
+pass
diff --git a/BaseTools/Source/Python/Common/RangeExpression.py 
b/BaseTools/Source/Python/Common/RangeExpression.py
index e9296e03f6..039d281467 100644
--- a/BaseTools/Source/Python/Common/RangeExpression.py
+++ b/BaseTools/Source/Python/Common/RangeExpression.py
@@ -342,7 +342,9 @@ class RangeExpression(BaseExpression):
 raise BadExpression(ERR_STRING_EXPR % Operator)
 
 
-def __init__(self, Expression, PcdDataType, SymbolTable = {}):
+def __init__(self, Expression, PcdDataType, SymbolTable = None):
+if SymbolTable is None:
+SymbolTable = {}
 super(RangeExpression, self).__init__(self, Expression, PcdDataType, 
SymbolTable)
 self._NoProcess = False
 if not isinstance(Expression, type('')):
diff --git a/BaseTools/Source/Python/Common/StringUtils.py 
b/BaseTools/Source/Python/Common/StringUtils.py
index 0febbc0034..73dafa797a 100644
--- a/BaseTools/Source/Python/Common/StringUtils.py
+++ b/BaseTools/Source/Python/Common/StringUtils.py
@@ -243,8 +243,10 @@ def SplitModuleType(Key):
 #
 # @retval NewList   A new string list whose macros are replaced
 #
-def ReplaceMacros(StringList, MacroDefinitions={}, SelfReplacement=False):
+def 

Re: [edk2-devel] [PATCH v2 1/1] BaseTools: Add more parameter checking for CopyFileOnChange()

[Bob Feng]

Reviewed-by: Bob Feng  

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Steven Shi
Sent: Wednesday, September 18, 2019 11:21 AM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [edk2-devel] [PATCH v2 1/1] BaseTools: Add more parameter checking for 
CopyFileOnChange()

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2193

The current CopyFileOnChange() method in Misc.py does not accept the input 
SrcFile parameter as a dir, but the method does not check the SrcFile is dir or 
not. This patch is to add more input parameter type checking and error message 
output for method CopyFileOnChange.

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Steven Shi 
---
 BaseTools/Source/Python/Common/Misc.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/Common/Misc.py 
b/BaseTools/Source/Python/Common/Misc.py
index 714eb840ea..a488536cb4 100755
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -536,7 +536,8 @@ def CopyFileOnChange(SrcFile, Dst, FileLock=None):
 SrcFile = LongFilePath(SrcFile)
 Dst = LongFilePath(Dst)
 
-if not os.path.exists(SrcFile):
+if os.path.isdir(SrcFile):
+EdkLogger.error(None, FILE_COPY_FAILURE, 
+ ExtraData='CopyFileOnChange SrcFile is a dir, not a file: %s' % 
+ SrcFile)
 return False
 
 if os.path.isdir(Dst):
--
2.17.1.windows.2





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47464): https://edk2.groups.io/g/devel/message/47464
Mute This Topic: https://groups.io/mt/34184105/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH 5/9] BaseTools: fix line endings in SetupGit.py Conf files

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Leif 
Lindholm
Sent: Thursday, September 19, 2019 6:43 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming 
Subject: [edk2-devel] [PATCH 5/9] BaseTools: fix line endings in SetupGit.py 
Conf files

A file header license/copyright header copied around in commit
5b3e695d8ac5 ("BaseTools: add centralized location for git config files") was 
missing a CR - add it in both faulty locations.

Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Leif Lindholm 
---
 BaseTools/Conf/diff.order| 2 +-
 BaseTools/Conf/gitattributes | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Conf/diff.order b/BaseTools/Conf/diff.order index 
68b926ca94aa..4361817012c9 100644
--- a/BaseTools/Conf/diff.order
+++ b/BaseTools/Conf/diff.order
@@ -2,7 +2,7 @@
 # Copyright (c) 2019, Linaro Ltd. All rights reserved.
 #
 # SPDX-License-Identifier: BSD-2-Clause-Patent -#
+#
 *.dec
 *.dsc.inc
 *.dsc
diff --git a/BaseTools/Conf/gitattributes b/BaseTools/Conf/gitattributes index 
9b6ab81a475b..58b93e9d4c27 100644
--- a/BaseTools/Conf/gitattributes
+++ b/BaseTools/Conf/gitattributes
@@ -2,7 +2,7 @@
 # Copyright (c) 2019, Linaro Ltd. All rights reserved.
 #
 # SPDX-License-Identifier: BSD-2-Clause-Patent -#
+#
 *.efi -diff
 *.EFI -diff
 *.bin -diff
--
2.20.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47528): https://edk2.groups.io/g/devel/message/47528
Mute This Topic: https://groups.io/mt/34193904/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH 1/9] BaseTools: add missing newlines at end of files

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Leif Lindholm [mailto:leif.lindh...@linaro.org] 
Sent: Thursday, September 19, 2019 6:43 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming 
Subject: [PATCH 1/9] BaseTools: add missing newlines at end of files

Some scripts in Source/Python were missing newlines at end of files, so add 
them.

Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Leif Lindholm 
---
 BaseTools/Source/Python/AutoGen/DataPipe.py  | 2 +-
 BaseTools/Source/Python/Common/DataType.py   | 2 +-
 BaseTools/Source/Python/Common/GlobalData.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py 
b/BaseTools/Source/Python/AutoGen/DataPipe.py
index 8b8cfd1c51eb..078bafecb45b 100755
--- a/BaseTools/Source/Python/AutoGen/DataPipe.py
+++ b/BaseTools/Source/Python/AutoGen/DataPipe.py
@@ -163,4 +163,4 @@ class MemoryDataPipe(DataPipe):
 
 self.DataContainer = {"BinCacheDest":GlobalData.gBinCacheDest}
 
-self.DataContainer = 
{"EnableGenfdsMultiThread":GlobalData.gEnableGenfdsMultiThread}
\ No newline at end of file
+self.DataContainer = 
+ {"EnableGenfdsMultiThread":GlobalData.gEnableGenfdsMultiThread}
diff --git a/BaseTools/Source/Python/Common/DataType.py 
b/BaseTools/Source/Python/Common/DataType.py
index 8ae1bd28fabf..5d49afb0a92a 100644
--- a/BaseTools/Source/Python/Common/DataType.py
+++ b/BaseTools/Source/Python/Common/DataType.py
@@ -532,4 +532,4 @@ PACK_CODE_BY_SIZE = {8:'=Q',
  0:'=B',
 16:""}
 
-TAB_COMPILER_MSFT = 'MSFT'
\ No newline at end of file
+TAB_COMPILER_MSFT = 'MSFT'
diff --git a/BaseTools/Source/Python/Common/GlobalData.py 
b/BaseTools/Source/Python/Common/GlobalData.py
index 8eb72aa1d6f5..74c6d0079bba 100755
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -129,4 +129,4 @@ file_lock = None
 # Common dictionary to share platform libraries' constant Pcd  libConstPcd = 
None  # Common dictionary to share platform libraries' reference info -Refes = 
None \ No newline at end of file
+Refes = None
--
2.20.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47526): https://edk2.groups.io/g/devel/message/47526
Mute This Topic: https://groups.io/mt/34193900/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH 8/9] BaseTools: correct line endings for ConvertFce Python script

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Leif 
Lindholm
Sent: Thursday, September 19, 2019 6:44 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming 
Subject: [edk2-devel] [PATCH 8/9] BaseTools: correct line endings for 
ConvertFce Python script

Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Leif Lindholm 
---
 BaseTools/Scripts/ConvertFceToStructurePcd.py | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/BaseTools/Scripts/ConvertFceToStructurePcd.py 
b/BaseTools/Scripts/ConvertFceToStructurePcd.py
index 10f19dbe2f3c..89e6a727a201 100644
--- a/BaseTools/Scripts/ConvertFceToStructurePcd.py
+++ b/BaseTools/Scripts/ConvertFceToStructurePcd.py
@@ -506,8 +506,8 @@ class mainprocess(object):
 for i in range(len(info_list)-1,-1,-1):
   if len(info_list[i]) == 0:
 info_list.remove(info_list[i])
-for i in (inf_list, title_all, header_list):
-  i.sort()
+for i in (inf_list, title_all, header_list):
+  i.sort()
 return keys,title_all,info_list,header_list,inf_list
 
   def remove_bracket(self,List):
@@ -519,9 +519,9 @@ class mainprocess(object):
   List[List.index(i)][i.index(j)] = "|".join(tmp)
 else:
   List[List.index(i)][i.index(j)] = j
-for i in List:
-  if type(i) == type([0,0]):
-i.sort()
+for i in List:
+  if type(i) == type([0,0]):
+i.sort()
 return List
 
   def write_all(self):
-- 
2.20.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47527): https://edk2.groups.io/g/devel/message/47527
Mute This Topic: https://groups.io/mt/34193908/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch] BaseTools: init DependencyHeaderFileSet for each MakeFile class

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2203

This patch is going to fix a regression issue that
build breaks when the custom makefile exist.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 940136248f..e447e43ec3 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -447,10 +447,11 @@ cleanlib:
 self.Macros["WORKSPACE"   ] = 
self._AutoGenObject.Macros["WORKSPACE"]
 self.Macros["FFS_OUTPUT_DIR"  ] = 
self._AutoGenObject.Macros["FFS_OUTPUT_DIR"]
 self.GenFfsList = ModuleAutoGen.GenFfsList
 self.MacroList = ['FFS_OUTPUT_DIR', 'MODULE_GUID', 'OUTPUT_DIR']
 self.FfsOutputFileList = []
+self.DependencyHeaderFileSet = set()
 
 # Compose a dict object containing information used to do replacement in 
template
 @property
 def _TemplateDict(self):
 if self._FileType not in self._SEP_:
@@ -908,11 +909,11 @@ cleanlib:
 SourceFileList,
 ForceIncludedFile,
 self._AutoGenObject.IncludePathList + 
self._AutoGenObject.BuildOptionIncPathList
 )
 
-self.DependencyHeaderFileSet = set()
+
 if FileDependencyDict:
 for Dependency in FileDependencyDict.values():
 self.DependencyHeaderFileSet.update(set(Dependency))
 
 # Get a set of unique package includes from MetaFile
@@ -1242,10 +1243,11 @@ ${BEGIN}\t-@${create_directory_command}\n${END}\
 #
 def __init__(self, ModuleAutoGen):
 BuildFile.__init__(self, ModuleAutoGen)
 self.PlatformInfo = self._AutoGenObject.PlatformInfo
 self.IntermediateDirectoryList = ["$(DEBUG_DIR)", "$(OUTPUT_DIR)"]
+self.DependencyHeaderFileSet = set()
 
 # Compose a dict object containing information used to do replacement in 
template
 @property
 def _TemplateDict(self):
 Separator = self._SEP_[self._FileType]
@@ -1432,10 +1434,11 @@ cleanlib:
 self.ModuleMakefileList = []
 self.IntermediateDirectoryList = []
 self.ModuleBuildDirectoryList = []
 self.LibraryBuildDirectoryList = []
 self.LibraryMakeCommandList = []
+self.DependencyHeaderFileSet = set()
 
 # Compose a dict object containing information used to do replacement in 
template
 @property
 def _TemplateDict(self):
 Separator = self._SEP_[self._FileType]
@@ -1537,10 +1540,11 @@ class TopLevelMakefile(BuildFile):
 #   @param  Workspace   Object of WorkspaceAutoGen class
 #
 def __init__(self, Workspace):
 BuildFile.__init__(self, Workspace)
 self.IntermediateDirectoryList = []
+self.DependencyHeaderFileSet = set()
 
 # Compose a dict object containing information used to do replacement in 
template
 @property
 def _TemplateDict(self):
 Separator = self._SEP_[self._FileType]
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47529): https://edk2.groups.io/g/devel/message/47529
Mute This Topic: https://groups.io/mt/34195983/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:Change the way that get some VpdPcd information

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Wednesday, September 18, 2019 5:43 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:Change the way that get some VpdPcd information

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2201

An error occurs using special VpdPcd that is not used in the Inf file

In dsc:
[PcdsDynamicExVpd.common.DEFAULT]
gBoardModuleTokenSpaceGuid.test1|*|{CODE({
  {0x0}  // terminator
})}

In dec:
[PcdsDynamicEx]
# Vpd GPIO table
  gBoardModuleTokenSpaceGuid.test1|{0}|GPIO_INIT_CONFIG[]|0x5018 {
  
Library/GpioLib.h
  
MdePkg/MdePkg.dec
}
ValueError: invalid literal for int() with base 0: '*'

This Patch is going to fix issue by Change the way that get some VpdPcd 
information

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/build/BuildReport.py | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Source/Python/build/BuildReport.py 
b/BaseTools/Source/Python/build/BuildReport.py
index 6b26f1c3b0..880459d367 100644
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -1419,10 +1419,19 @@ class PcdReport(object):
 FileWrite(File, '%*s' % (self.MaxLen + 4, 
SkuInfo.VpdOffset))
 VPDPcdItem = (Pcd.TokenSpaceGuidCName + '.' + 
PcdTokenCName, SkuIdName, SkuInfo.VpdOffset, Pcd.MaxDatumSize, 
SkuInfo.DefaultValue)
 if VPDPcdItem not in VPDPcdList:
-VPDPcdList.append(VPDPcdItem)
+PcdGuidList = 
self.UnusedPcds.get(Pcd.TokenSpaceGuidCName)
+if PcdGuidList:
+PcdList = PcdGuidList.get(Pcd.Type)
+if not PcdList:
+VPDPcdList.append(VPDPcdItem)
+for VpdPcd in PcdList:
+if PcdTokenCName == VpdPcd.TokenCName:
+break
+else:
+VPDPcdList.append(VPDPcdItem)
 if IsStructure:
 FiledOverrideFlag = False
-OverrideValues = Pcd.SkuOverrideValues[Sku]
+OverrideValues = Pcd.SkuOverrideValues.get(Sku)
 if OverrideValues:
 Keys = list(OverrideValues.keys())
 OverrideFieldStruct = self.OverrideFieldValue(Pcd, 
OverrideValues[Keys[0]])
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47567): https://edk2.groups.io/g/devel/message/47567
Mute This Topic: https://groups.io/mt/34186106/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:Change the way that get some VpdPcd information

[Bob Feng]

Zhiju,

Please update the patch title, "Change the way that get some VpdPcd 
information" is not clear for what this patch dose.

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Bob Feng
Sent: Thursday, September 19, 2019 4:27 PM
To: Fan, ZhijuX ; devel@edk2.groups.io
Cc: Gao, Liming 
Subject: Re: [edk2-devel] [PATCH] BaseTools:Change the way that get some VpdPcd 
information

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Wednesday, September 18, 2019 5:43 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:Change the way that get some VpdPcd information

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2201

An error occurs using special VpdPcd that is not used in the Inf file

In dsc:
[PcdsDynamicExVpd.common.DEFAULT]
gBoardModuleTokenSpaceGuid.test1|*|{CODE({
  {0x0}  // terminator
})}

In dec:
[PcdsDynamicEx]
# Vpd GPIO table
  gBoardModuleTokenSpaceGuid.test1|{0}|GPIO_INIT_CONFIG[]|0x5018 {
  
Library/GpioLib.h
  
MdePkg/MdePkg.dec
}
ValueError: invalid literal for int() with base 0: '*'

This Patch is going to fix issue by Change the way that get some VpdPcd 
information

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/build/BuildReport.py | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Source/Python/build/BuildReport.py 
b/BaseTools/Source/Python/build/BuildReport.py
index 6b26f1c3b0..880459d367 100644
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -1419,10 +1419,19 @@ class PcdReport(object):
 FileWrite(File, '%*s' % (self.MaxLen + 4, 
SkuInfo.VpdOffset))
 VPDPcdItem = (Pcd.TokenSpaceGuidCName + '.' + 
PcdTokenCName, SkuIdName, SkuInfo.VpdOffset, Pcd.MaxDatumSize, 
SkuInfo.DefaultValue)
 if VPDPcdItem not in VPDPcdList:
-VPDPcdList.append(VPDPcdItem)
+PcdGuidList = 
self.UnusedPcds.get(Pcd.TokenSpaceGuidCName)
+if PcdGuidList:
+PcdList = PcdGuidList.get(Pcd.Type)
+if not PcdList:
+VPDPcdList.append(VPDPcdItem)
+for VpdPcd in PcdList:
+if PcdTokenCName == VpdPcd.TokenCName:
+break
+else:
+VPDPcdList.append(VPDPcdItem)
 if IsStructure:
 FiledOverrideFlag = False
-OverrideValues = Pcd.SkuOverrideValues[Sku]
+OverrideValues = Pcd.SkuOverrideValues.get(Sku)
 if OverrideValues:
 Keys = list(OverrideValues.keys())
 OverrideFieldStruct = self.OverrideFieldValue(Pcd, 
OverrideValues[Keys[0]])
--
2.14.1.windows.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47569): https://edk2.groups.io/g/devel/message/47569
Mute This Topic: https://groups.io/mt/34186106/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH V2] BaseTools:Fix the issue that build report failed

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Thursday, September 19, 2019 5:04 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH V2] BaseTools:Fix the issue that build report failed

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2201

An error occurs using special VpdPcd that is not used in the Inf file

In dsc:
[PcdsDynamicExVpd.common.DEFAULT]
gBoardModuleTokenSpaceGuid.test1|*|{CODE({
  {0x0}  // terminator
})}

In dec:
[PcdsDynamicEx]
# Vpd GPIO table
  gBoardModuleTokenSpaceGuid.test1|{0}|GPIO_INIT_CONFIG[]|0x5018 {
  
Library/GpioLib.h
  
MdePkg/MdePkg.dec
}
ValueError: invalid literal for int() with base 0: '*'

This Patch is going to fix issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
Old title:Change the way that get some VpdPcd information 

 BaseTools/Source/Python/build/BuildReport.py | 13 +++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Source/Python/build/BuildReport.py 
b/BaseTools/Source/Python/build/BuildReport.py
index 6b26f1c3b0..880459d367 100644
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -1419,10 +1419,19 @@ class PcdReport(object):
 FileWrite(File, '%*s' % (self.MaxLen + 4, 
SkuInfo.VpdOffset))
 VPDPcdItem = (Pcd.TokenSpaceGuidCName + '.' + 
PcdTokenCName, SkuIdName, SkuInfo.VpdOffset, Pcd.MaxDatumSize, 
SkuInfo.DefaultValue)
 if VPDPcdItem not in VPDPcdList:
-VPDPcdList.append(VPDPcdItem)
+PcdGuidList = 
self.UnusedPcds.get(Pcd.TokenSpaceGuidCName)
+if PcdGuidList:
+PcdList = PcdGuidList.get(Pcd.Type)
+if not PcdList:
+VPDPcdList.append(VPDPcdItem)
+for VpdPcd in PcdList:
+if PcdTokenCName == VpdPcd.TokenCName:
+break
+else:
+VPDPcdList.append(VPDPcdItem)
 if IsStructure:
 FiledOverrideFlag = False
-OverrideValues = Pcd.SkuOverrideValues[Sku]
+OverrideValues = Pcd.SkuOverrideValues.get(Sku)
 if OverrideValues:
 Keys = list(OverrideValues.keys())
 OverrideFieldStruct = self.OverrideFieldValue(Pcd, 
OverrideValues[Keys[0]])
-- 
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47574): https://edk2.groups.io/g/devel/message/47574
Mute This Topic: https://groups.io/mt/34197653/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch] Revert "BaseTools: Improve GetDependencyList function"

[Bob Feng]

I agree. 

Reviewed-by: Bob Feng

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Liming Gao
Sent: Friday, September 20, 2019 2:00 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C 
Subject: [edk2-devel] [Patch] Revert "BaseTools: Improve GetDependencyList 
function"

This reverts commit bc9e4194cf3edaf9524c83098ba3f72008c70190.
This change causes the dependent header files are missing in Makefile.
It makes the incremental build not work. So, revert this change.

Cc: Bob Feng
Signed-off-by: Liming Gao 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 29 +
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 940136248f..2fe0e78bec 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1696,25 +1696,22 @@ def GetDependencyList(AutoGenObject, FileCache, File, 
ForceList, SearchPathList)
 CurrentFileDependencyList = DepDb[F]
 else:
 try:
-with open(F.Path, 'rb') as Fd:
-FileContent = Fd.read(1)
-Fd.seek(0)
-if not FileContent:
-continue
-if FileContent[0] == 0xff or FileContent[0] == 0xfe:
-FileContent2 = Fd.read()
-FileContent2 = FileContent2.decode('utf-16')
-IncludedFileList = 
gIncludePattern.findall(FileContent2)
-else:
-FileLines = Fd.readlines()
-FileContent2 = [line for line in FileLines if 
str(line).lstrip("#\t ")[:8] == "include "]
-simpleFileContent="".join(FileContent2)
-
-IncludedFileList = 
gIncludePattern.findall(simpleFileContent)
+Fd = open(F.Path, 'rb')
+FileContent = Fd.read()
+Fd.close()
 except BaseException as X:
 EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F.Path + 
"\n\t" + str(X))
-if not FileContent:
+if len(FileContent) == 0:
+continue
+try:
+if FileContent[0] == 0xff or FileContent[0] == 0xfe:
+FileContent = FileContent.decode('utf-16')
+else:
+FileContent = FileContent.decode()
+except:
+# The file is not txt file. for example .mcb file
 continue
+IncludedFileList = gIncludePattern.findall(FileContent)
 
 for Inc in IncludedFileList:
 Inc = Inc.strip()
-- 
2.13.0.windows.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47714): https://edk2.groups.io/g/devel/message/47714
Mute This Topic: https://groups.io/mt/34227614/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 05/10 V8] BaseTools: Enable Multiple Process AutoGen

[Bob Feng]

Hi Dann,

Thanks for raising this issue. 

Would you provide the static_library_files.list file, so that I can have a 
check?

As this patch was pushed month ago, I'd like to ask if this build failure 
always happens since this patch was pushed?

Thanks,
Bob 
-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of dann 
frazier
Sent: Tuesday, September 24, 2019 9:19 AM
To: devel@edk2.groups.io; Feng, Bob C 
Cc: Gao, Liming 
Subject: Re: [edk2-devel] [Patch 05/10 V8] BaseTools: Enable Multiple Process 
AutoGen

On Wed, Aug 07, 2019 at 12:25:32PM +0800, Bob Feng wrote:
> BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875
> 
> Assign the Module AutoGen tasks into multiple sub process.
> 
> Cc: Liming Gao 
> Signed-off-by: Bob Feng 
> ---
>  .../Source/Python/AutoGen/AutoGenWorker.py| 191 ++
>  BaseTools/Source/Python/AutoGen/DataPipe.py   |  17 +-
>  BaseTools/Source/Python/AutoGen/GenC.py   |   4 +-
>  .../Source/Python/AutoGen/ModuleAutoGen.py|   2 +-
>  .../Source/Python/AutoGen/PlatformAutoGen.py  |  16 +-
>  .../Source/Python/AutoGen/WorkspaceAutoGen.py |   6 +-
>  BaseTools/Source/Python/build/build.py| 125 
>  7 files changed, 306 insertions(+), 55 deletions(-)  create mode 
> 100644 BaseTools/Source/Python/AutoGen/AutoGenWorker.py

I've bisected a regression building the ArmVirt architecture of 
ArmVirtPkg/ArmVirtQemu.dsc to this commit.

$ build -a ARM -p ArmVirtPkg/ArmVirtQemu.dsc -t GCC49 [...] 
"arm-linux-gnueabihf-gcc" -o 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/DEBUG/LogoDxe.dll
 -Wl,--emit-relocs -nostdlib -Wl,--gc-sections -u _ModuleEntryPoint 
-Wl,-e,_ModuleEntryPoint,-Map,/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/DEBUG/LogoDxe.map
 -z common-page-size=0x20 -Wl,--pic-veneer -Wl,--oformat=elf32-littlearm -z 
common-page-size=0x1000 
-Wl,--start-group,@/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/OUTPUT/static_library_files.lst,--end-group
 -mthumb -march=armv7-a -g -Os -fshort-wchar -fno-builtin -fno-strict-aliasing 
-Wall -Werror -Wno-array-bounds -include AutoGen.h -fno-common -mlittle-endian 
-mabi=aapcs -fno-short-enums -funsigned-char -ffunction-sections 
-fdata-sections -fomit-frame-pointer -Wno-address -mthumb -mfloat-abi=soft 
-fno-pic -fno-pie -fstack-protector -mword-relocations -O0 
-DDISABLE_NEW_DEPRECATED_INTERFACES 
-Wl,--script=/home/dannf/git/edk2.debug/BaseTools/Scripts/GccBase.lds 
-Wl,--defsym=PECOFF_HEADER_SIZE=0x220
/usr/lib/gcc-cross/arm-linux-gnueabihf/9/../../../../arm-linux-gnueabihf/bin/ld:
 error: source object 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdePkg/Library/UefiDriverEntryPoint/UefiDriverEntryPoint/OUTPUT/UefiDriverEntryPoint.lib(DriverEntryPoint.obj)
 has EABI version 5, but target 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/DEBUG/LogoDxe.dll
 has EABI version 0
/usr/lib/gcc-cross/arm-linux-gnueabihf/9/../../../../arm-linux-gnueabihf/bin/ld:
 failed to merge target specific data of file 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdePkg/Library/UefiDriverEntryPoint/UefiDriverEntryPoint/OUTPUT/UefiDriverEntryPoint.lib(DriverEntryPoint.obj)
/usr/lib/gcc-cross/arm-linux-gnueabihf/9/../../../../arm-linux-gnueabihf/bin/ld:
 error: source object 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/OUTPUT/LogoDxe.lib(AutoGen.obj)
 has EABI version 5, but target 
/home/dannf/git/edk2.debug/Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/DEBUG/LogoDxe.dll
 has EABI version 0 [...]

It looks like this maybe a side-effect of the ordering of the entries in 
static_library_files.list. Previous to this commit, LogoDxehii.lib would appear 
near the end of the file - now it is near the beginning.
Moving it back to the end of the file allows the linking to continue.

Also, is it expected that this does not report being an EABI5 object?
$ file 
Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/OUTPUT/LogoDxehii.lib
Build/ArmVirtQemu-ARM/DEBUG_GCC49/ARM/MdeModulePkg/Logo/LogoDxe/OUTPUT/LogoDxehii.lib:
 ELF 32-bit LSB relocatable, ARM, version 1 (ARM), not stripped

 -dann




-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#47904): https://edk2.groups.io/g/devel/message/47904
Mute This Topic: https://groups.io/mt/32779332/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH 1/1] BaseTools: use stdint.h for GCC ProcessorBind.h typedefs

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Leif Lindholm [mailto:leif.lindh...@linaro.org] 
Sent: Friday, September 27, 2019 3:28 AM
To: devel@edk2.groups.io
Cc: Ard Biesheuvel ; Feng, Bob C 
; Gao, Liming ; Laszlo Ersek 

Subject: [PATCH 1/1] BaseTools: use stdint.h for GCC ProcessorBind.h typedefs

The AArch64 definitions of UINT64/INT64 differ from the X64 ones.
Since this is on the tool side, doing like X64 and picking the definitions from 
stdint.h feels like a better idea than hardcoding them. So copy the pattern 
from X64/ProcesorBind.h.

Cc: Ard Biesheuvel 
Cc: Bob Feng 
Cc: Liming Gao 
Cc: Laszlo Ersek 
Signed-off-by: Leif Lindholm 
---

This was triggered by one of the Risc-V patches which may need to end up being 
modified to the point where this issue goes away, but the current situation 
seems suboptimal. (Do you use %llx or %lx to print an Elf64_Addr on a 64-bit LP 
architecture?)

 BaseTools/Source/C/Include/AArch64/ProcessorBind.h | 26 ++--
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/BaseTools/Source/C/Include/AArch64/ProcessorBind.h 
b/BaseTools/Source/C/Include/AArch64/ProcessorBind.h
index bfaf1e28e446..dfa725b2e363 100644
--- a/BaseTools/Source/C/Include/AArch64/ProcessorBind.h
+++ b/BaseTools/Source/C/Include/AArch64/ProcessorBind.h
@@ -41,21 +41,21 @@
   typedef signed char INT8;
 #else
   //
-  // Assume standard AARCH64 alignment.
+  // Use ANSI C 2000 stdint.h integer width declarations
   //
-  typedef unsigned long long  UINT64;
-  typedef long long   INT64;
-  typedef unsigned intUINT32;
-  typedef int INT32;
-  typedef unsigned short  UINT16;
-  typedef unsigned short  CHAR16;
-  typedef short   INT16;
-  typedef unsigned char   BOOLEAN;
-  typedef unsigned char   UINT8;
-  typedef charCHAR8;
-  typedef signed char INT8;
+  #include 
+  typedef uint8_t   BOOLEAN;
+  typedef int8_tINT8;
+  typedef uint8_t   UINT8;
+  typedef int16_t   INT16;
+  typedef uint16_t  UINT16;
+  typedef int32_t   INT32;
+  typedef uint32_t  UINT32;
+  typedef int64_t   INT64;
+  typedef uint64_t  UINT64;
+  typedef char  CHAR8;
+  typedef uint16_t  CHAR16;
 
-  #define UINT8_MAX 0xff
 #endif
 
 ///
--
2.20.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48161): https://edk2.groups.io/g/devel/message/48161
Mute This Topic: https://groups.io/mt/34302597/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fix a bug of genffs command generation

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2221

The command used by multiple thread genffs feature in makefile
for testing if file exist is generated based on the toolchain family.
It should be based on the OS type.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py 
b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
index 4d8b2bdee299..bd97aa9dd208 100644
--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
@@ -11,10 +11,11 @@
 #
 from __future__ import print_function
 from __future__ import absolute_import
 
 import Common.LongFilePathOs as os
+import sys
 from sys import stdout
 from subprocess import PIPE,Popen
 from struct import Struct
 from array import array
 
@@ -484,11 +485,11 @@ class GenFdsGlobalVariable:
 Cmd += ("-o", Output)
 Cmd += Input
 
 SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
 if IsMakefile:
-if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
+if sys.platform == "win32":
 Cmd = ['if', 'exist', Input[0]] + Cmd
 else:
 Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
 if ' '.join(Cmd).strip() not in 
GenFdsGlobalVariable.SecCmdList:
 GenFdsGlobalVariable.SecCmdList.append(' 
'.join(Cmd).strip())
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48180): https://edk2.groups.io/g/devel/message/48180
Mute This Topic: https://groups.io/mt/34307097/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fix the lib order in static_library_files.lst

[Bob Feng]

Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2216

This patch is going to fix the lib order in static_library_files.lst.
This issue is introduced by commit 673d09a2dd.
Before 673d09a2dd, build tool apply build rule for the module's library
firstly and then apply build rule for module itself. Now, build tool
apply build rule for module self and then for its library. That behavior
impact the lib order in static_library_files.lst.

This patch is to call module's LibraryAutoGenList function
explicitly, where the applying build rule action for
library is triggered.

Cc: Liming Gao 
Cc: dann frazier 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py 
b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index fad5bab0f24d..f0812b6887be 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -1841,11 +1841,11 @@ class ModuleAutoGen(AutoGen):
 LibraryAutoGen.CreateCodeFile()
 
 # CanSkip uses timestamps to determine build skipping
 if self.CanSkip():
 return
-
+self.LibraryAutoGenList
 AutoGenList = []
 IgoredAutoGenList = []
 
 for File in self.AutoGenFileList:
 if GenC.Generate(File.Path, self.AutoGenFileList[File], 
File.IsBinary):
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48219): https://edk2.groups.io/g/devel/message/48219
Mute This Topic: https://groups.io/mt/34310291/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1 V2] BaseTools: Fix a bug of genffs command generation

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2221

The command used by multiple thread genffs feature in makefile
for testing if file exist is generated based on the toolchain family.
It should be based on the make program name.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 4 +++-
 BaseTools/Source/Python/build/build.py | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py 
b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
index 4d8b2bdee299..1b4c75034333 100644
--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
@@ -68,10 +68,11 @@ class GenFdsGlobalVariable:
 FfsCmdDict = {}
 SecCmdList = []
 CopyList   = []
 ModuleFile = ''
 EnableGenfdsMultiThread = True
+Make = ""
 
 #
 # The list whose element are flags to indicate if large FFS or SECTION 
files exist in FV.
 # At the beginning of each generation of FV, false flag is appended to the 
list,
 # after the call to GenerateSection returns, check the size of the output 
file,
@@ -320,10 +321,11 @@ class GenFdsGlobalVariable:
 GenFdsGlobalVariable.ModuleFile = WorkSpace.ModuleFile
 GenFdsGlobalVariable.FdfParser = FdfParser
 GenFdsGlobalVariable.WorkSpace = WorkSpace.Db
 GenFdsGlobalVariable.ArchList = ArchList
 GenFdsGlobalVariable.ToolChainTag = 
GlobalData.gGlobalDefines["TOOL_CHAIN_TAG"]
+GenFdsGlobalVariable.Make = GlobalData.gGlobalDefines['Make']
 GenFdsGlobalVariable.TargetName = GlobalData.gGlobalDefines["TARGET"]
 GenFdsGlobalVariable.ActivePlatform = GlobalData.gActivePlatform
 GenFdsGlobalVariable.ConfDir  = GlobalData.gConfDirectory
 GenFdsGlobalVariable.EnableGenfdsMultiThread = 
GlobalData.gEnableGenfdsMultiThread
 for Arch in ArchList:
@@ -484,11 +486,11 @@ class GenFdsGlobalVariable:
 Cmd += ("-o", Output)
 Cmd += Input
 
 SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
 if IsMakefile:
-if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
+if "nmake" in GenFdsGlobalVariable.Make:
 Cmd = ['if', 'exist', Input[0]] + Cmd
 else:
 Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
 if ' '.join(Cmd).strip() not in 
GenFdsGlobalVariable.SecCmdList:
 GenFdsGlobalVariable.SecCmdList.append(' 
'.join(Cmd).strip())
diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index bcd832c525f1..1c02d144d19d 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1726,11 +1726,11 @@ class Build():
 )
 self.Fdf = Wa.FdfFile
 self.LoadFixAddress = Wa.Platform.LoadFixAddress
 self.BuildReport.AddPlatformReport(Wa)
 self.Progress.Stop("done!")
-
+GlobalData.gGlobalDefines['Make'] = 
os.path.basename(Wa.ToolDefinition["MAKE"]["PATH"])
 # Add ffs build to makefile
 CmdListDict = {}
 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
 CmdListDict = self._GenFfsCmd(Wa.ArchList)
 
@@ -1827,10 +1827,11 @@ class Build():
 self.Progress,
 self.ModuleFile
 )
 self.Fdf = Wa.FdfFile
 self.LoadFixAddress = Wa.Platform.LoadFixAddress
+GlobalData.gGlobalDefines['Make'] = 
os.path.basename(Wa.ToolDefinition["MAKE"]["PATH"])
 Wa.CreateMakeFile(False)
 # Add ffs build to makefile
 CmdListDict = None
 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
 CmdListDict = self._GenFfsCmd(Wa.ArchList)
@@ -2096,11 +2097,11 @@ class Build():
 )
 self.Fdf = Wa.FdfFile
 self.LoadFixAddress = Wa.Platform.LoadFixAddress
 self.BuildReport.AddPlatformReport(Wa)
 Wa.CreateMakeFile(False)
-
+GlobalData.gGlobalDefines['Make'] = 
os.path.basename(Wa.ToolDefinition["MAKE"]["PATH"])
 # Add ffs build to makefile
 CmdListDict = {}
 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
 CmdListDict = self._GenFfsCmd(Wa.ArchList)
 
@@ -2214,11 +2215,10 @@ class Build():
 self.BuildModules = self.SetupMakeSetting(Wa)

Re: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake multi-workspace failure

[Bob Feng]

Thanks for the fix.

I agree. 
Reviewed-by:  Bob Feng 


-Original Message-
From: devel@edk2.groups.io  On Behalf Of Michael D Kinney
Sent: Friday, October 4, 2019 5:14 AM
To: devel@edk2.groups.io; Kubacki, Michael A ; 
Kinney, Michael D 
Cc: Feng, Bob C ; Gao, Liming 
Subject: Re: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake multi-workspace 
failure

Hi Michael,

Reviewed-by: Michael D Kinney 

This look really important.  Do you want me to push today?

Mike

> -Original Message-
> From: devel@edk2.groups.io  On Behalf Of 
> Kubacki, Michael A
> Sent: Tuesday, October 1, 2019 3:58 PM
> To: devel@edk2.groups.io
> Cc: Feng, Bob C ; Gao, Liming 
> ; Kinney, Michael D 
> Subject: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake 
> multi-workspace failure
> 
> REF:https://bugzilla.tianocore.org/show_bug.cgi?id=2232
> 
> Commit 0075ab2cec introduced an issue that causes an exception when 
> multiple workspace packages paths are specified. For example, if 
> edk2-platforms is used, the root directory will contain an edk and 
> edk2-platforms directory representing the respective repositories.
> 
> In GenMake, the path to the package DEC file for a module is 
> discovered by getting the relative path of the INF to the workspace 
> root directory. Each directory in the relative path is incrementally 
> joined to the WORKSPACE directory. The file list in the joined path is 
> searched for a DEC file.
> 
> As an example, if the build command is used on a package outside the 
> edk2 repository, the INF file path is relative to the edk2-platforms 
> directory not edk2. This causes directory paths to be built that do 
> not exist.
> Commit 0075ab2cec replaced the
> os.path.exists() call with a try except block that always fails when 
> os.listdir() is invoked to enumerate the list of files in the built 
> directory path on packages outside edk2.
> 
> This commit restores the original conditional statement which avoids 
> calling os.listdir() with an invalid directory path.
> 
> Cc: Bob Feng 
> Cc: Liming Gao 
> Cc: Michael D Kinney 
> Signed-off-by: Michael Kubacki
> 
> ---
>  BaseTools/Source/Python/AutoGen/GenMake.py | 4 +---
>  1 file changed, 1 insertion(+), 3 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index 584156dab9..97ba158ff2 100755
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -637,13 +637,11 @@ cleanlib:
>  while not found and os.sep in package_rel_dir:
>  index = package_rel_dir.index(os.sep)
>  current_dir = mws.join(current_dir,
> package_rel_dir[:index])
> -try:
> +if os.path.exists(current_dir):
>  for fl in os.listdir(current_dir):
>  if fl.endswith('.dec'):
>  found = True
>  break
> -except:
> -EdkLogger.error('build',
> FILE_NOT_FOUND, "WORKSPACE does not exist.")
>  package_rel_dir = package_rel_dir[index + 1:]
> 
>  MakefileTemplateDict = {
> --
> 2.16.2.windows.1
> 
> 
> 





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48467): https://edk2.groups.io/g/devel/message/48467
Mute This Topic: https://groups.io/mt/34364042/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake multi-workspace failure

[Bob Feng]

Pushed at SHA-1: 61af5f249495b18f45ca164376c871081448c0e4

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Bob Feng
Sent: Friday, October 4, 2019 11:04 AM
To: devel@edk2.groups.io; Kinney, Michael D ; 
Kubacki, Michael A 
Cc: Gao, Liming 
Subject: Re: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake multi-workspace 
failure

Thanks for the fix.

I agree. 
Reviewed-by:  Bob Feng 


-Original Message-
From: devel@edk2.groups.io  On Behalf Of Michael D Kinney
Sent: Friday, October 4, 2019 5:14 AM
To: devel@edk2.groups.io; Kubacki, Michael A ; 
Kinney, Michael D 
Cc: Feng, Bob C ; Gao, Liming 
Subject: Re: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake multi-workspace 
failure

Hi Michael,

Reviewed-by: Michael D Kinney 

This look really important.  Do you want me to push today?

Mike

> -Original Message-
> From: devel@edk2.groups.io  On Behalf Of 
> Kubacki, Michael A
> Sent: Tuesday, October 1, 2019 3:58 PM
> To: devel@edk2.groups.io
> Cc: Feng, Bob C ; Gao, Liming 
> ; Kinney, Michael D 
> Subject: [edk2-devel] [PATCH V1 1/1] BaseTools: Fix GenMake 
> multi-workspace failure
> 
> REF:https://bugzilla.tianocore.org/show_bug.cgi?id=2232
> 
> Commit 0075ab2cec introduced an issue that causes an exception when 
> multiple workspace packages paths are specified. For example, if 
> edk2-platforms is used, the root directory will contain an edk and 
> edk2-platforms directory representing the respective repositories.
> 
> In GenMake, the path to the package DEC file for a module is 
> discovered by getting the relative path of the INF to the workspace 
> root directory. Each directory in the relative path is incrementally 
> joined to the WORKSPACE directory. The file list in the joined path is 
> searched for a DEC file.
> 
> As an example, if the build command is used on a package outside the
> edk2 repository, the INF file path is relative to the edk2-platforms 
> directory not edk2. This causes directory paths to be built that do 
> not exist.
> Commit 0075ab2cec replaced the
> os.path.exists() call with a try except block that always fails when
> os.listdir() is invoked to enumerate the list of files in the built 
> directory path on packages outside edk2.
> 
> This commit restores the original conditional statement which avoids 
> calling os.listdir() with an invalid directory path.
> 
> Cc: Bob Feng 
> Cc: Liming Gao 
> Cc: Michael D Kinney 
> Signed-off-by: Michael Kubacki
> 
> ---
>  BaseTools/Source/Python/AutoGen/GenMake.py | 4 +---
>  1 file changed, 1 insertion(+), 3 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
> b/BaseTools/Source/Python/AutoGen/GenMake.py
> index 584156dab9..97ba158ff2 100755
> --- a/BaseTools/Source/Python/AutoGen/GenMake.py
> +++ b/BaseTools/Source/Python/AutoGen/GenMake.py
> @@ -637,13 +637,11 @@ cleanlib:
>  while not found and os.sep in package_rel_dir:
>  index = package_rel_dir.index(os.sep)
>  current_dir = mws.join(current_dir,
> package_rel_dir[:index])
> -try:
> +if os.path.exists(current_dir):
>  for fl in os.listdir(current_dir):
>  if fl.endswith('.dec'):
>  found = True
>  break
> -except:
> -EdkLogger.error('build',
> FILE_NOT_FOUND, "WORKSPACE does not exist.")
>  package_rel_dir = package_rel_dir[index + 1:]
> 
>  MakefileTemplateDict = {
> --
> 2.16.2.windows.1
> 
> 
> 








-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48468): https://edk2.groups.io/g/devel/message/48468
Mute This Topic: https://groups.io/mt/34364042/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/1] BaseTools: Fix a bug of genffs command generation

[Bob Feng]

Pushed at a1f94045ffe9218ec438c4d23980de4243d21cd0

-Original Message-
From: Gao, Liming 
Sent: Saturday, October 12, 2019 2:04 PM
To: Feng, Bob C ; devel@edk2.groups.io
Subject: RE: [Patch 1/1] BaseTools: Fix a bug of genffs command generation

Bob:
  I think this version patch is correct, because this case uses the different 
command in Makefile instead of the different syntax. 
 The command 'test' depends on batch environment. It is only available in bash, 
not in ms-dos. So, the check condition should be OS instead of make command.

Reviewed-by: Liming Gao 

Thanks
Liming
>-Original Message-
>From: Feng, Bob C
>Sent: Friday, September 27, 2019 10:30 AM
>To: devel@edk2.groups.io
>Cc: Gao, Liming ; Feng, Bob C 
>
>Subject: [Patch 1/1] BaseTools: Fix a bug of genffs command generation
>
>REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2221
>
>The command used by multiple thread genffs feature in makefile for 
>testing if file exist is generated based on the toolchain family.
>It should be based on the OS type.
>
>Cc: Liming Gao 
>Signed-off-by: Bob Feng 
>---
> BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
>diff --git a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>index 4d8b2bdee299..bd97aa9dd208 100644
>--- a/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>+++ b/BaseTools/Source/Python/GenFds/GenFdsGlobalVariable.py
>@@ -11,10 +11,11 @@
> #
> from __future__ import print_function
> from __future__ import absolute_import
>
> import Common.LongFilePathOs as os
>+import sys
> from sys import stdout
> from subprocess import PIPE,Popen
> from struct import Struct
> from array import array
>
>@@ -484,11 +485,11 @@ class GenFdsGlobalVariable:
> Cmd += ("-o", Output)
> Cmd += Input
>
> SaveFileOnChange(CommandFile, ' '.join(Cmd), False)
> if IsMakefile:
>-if GlobalData.gGlobalDefines.get("FAMILY") == "MSFT":
>+if sys.platform == "win32":
> Cmd = ['if', 'exist', Input[0]] + Cmd
> else:
> Cmd = ['-test', '-e', Input[0], "&&"] + Cmd
> if ' '.join(Cmd).strip() not in 
> GenFdsGlobalVariable.SecCmdList:
> GenFdsGlobalVariable.SecCmdList.append(' 
>'.join(Cmd).strip())
>--
>2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#48874): https://edk2.groups.io/g/devel/message/48874
Mute This Topic: https://groups.io/mt/34307097/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH v1] BaseTools/Capsule: Add capsule dependency support

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Li, Aaron 
Sent: Friday, January 10, 2020 9:58 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming 
Subject: [PATCH v1] BaseTools/Capsule: Add capsule dependency support

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2412

Capsule generate tool support encode capsule dependencies through '-j'
command with a JSON file. To enable dependency feature, "Dependencies"
field for each payload in JSON file is required.
The value of "Dependencies" field is C style infix notation expression.
For example:
  "Dependencies":"72E2945A-00DA-448E-9AA7-075AD840F9D4 > 0x0001"

The relation of Dependency Expression Opcode in UEFI2.8 chap 23.2 and infix 
notation expression value is as follows:
+-+--+
| OPCODE  | INFIX EXPRESSION VALUE   |
+-+--+
| 0x00 (PUSH_GUID)| {GUID}   |
| 0x01 (PUSH_VERSION) | {UINT32} |
| 0x02 (DECLEAR_VERSION_NAME} | DECLEAR "{VERSION_NAME}" |
| 0x03 (AND)  | &&   |
| 0x04 (OR)   | ||   |
| 0x05 (NOT)  | ~|
| 0x06 (TRUE) | TRUE |
| 0x07 (FALSE)| FALSE|
| 0x08 (EQ)   | ==   |
| 0x09 (GT)   | >|
| 0x0A (GTE)  | >=   |
| 0x0B (LT)   | <|
| 0x0C (LTE)  | <=       |
+-+--+

Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Aaron Li 
---
 BaseTools/Source/Python/Capsule/GenerateCapsule.py   |  62 ++-
 BaseTools/Source/Python/Common/Uefi/Capsule/CapsuleDependency.py | 409 

 2 files changed, 464 insertions(+), 7 deletions(-)

diff --git a/BaseTools/Source/Python/Capsule/GenerateCapsule.py 
b/BaseTools/Source/Python/Capsule/GenerateCapsule.py
index 6838beb682..a8de988253 100644
--- a/BaseTools/Source/Python/Capsule/GenerateCapsule.py
+++ b/BaseTools/Source/Python/Capsule/GenerateCapsule.py
@@ -31,6 +31,7 @@ import json
 from Common.Uefi.Capsule.UefiCapsuleHeader import UefiCapsuleHeaderClass  from 
Common.Uefi.Capsule.FmpCapsuleHeader  import FmpCapsuleHeaderClass
 from Common.Uefi.Capsule.FmpAuthHeader import FmpAuthHeaderClass
+from Common.Uefi.Capsule.CapsuleDependency import 
+CapsuleDependencyClass
 from Common.Edk2.Capsule.FmpPayloadHeader  import FmpPayloadHeaderClass
 
 #
@@ -306,6 +307,7 @@ if __name__ == '__main__':
 OpenSslOtherPublicCertFile   = ConvertJsonValue (Config, 
'OpenSslOtherPublicCertFile', os.path.expandvars, Required = False, Default = 
None, Open = True)
 OpenSslTrustedPublicCertFile = ConvertJsonValue (Config, 
'OpenSslTrustedPublicCertFile', os.path.expandvars, Required = False, Default = 
None, Open = True)
 SigningToolPath  = ConvertJsonValue (Config, 
'SigningToolPath', os.path.expandvars, Required = False, Default = None)
+DepexExp = ConvertJsonValue (Config, 
'Dependencies', str, Required = False, Default = None)
 
 #
 # Read binary input file
@@ -330,7 +332,8 @@ if __name__ == '__main__':
 OpenSslSignerPrivateCertFile,
 OpenSslOtherPublicCertFile,
 OpenSslTrustedPublicCertFile,
-SigningToolPath
+SigningToolPath,
+DepexExp
 ))
 
 def GenerateOutputJson (PayloadJsonDescriptorList):
@@ -348,7 +351,8 @@ if __name__ == '__main__':
   "OpenSslSignerPrivateCertFile": 
str(PayloadDescriptor.OpenSslSignerPrivateCertFile),
   "OpenSslOtherPublicCertFile": 
str(PayloadDescriptor.OpenSslOtherPublicCertFile),
   "OpenSslTrustedPublicCertFile": 
str(PayloadDescriptor.OpenSslTrustedPublicCertFile),
-  "SigningToolPath": 
str(PayloadDescriptor.SigningToolPath)
+  "SigningToolPath": 
str(PayloadDescriptor.SigningToolPath),
+  "Dependencies" : 
+ str(PayloadDescriptor.DepexExp)
   }for PayloadDescriptor in 
PayloadJsonDescriptorList

[edk2-devel] [Patch 1/1] BaseTools: Fixed a incremental build bug

[Bob Feng]

REF:https://bugzilla.tianocore.org/show_bug.cgi?id=2451

If removing a header file from source code and file
system, the incremental build will fail.

This patch is to fix this issue.

Cc: Liming Gao 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 9 +++--
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py | 5 -
 BaseTools/Source/Python/build/build.py | 3 ++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index fe94f9a4c232..ba199c1aa73d 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1,9 +1,9 @@
 ## @file
 # Create makefile for MS nmake and GNU make
 #
-# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
+# Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
 ## Import Modules
 #
@@ -189,10 +189,13 @@ class BuildFile(object):
 with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt"),"w+") as fd:
 fd.write("")
 if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency")):
 with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency"),"w+") as fd:
 fd.write("")
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps_target")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps_target"),"w+") as fd:
+fd.write("")
 return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, 
FileName), FileContent, False)
 
 ## Return a list of directory creation command string
 #
 #   @param  DirList The list of directory to be created
@@ -694,11 +697,13 @@ cleanlib:
 "dependent_library_build_directory" : 
self.LibraryBuildDirectoryList,
 "library_build_command" : LibraryMakeCommandList,
 "file_macro": FileMacroList,
 "file_build_target" : self.BuildTargetList,
 "backward_compatible_target": BcTargetList,
-"INCLUDETAG"   : 
self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","dependency")
+"INCLUDETAG"   : 
"\n".join([self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","dependency"),
+  
self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","deps_target")
+  ])
 }
 
 return MakefileTemplateDict
 
 def ParserGenerateFfsCmd(self):
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index bb6e883d84ca..1ca1798907ef 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -1,9 +1,9 @@
 ## @file
 # Build cache intermediate result and state
 #
-# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 from Common.caching import cached_property
 import Common.EdkLogger as EdkLogger
 import Common.LongFilePathOs as os
@@ -63,10 +63,13 @@ class IncludesAutoGen():
 deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
 except Exception as e:
 print(e)
 
SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
 
+def CreateDepsTarget(self):
+
SaveFileOnChange(os.path.join(self.makefile_folder,"deps_target"),"\n".join([item
 +":" for item in self.DepsCollection]),False)
+
 @cached_property
 def deps_files(self):
 """ Get all .deps file under module build folder. """
 deps_files = []
 for root, _, files in os.walk(self.d_folder, topdown=False):
diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index 34acdccbdbd0..1e47e382cba9 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1,10 +1,10 @@
 ## @file
 # build a platform or a module
 #
 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.
-#  Copyright (c) 2007 - 2019, Intel Corporation. All rights reserved.
+#  Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.
 #  Copyright 

[edk2-devel] [Patch 1/1 V2] BaseTools: Fixed a incremental build bug

[Bob Feng]

REF:https://bugzilla.tianocore.org/show_bug.cgi?id=2451

If removing a header file from source code and file
system, the incremental build will fail.

This patch is to fix this issue by setting each header file
as a target without any actions in makefile.

Signed-off-by: Bob Feng 

Cc: Liming Gao 
Reviewed-by: Liming Gao 
---
V2: add solution details in commit message.

 BaseTools/Source/Python/AutoGen/GenMake.py | 9 +++--
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py | 5 -
 BaseTools/Source/Python/build/build.py | 3 ++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index fe94f9a4c232..ba199c1aa73d 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1,9 +1,9 @@
 ## @file
 # Create makefile for MS nmake and GNU make
 #
-# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
+# Copyright (c) 2007 - 2020, Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 
 ## Import Modules
 #
@@ -189,10 +189,13 @@ class BuildFile(object):
 with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt"),"w+") as fd:
 fd.write("")
 if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency")):
 with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency"),"w+") as fd:
 fd.write("")
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps_target")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps_target"),"w+") as fd:
+fd.write("")
 return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, 
FileName), FileContent, False)
 
 ## Return a list of directory creation command string
 #
 #   @param  DirList The list of directory to be created
@@ -694,11 +697,13 @@ cleanlib:
 "dependent_library_build_directory" : 
self.LibraryBuildDirectoryList,
 "library_build_command" : LibraryMakeCommandList,
 "file_macro": FileMacroList,
 "file_build_target" : self.BuildTargetList,
 "backward_compatible_target": BcTargetList,
-"INCLUDETAG"   : 
self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","dependency")
+"INCLUDETAG"   : 
"\n".join([self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","dependency"),
+  
self._INCLUDE_CMD_[self._FileType] + " " + 
os.path.join("$(MODULE_BUILD_DIR)","deps_target")
+  ])
 }
 
 return MakefileTemplateDict
 
 def ParserGenerateFfsCmd(self):
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index bb6e883d84ca..1ca1798907ef 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -1,9 +1,9 @@
 ## @file
 # Build cache intermediate result and state
 #
-# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 from Common.caching import cached_property
 import Common.EdkLogger as EdkLogger
 import Common.LongFilePathOs as os
@@ -63,10 +63,13 @@ class IncludesAutoGen():
 deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
 except Exception as e:
 print(e)
 
SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
 
+def CreateDepsTarget(self):
+
SaveFileOnChange(os.path.join(self.makefile_folder,"deps_target"),"\n".join([item
 +":" for item in self.DepsCollection]),False)
+
 @cached_property
 def deps_files(self):
 """ Get all .deps file under module build folder. """
 deps_files = []
 for root, _, files in os.walk(self.d_folder, topdown=False):
diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index 34acdccbdbd0..1e47e382cba9 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1,10 +1,10 @@
 ## @file
 # build a platform or a module
 #
 #  Copyright (c) 2014, Hewlett-Packard Development Company, L.P.
-#  Copyrig

Re: [edk2-devel] [PATCH] BaseTools:ECC fails to detect function header comments issue

[Bob Feng]

I have 2 comments.
1. CheckParamName may return None. But according to other part of this 
function, it should return True or False.
2. The meaning of ParamList.pop(0) is not much clear. 

Thanks,
Bob

-Original Message-
From: Fan, ZhijuX 
Sent: Monday, January 20, 2020 4:40 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:ECC fails to detect function header comments issue

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1523

When the keyword after @param doesn't match the actual function parameter name, 
ECC doesn't detect such issue

The patch is going to fix this issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/Ecc/c.py | 16 +++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/Ecc/c.py b/BaseTools/Source/Python/Ecc/c.py
index a30122a45f..e42463952d 100644
--- a/BaseTools/Source/Python/Ecc/c.py
+++ b/BaseTools/Source/Python/Ecc/c.py
@@ -2554,6 +2554,20 @@ def CheckGeneralDoxygenCommentLayout(Str, StartLine, 
ErrorMsgList, CommentId= -1
 ErrorMsgList.append('Line %d : @retval appear before @param ' % 
StartLine)
 PrintErrorMsg(ERROR_DOXYGEN_CHECK_FUNCTION_HEADER, 'in Comment, 
@retval appear before @param  ', TableName, CommentId)
 
+def CheckParamName(ParamName, Tag):
+ParamList = Tag.split()
+if len(ParamList) > 1:
+ParamList.pop(0)
+for Item in ParamList:
+if Item.find('[') > 0 and Item.find(']') > 0:
+continue
+if ParamName != Item.strip():
+return True
+else:
+return False
+else:
+return True
+
 def CheckFunctionHeaderConsistentWithDoxygenComment(FuncModifier, FuncHeader, 
FuncStartLine, CommentStr, CommentStartLine, ErrorMsgList, CommentId= -1, 
TableName=''):
 
 ParamList = GetParamList(FuncHeader) @@ -2608,7 +2622,7 @@ def 
CheckFunctionHeaderConsistentWithDoxygenComment(FuncModifier, FuncHeader, Fu
 
PrintErrorMsg(ERROR_DOXYGEN_CHECK_FUNCTION_HEADER, 'in Comment, <%s> does NOT 
have %s ' % ((TagPartList[0] + ' ' + TagPartList[1]).replace('\n', 
'').replace('\r', ''), '[' + InOutStr + ']'), TableName, CommentId)
 
 
-if Tag.find(ParamName) == -1 and ParamName != 'VOID' and ParamName 
!= 'void':
+if (Tag.find(ParamName) == -1 or CheckParamName(ParamName, Tag)) 
and ParamName != 'VOID' and ParamName != 'void':
 ErrorMsgList.append('Line %d : in Comment, <%s> does NOT 
consistent with parameter name %s ' % (CommentStartLine, (TagPartList[0] + ' ' 
+ TagPartList[1]).replace('\n', '').replace('\r', ''), ParamName))
 PrintErrorMsg(ERROR_DOXYGEN_CHECK_FUNCTION_HEADER, 'in 
Comment, <%s> does NOT consistent with parameter name %s ' % ((TagPartList[0] + 
' ' + TagPartList[1]).replace('\n', '').replace('\r', ''), ParamName), 
TableName, CommentId)
 Index += 1
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53423): https://edk2.groups.io/g/devel/message/53423
Mute This Topic: https://groups.io/mt/69928321/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [edk2-platform] FitGen: Fix the issue to run in X64 linux machine

[Bob Feng]

Hi Liming,

Would you add more description for this patch?

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Liming Gao
Sent: Thursday, January 16, 2020 3:11 PM
To: devel@edk2.groups.io
Cc: Oram, Isaac W 
Subject: [edk2-devel] [edk2-platform] FitGen: Fix the issue to run in X64 linux 
machine

Cc: Isaac Oram 
Signed-off-by: Liming Gao 
---
 Silicon/Intel/Tools/FitGen/FitGen.c | 29 +++--  
Silicon/Intel/Tools/FitGen/FitGen.h |  2 +-
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/Silicon/Intel/Tools/FitGen/FitGen.c 
b/Silicon/Intel/Tools/FitGen/FitGen.c
index 833610f2a0..b6ec551333 100644
--- a/Silicon/Intel/Tools/FitGen/FitGen.c
+++ b/Silicon/Intel/Tools/FitGen/FitGen.c
@@ -226,9 +226,17 @@ typedef struct {
 #define FIT_TABLE_TYPE_BIOS_DATA_AREA 13
 #define FIT_TABLE_TYPE_CSE_SECURE_BOOT16
 
+//
+// With OptionalModule Address isn't known until free space has been // 
+identified and the optional module has been copied into the FLASH // 
+image buffer (or initialized to be populated later by another program).
+// This is very dangerous code as it can truncate 64b pointers to // 
+allocated memory buffers.  The full pointer is in Buffer for that case.
+//
 typedef struct {
   UINT32  Type;
   UINT32  Address;
+  UINT8   *Buffer; // Used by OptionalModule only
   UINT32  Size;
   UINT32  Version; // Used by OptionalModule and PortModule only  } 
FIT_TABLE_CONTEXT_ENTRY; @@ -575,9 +583,9 @@ Returns:
   UINT64  FvLength;
   EFI_GUID*TempGuid;
   UINT8   *FixPoint;
-  UINT32  Offset;
-  UINT32  FileLength;
-  UINT32  FileOccupiedSize;
+  UINTN  Offset;
+  UINTN  FileLength;
+  UINTN  FileOccupiedSize;
 
   //
   // Find the FFS file
@@ -595,7 +603,7 @@ Returns:
 InitializeFvLib (FvHeader, (UINT32)FvLength);
 
 FileHeader   = (EFI_FFS_FILE_HEADER *)((UINTN)FvHeader + 
FvHeader->HeaderLength);
-Offset   = (UINT32) (UINTN) FileHeader - (UINT32) (UINTN) FvHeader;
+Offset   = (UINTN) FileHeader - (UINTN) FvHeader;
 
 while (Offset < FvLength) {
   TempGuid = (EFI_GUID *)&(FileHeader->Name); @@ -625,7 +633,7 @@ Returns:
 return FixPoint;
   }
   FileHeader = (EFI_FFS_FILE_HEADER *)((UINTN)FileHeader + 
FileOccupiedSize);
-  Offset = (UINT32) (UINTN) FileHeader - (UINT32) (UINTN) FvHeader;
+  Offset = (UINTN) FileHeader - (UINTN) FvHeader;
 }
 
 //
@@ -1082,7 +1090,7 @@ Returns:
 return 0;
   }
   
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-  
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuffer - (UINT32) (UINTN) 
MicrocodeFileBuffer);
+
+ gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
+ MicrocodeBase + (UINT32)((UINTN) MicrocodeBuffer - (UINTN) 
+ MicrocodeFileBuffer);
   
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Size = 
MicrocodeSize;
   gFitTableContext.MicrocodeNumber++;
   gFitTableContext.FitEntryNumber++;
@@ -1110,7 +1118,7 @@ Returns:
   ///
   while (MicrocodeBuffer + SlotSize <= MicrocodeFileBuffer + 
MicrocodeFileSize) {
 
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuffer - (UINT32) (UINTN) 
MicrocodeFileBuffer);
+
+ gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
+ MicrocodeBase + (UINT32)((UINTN) MicrocodeBuffer - (UINTN) 
+ MicrocodeFileBuffer);
 gFitTableContext.MicrocodeNumber++;
 gFitTableContext.FitEntryNumber++;
 
@@ -1428,7 +1436,7 @@ Returns:
 return 0;
   }
   gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-  gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuffer - (UINT32) (UINTN) 
MicrocodeFileBuffer);
+
+ gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
+ MicrocodeBase + (UINT32)((UINTN) MicrocodeBuffer - (UINTN) 
+ MicrocodeFileBuffer);
   gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Size = 
MicrocodeSize;
   gFitTableContext.MicrocodeNumber++;
   gFitTableContext.FitEntryNumber++;
@@ -1557,6 +1565,7 @@ Returns:
 }
 
gFitTableContext.OptionalModule[gFitTableContext.OptionalModuleNumber].Type = 
Type;
 
gFitTableContext.OptionalModule[gFitTableContext.OptionalModuleNumber].Address 
= (UINT32) (UINTN) File

Re: [edk2-devel] [PATCH v2 1/1] BaseTools: Rationalise makefile generation

[Bob Feng]

Hi Pierre,

I agree with this change.

But for this patch, I found there are two bugs.
1. 
+# Get Makefile name.
+def getMakefileName(self):
+if not self._FileType:
+return _DEFAULT_FILE_NAME_   
Should be self. _DEFAULT_FILE_NAME_   
+else:
+return self._FILE_NAME_[self._FileType]
+

2.
-if not 
os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+if not 
os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile.getMakefileName())):
 return None
 for m_build_dir in ModuleBuildDirectoryList:
-if not 
os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
+if not 
os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile.getMakefileName())):
 return None

getMakefileName() can't be called from its class. 

This bug can be triggered by 
build genmake -p OvmfPkg\OvmfPkgIa32.dsc -a IA32 -t VS2015x86
build -u -p OvmfPkg\OvmfPkgIa32.dsc -a IA32 -t VS2015x86

The second build will fail.

Thanks,
Bob

-Original Message-
From: Pierre Gondois  
Sent: Tuesday, February 4, 2020 8:03 PM
To: devel@edk2.groups.io; Pierre Gondois 
Cc: Feng, Bob C ; Gao, Liming ; 
Sami Mujawar ; nd 
Subject: RE: [edk2-devel] [PATCH v2 1/1] BaseTools: Rationalise makefile 
generation

Hello Liming and Bob,
To answer Liming's questions, I am building the DynamicTablesPkg with the patch 
with the AARCH64-DEBUG and the following configurations:
Windows - GCC- GNUmake
Windows - GCC- nmake
Windows - VS2017 - GNUmake
Windows - VS2017 - nmake
Linux   - GCC5   - GNUmake
For the record, when building with the Windows-VS2017-GNUmake configuration, 
the *_*_*_MAKE_FLAG(S) variable is set to "/nologo". As this is a nmake 
specific flag, it has to be removed when using GNUmake. Some warning flags also 
need to be slightly modified when building with VS2017.
To modify the type of make tool to use (GNUmake or nmake), I modify the 
following variable in Conf/tools_def.txt:
*_[GCC5|VS2017]_*_MAKE_PATH = [nmake|GNUmake]

I am building the ShellPkg with the patch and a small modification on how nasm 
files are added for IA32 and X64 for the following configurations:
Windows - VS2017 - GNUmake
Windows - VS2017 - nmake
Linux   - GCC- GNUmake

Unfortunately, the gcc compiler available on windows doesn't provide all the 
options of the one available on linux. For instances:
 * When building ShellPkg-DEBUG build-X64 -Windows-GCC-[nmake], the 
"-mcmodel=small" option provided is not supported.
 * When building ShellPkg-DEBUG build-IA32-Windows-GCC-[nmake], the 
"-m,elf_i386" option provided is not supported.
These issues are related to the compiler and not to the patch.

The Windows-GCC-GNUmake configuration is very important for Arm. This 
configuration was working fine until patch 
"0c3e8e9947a6c13b4327dd11b20acb95441701cf BaseTools: Enhance Basetool for 
incremental build" was merged in edk2.

Regards,
Pierre

-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois 
via Groups.Io
Sent: 04 February 2020 12:01
To: devel@edk2.groups.io
Cc: Pierre Gondois ; bob.c.f...@intel.com; 
liming@intel.com; Sami Mujawar ; Pierre Gondois 
; nd 
Subject: [edk2-devel] [PATCH v2 1/1] BaseTools: Rationalise makefile generation

From: Pierre Gondois 

The GenMake.py script tests the platform environment to determine the type of 
makefile that needs to be generated. If a Windows build host is detected, the 
makefile generated is of Nmake type. Otherwise a GNUmake type is generated.

Furthermore, the ___MAKE_PATH
option in tools_def.template defines the make tool to use.
E.g.: for VS2017 this is configured to use Nmake, cf.
*_VS2017_*_MAKE_PATH = DEF(VS2017_BIN_HOST)\nmake.exe while for GCC5 it is 
setup to use GNU make.
*_GCC5_*_MAKE_PATH = DEF(GCC_HOST_PREFIX)make

This prevents using the GCC compiler toolchain on a Windows build host.

To address this issue this patch introduces 2 factors to determine the 
generated makefile output.
  1. Platform -> to determine shell commands used
 in makefile.
  2. MakeTool -> to determine the type of makefile
 that needs to be generated.

Signed-off-by: Pierre Gondois 
Signed-off-by: Sami Mujawar 
---

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/720_BaseTools_Rationalise_makefile_generation_v2

Notes:
v2:
  - Rebase on latest master [Pierre]

 BaseTools/Source/Python/AutoGen/GenMake.py | 122 ++--
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py |  34 +++---
 BaseTools/Source/Python/build/build.py |   7 +-
 3 files changed, 88 insertions(+), 75 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 
ba199c1aa73dc46856b41c13e07e3d9770081acd..f49c765c791d57c06fcccf7059b37a018dc68ae6
 100755
--- a

Re: [edk2-devel] [PATCH v3 1/1] BaseTools: Script for converting .aml to .hex

[Bob Feng]

I think INFO or DEBUG would be better. I think some project may care about the 
warning message and try to resolve it, but actually it's no need to resolve.

-Original Message-
From: Pierre Gondois  
Sent: Tuesday, February 4, 2020 11:53 PM
To: Pierre Gondois ; devel@edk2.groups.io
Cc: ard.biesheu...@linaro.org; Feng, Bob C ; Gao, Liming 
; Sami Mujawar ; nd 
Subject: RE: [PATCH v3 1/1] BaseTools: Script for converting .aml to .hex

[From Liming]
> If so, this is not the error for other ACPI table. I suggest to print INFO 
> message and directly return with success return value.
I've put a EdkLogger.warn(), so it doesn't stop the execution of the script and 
the message has a log level high enough to be printed.

Regards,
Pierre
 
-Original Message-
From: PierreGondois 
Sent: 04 February 2020 15:48
To: devel@edk2.groups.io
Cc: Pierre Gondois ; ard.biesheu...@linaro.org; 
bob.c.f...@intel.com; liming@intel.com; Sami Mujawar 
; nd 
Subject: [PATCH v3 1/1] BaseTools: Script for converting .aml to .hex

From: Pierre Gondois 

The "-tc" option of the iasl compiler allows to generate a .hex file containing 
a C array storing AML bytecode.

An online discussion suggested that this "-tc" option was specific to the iasl 
compiler and it shouldn't be relied on. This conversation is available at:
https://edk2.groups.io/g/devel/topic/39786201#49659

A way to address this issue is to implement a compiler independent script that 
takes an AML file as input, and generates a .hex file.

This patch implements a Python script that converts an AML file to a .hex file, 
containing a C array storing AML bytecode.
This scipt has been tested with the AML output from the following compilers 
supported by the EDKII implementation:
  * Intel ASL compiler
  * Microsoft ASL compiler

Signed-off-by: Pierre Gondois 
---

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/718_asl_to_hex_script_converter_v3

Notes:
v3:
 - When a file without a DSDT or SSDT signature is given as input,
   give a warning instead of an error. [Pierre]
 - Return None value instead of nothing in functions. [Pierre]

 BaseTools/BinWrappers/PosixLike/AmlToHex   |  14 ++
 BaseTools/BinWrappers/WindowsLike/AmlToHex.bat |   3 +
 BaseTools/Conf/build_rule.template |   3 +
 BaseTools/Source/Python/AmlToHex/AmlToHex.py   | 158 
 4 files changed, 178 insertions(+)

diff --git a/BaseTools/BinWrappers/PosixLike/AmlToHex 
b/BaseTools/BinWrappers/PosixLike/AmlToHex
new file mode 100755
index 
..1dd28e966288f6ea4fc52d42e2dc7b1f74226c23
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/AmlToHex
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to 
+python if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see
+http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is 
+not a good choice here dir=$(dirname "$full_cmd") exe=$(basename
+"$full_cmd")
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" "$dir/../../Source/Python/$exe/$exe.py" "$@"
diff --git a/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat 
b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
new file mode 100644
index 
..9616cd893bec9902451e6d8591f537cc408bd5e5
--- /dev/null
+++ b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND%
+%BASE_TOOLS_PATH%\Source\Python\%ToolName%\%ToolName%.py %*
diff --git a/BaseTools/Conf/build_rule.template 
b/BaseTools/Conf/build_rule.template
index 
51748bc0655a5c656258a3007b4db6b2dc941ea0..0822b681fcd9f61c6508e6f93ffc31fa70fd7059
 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -1,6 +1,7 @@
 #
 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.  #  
Portions copyright (c) 2008 - 2010, Apple Inc. All rights reserved.
+#  Copyright (c) 2020, ARM Ltd. All rights reserved.
 #  SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -427,12 +428,14 @@
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) /I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --source-code -l -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}. 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii 
 "$(ASL)" $(ASL_FLAGS) $(ASL_OUTFLAGS)${dst} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.
+-AmlToHex $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.aml
 
 
 Trim --asl-file --asl-deps -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i 
-i $(INC_LIST) ${src}
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) -I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --sour

Re: [edk2-devel] [PATCH v3 1/1] BaseTools: Build ASL files before C files

[Bob Feng]

Hi Pierre,

This patch looks good to me.

Is there a Bugzilla for this patch? And will you send the patch for INF spec? 

There may be a typo in the log message:
[Sources]
  FileName1.X
  FileName2.Y : FileName1.X
  FileName3.Z : FileName1.X FileName3.Z# here the FileName3.Z  should be 
FileName2.Y   ?

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Wednesday, February 5, 2020 1:22 AM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; ard.biesheu...@linaro.org; Feng, 
Bob C ; Gao, Liming ; 
sami.muja...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v3 1/1] BaseTools: Build ASL files before C files

From: Pierre Gondois 

The dependencies for C files are satisfied by the build system.
However, there are use cases where source files with different languages are 
inter-dependent. The EDKII build framework currently doesn't have options to 
specify such dependencies.
E.g. It may be necessary to specify the build order between
 ASL files and C files. The use case being that the AML
 blob generated by compiling the ASL files is loaded and
 parsed by some C code.

This patch allows to describe dependencies between files listed in the 
'[Sources]' section of '.inf' files. The list of source files to build prior 
starts with a colon (':'), e.g.:
[Sources]
  FileName1.X
  FileName2.Y : FileName1.X
  FileName3.Z : FileName1.X FileName3.Z

In the example above:
  * FileName1.X will be built prior to FileName2.Y.
  * FileName1.X and FileName2.Y will be built prior to
FileName3.Z.

This does not affect the file specific build options, which are pipe separated, 
e.g.:
  FileName2.Y | GCC : FileName1.X

The list of colon separated files must be part of the module, i.e. they must be 
present in the [Sources] section.

Their file extension must be described in the 
BaseTools/Conf/build_rule.template file, and generate an output file, i.e. have 
a non-empty '' section.

Declaring a dependency in the [Sources] sections leads to the creation of 
makefile rules between these files in the build. The makefile rule is between 
the generated files.
E.g.:
  FileName1.X generates FileName1.objx
  FileName2.Y generates FileName2.objy
  Then
FileName2.Y : FileName1.X
  will create the following makefile rule:
FileName2.objy : FileName1.objx

INF Specification updates:
s3.2.1, "Common Definitions":
   ::= ":"
 ::=   

s2.5, "[Sources] Section":
  To describe a build order dependency between source
  files, specify the source files to build prior by
  listing them, colon separated.
   ::=  [] []
   ::=  [FileNameDependency]*

Signed-off-by: Pierre Gondois 
---

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/676_build_asl_first_v3

Notes:
v3:
 - In .inf files, after the first colon starting the list of file
   dependencies, the list of files are space separated. In v2,
   a colon was required before each file. Cf the example in the
   commit message. [Pierre]

 BaseTools/Source/Python/AutoGen/BuildEngine.py  |  6 
 BaseTools/Source/Python/AutoGen/GenMake.py  |  7 +
 BaseTools/Source/Python/AutoGen/ModuleAutoGen.py| 17 +++
 BaseTools/Source/Python/Common/DataType.py  |  3 +-
 BaseTools/Source/Python/Common/Misc.py  |  3 ++
 BaseTools/Source/Python/Workspace/InfBuildData.py   | 32 ++--
 BaseTools/Source/Python/Workspace/MetaFileParser.py | 18 +--
 7 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py 
b/BaseTools/Source/Python/AutoGen/BuildEngine.py
index 
d602414ca41f37155c9c6d00eec54ea3918840c3..687617756619a5b547f18c99c4773842a8328ae8
 100644
--- a/BaseTools/Source/Python/AutoGen/BuildEngine.py
+++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py
@@ -2,6 +2,8 @@
 # The engine for building files
 #
 # Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.
+# Copyright (c) 2020, ARM Limited. All rights reserved. #
 # SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -53,6 +55,7 @@ class TargetDescBlock(object):
 self.Outputs = Outputs
 self.Commands = Commands
 self.Dependencies = Dependencies
+self.SourceFileDependencies = None
 if self.Outputs:
 self.Target = self.Outputs[0]
 else:
@@ -277,6 +280,9 @@ class FileBuildRule:
 TargetDesc.GenListFile = self.GenListFile
 TargetDesc.GenIncListFile = self.GenIncListFile
 self.BuildTargets[DstFile[0]] = TargetDesc
+
+TargetDesc.SourceFileDependencies = 
+ SourceFile.SourceFileDependencies
+
 return TargetDesc
 
 def _BuildCommand(self, Macros):
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 
ba199c1aa73dc46856b41c13e07e3d9770081acd..9d9ac328b1f1fd781e3057d34116485e5ac99afd
 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/Bas

Re: [edk2-devel] [PATCH 0/3] BaseTools/Scripts: .mailmap improvements

[Bob Feng]

This patch set looks good to me.

Reviewed-by: Bob Feng 

-Original Message-
From: Philippe Mathieu-Daude [mailto:phi...@redhat.com] 
Sent: Wednesday, February 5, 2020 6:49 AM
To: devel@edk2.groups.io
Cc: Philippe Mathieu-Daude ; Feng, Bob C 
; Gao, Liming 
Subject: [PATCH 0/3] BaseTools/Scripts: .mailmap improvements

Hi,

This series improves PatchCheck.py so Mergify can catch the emails rewritten by 
the mailing list.
Also it enable mailmap usage by default in the git config.

Regards,

Phil.

Cc: Bob Feng 
Cc: Liming Gao 

Philippe Mathieu-Daudé (3):
  BaseTools/Scripts/PatchCheck.py: Do not use mailmap
  BaseTools/Scripts/PatchCheck.py: Detect emails rewritten by Groups.Io
  BaseTools/Scripts: Add log.mailmap to SetupGit.py

 BaseTools/Scripts/PatchCheck.py | 10 --
 BaseTools/Scripts/SetupGit.py   |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

--
2.21.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53824): https://edk2.groups.io/g/devel/message/53824
Mute This Topic: https://groups.io/mt/70984702/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch] BaseTools tools_def.template: Add back -fno-pie option in GCC49 tool chain

[Bob Feng]

Reviewed-by: Bob Feng  

-Original Message-
From: Gao, Liming  
Sent: Tuesday, February 4, 2020 12:55 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C 
Subject: [Patch] BaseTools tools_def.template: Add back -fno-pie option in 
GCC49 tool chain

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2502
This option is required to make GCC49 tool chain work with the high version GCC 
compiler.

Cc: Bob Feng 
Signed-off-by: Liming Gao 
---
 BaseTools/Conf/tools_def.template | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/BaseTools/Conf/tools_def.template 
b/BaseTools/Conf/tools_def.template
index feee2bbf16..d02424ae44 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -1974,7 +1974,7 @@ DEFINE GCC48_ARM_ASLDLINK_FLAGS  = 
DEF(GCC_ARM_ASLDLINK_FLAGS) -Wl,--oformat
 DEFINE GCC48_AARCH64_ASLDLINK_FLAGS  = DEF(GCC_AARCH64_ASLDLINK_FLAGS)
 DEFINE GCC48_ASLCC_FLAGS = DEF(GCC_ASLCC_FLAGS)
 
-DEFINE GCC49_IA32_CC_FLAGS   = DEF(GCC48_IA32_CC_FLAGS)
+DEFINE GCC49_IA32_CC_FLAGS   = DEF(GCC48_IA32_CC_FLAGS) -fno-pic 
-fno-pie
 DEFINE GCC49_X64_CC_FLAGS= DEF(GCC48_X64_CC_FLAGS)
 DEFINE GCC49_IA32_X64_DLINK_COMMON   = -nostdlib -Wl,-n,-q,--gc-sections -z 
common-page-size=0x40
 DEFINE GCC49_IA32_X64_ASLDLINK_FLAGS = DEF(GCC49_IA32_X64_DLINK_COMMON) 
-Wl,--defsym=PECOFF_HEADER_SIZE=0 DEF(GCC_DLINK2_FLAGS_COMMON) 
-Wl,--entry,ReferenceAcpiTable -u ReferenceAcpiTable
@@ -1997,7 +1997,7 @@ DEFINE GCC49_ARM_ASLDLINK_FLAGS  = 
DEF(GCC48_ARM_ASLDLINK_FLAGS)
 DEFINE GCC49_AARCH64_ASLDLINK_FLAGS  = DEF(GCC48_AARCH64_ASLDLINK_FLAGS)
 DEFINE GCC49_ASLCC_FLAGS = DEF(GCC48_ASLCC_FLAGS)
 
-DEFINE GCC5_IA32_CC_FLAGS= DEF(GCC49_IA32_CC_FLAGS) -fno-pic 
-fno-pie
+DEFINE GCC5_IA32_CC_FLAGS= DEF(GCC49_IA32_CC_FLAGS)
 DEFINE GCC5_X64_CC_FLAGS = DEF(GCC49_X64_CC_FLAGS)
 DEFINE GCC5_IA32_X64_DLINK_COMMON= DEF(GCC49_IA32_X64_DLINK_COMMON)
 DEFINE GCC5_IA32_X64_ASLDLINK_FLAGS  = DEF(GCC49_IA32_X64_ASLDLINK_FLAGS)
--
2.24.1.windows.2


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53855): https://edk2.groups.io/g/devel/message/53855
Mute This Topic: https://groups.io/mt/70966421/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [edk2-platform][patch v3] FitGen: Fix the issue to run in X64 linux machine

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Gao, Liming  
Sent: Wednesday, February 5, 2020 10:06 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Oram, Isaac W 
Subject: [edk2-platform][patch v3] FitGen: Fix the issue to run in X64 linux 
machine

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2466
Memory allocation (malloc) may return the buffer address be above 4G.
Current logic always converts the memory address to UINT32. It will cause 
memory read and free corrupt. This patch uses pointer to store the allocated 
memory address.

Cc: Bob Feng 
Cc: Isaac Oram 
Signed-off-by: Liming Gao 
---
In v2: update the commit message. 
In v3: correct Index to access OptionalModule field  
Silicon/Intel/Tools/FitGen/FitGen.c | 31 ---  
Silicon/Intel/Tools/FitGen/FitGen.h |  2 +-
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/Silicon/Intel/Tools/FitGen/FitGen.c 
b/Silicon/Intel/Tools/FitGen/FitGen.c
index 833610f2a0..9f1db32a15 100644
--- a/Silicon/Intel/Tools/FitGen/FitGen.c
+++ b/Silicon/Intel/Tools/FitGen/FitGen.c
@@ -2,7 +2,7 @@
 This utility is part of build process for IA32/X64 FD.
 It generates FIT table.
 
-Copyright (c) 2010-2019, Intel Corporation. All rights reserved.
+Copyright (c) 2010-2020, Intel Corporation. All rights reserved.
 SPDX-License-Identifier: BSD-2-Clause-Patent
 
 **/
@@ -226,9 +226,17 @@ typedef struct {
 #define FIT_TABLE_TYPE_BIOS_DATA_AREA 13
 #define FIT_TABLE_TYPE_CSE_SECURE_BOOT16
 
+//
+// With OptionalModule Address isn't known until free space has been // 
+identified and the optional module has been copied into the FLASH // 
+image buffer (or initialized to be populated later by another program).
+// This is very dangerous code as it can truncate 64b pointers to // 
+allocated memory buffers.  The full pointer is in Buffer for that case.
+//
 typedef struct {
   UINT32  Type;
   UINT32  Address;
+  UINT8   *Buffer; // Used by OptionalModule only
   UINT32  Size;
   UINT32  Version; // Used by OptionalModule and PortModule only  } 
FIT_TABLE_CONTEXT_ENTRY; @@ -575,9 +583,9 @@ Returns:
   UINT64  FvLength;
   EFI_GUID*TempGuid;
   UINT8   *FixPoint;
-  UINT32  Offset;
-  UINT32  FileLength;
-  UINT32  FileOccupiedSize;
+  UINTN   Offset;
+  UINTN   FileLength;
+  UINTN   FileOccupiedSize;
 
   //
   // Find the FFS file
@@ -595,7 +603,7 @@ Returns:
 InitializeFvLib (FvHeader, (UINT32)FvLength);
 
 FileHeader   = (EFI_FFS_FILE_HEADER *)((UINTN)FvHeader + 
FvHeader->HeaderLength);
-Offset   = (UINT32) (UINTN) FileHeader - (UINT32) (UINTN) FvHeader;
+Offset   = (UINTN) FileHeader - (UINTN) FvHeader;
 
 while (Offset < FvLength) {
   TempGuid = (EFI_GUID *)&(FileHeader->Name); @@ -625,7 +633,7 @@ Returns:
 return FixPoint;
   }
   FileHeader = (EFI_FFS_FILE_HEADER *)((UINTN)FileHeader + 
FileOccupiedSize);
-  Offset = (UINT32) (UINTN) FileHeader - (UINT32) (UINTN) FvHeader;
+  Offset = (UINTN) FileHeader - (UINTN) FvHeader;
 }
 
 //
@@ -1082,7 +1090,7 @@ Returns:
 return 0;
   }
   
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-  
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuffer - (UINT32) (UINTN) 
MicrocodeFileBuffer);
+
+ gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
+ MicrocodeBase + (UINT32)((UINTN) MicrocodeBuffer - (UINTN) 
+ MicrocodeFileBuffer);
   
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Size = 
MicrocodeSize;
   gFitTableContext.MicrocodeNumber++;
   gFitTableContext.FitEntryNumber++;
@@ -1110,7 +1118,7 @@ Returns:
   ///
   while (MicrocodeBuffer + SlotSize <= MicrocodeFileBuffer + 
MicrocodeFileSize) {
 
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-
gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuffer - (UINT32) (UINTN) 
MicrocodeFileBuffer);
+
+ gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
+ MicrocodeBase + (UINT32)((UINTN) MicrocodeBuffer - (UINTN) 
+ MicrocodeFileBuffer);
 gFitTableContext.MicrocodeNumber++;
 gFitTableContext.FitEntryNumber++;
 
@@ -1428,7 +1436,7 @@ Returns:
 return 0;
   }
   gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Type = 
FIT_TABLE_TYPE_MICROCODE;
-  gFitTableContext.Microcode[gFitTableContext.MicrocodeNumber].Address = 
MicrocodeBase + ((UINT32) (UINTN) MicrocodeBuf

Re: [edk2-devel] [PATCH v4 1/1] BaseTools: Script for converting .aml to .hex

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Wednesday, February 5, 2020 10:52 PM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; ard.biesheu...@linaro.org; Feng, 
Bob C ; Gao, Liming ; 
sami.muja...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v4 1/1] BaseTools: Script for converting .aml to 
.hex

From: Pierre Gondois 

The "-tc" option of the iasl compiler allows to generate a .hex file containing 
a C array storing AML bytecode.

An online discussion suggested that this "-tc" option was specific to the iasl 
compiler and it shouldn't be relied on. This conversation is available at:
https://edk2.groups.io/g/devel/topic/39786201#49659

A way to address this issue is to implement a compiler independent script that 
takes an AML file as input, and generates a .hex file.

This patch implements a Python script that converts an AML file to a .hex file, 
containing a C array storing AML bytecode.
This scipt has been tested with the AML output from the following compilers 
supported by the EDKII implementation:
  * Intel ASL compiler
  * Microsoft ASL compiler

Signed-off-by: Pierre Gondois 
---

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/718_asl_to_hex_script_converter_v4

Notes:
v4:
 - Replacing warning message by info message [Pierre]

 BaseTools/BinWrappers/PosixLike/AmlToHex   |  14 ++
 BaseTools/BinWrappers/WindowsLike/AmlToHex.bat |   3 +
 BaseTools/Conf/build_rule.template |   3 +
 BaseTools/Source/Python/AmlToHex/AmlToHex.py   | 156 
 4 files changed, 176 insertions(+)

diff --git a/BaseTools/BinWrappers/PosixLike/AmlToHex 
b/BaseTools/BinWrappers/PosixLike/AmlToHex
new file mode 100755
index 
..1dd28e966288f6ea4fc52d42e2dc7b1f74226c23
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/AmlToHex
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to 
+python if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see 
+http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is 
+not a good choice here dir=$(dirname "$full_cmd") exe=$(basename 
+"$full_cmd")
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" "$dir/../../Source/Python/$exe/$exe.py" "$@"
diff --git a/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat 
b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
new file mode 100644
index 
..9616cd893bec9902451e6d8591f537cc408bd5e5
--- /dev/null
+++ b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND% 
+%BASE_TOOLS_PATH%\Source\Python\%ToolName%\%ToolName%.py %*
diff --git a/BaseTools/Conf/build_rule.template 
b/BaseTools/Conf/build_rule.template
index 
51748bc0655a5c656258a3007b4db6b2dc941ea0..0822b681fcd9f61c6508e6f93ffc31fa70fd7059
 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -1,6 +1,7 @@
 #
 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.  #  
Portions copyright (c) 2008 - 2010, Apple Inc. All rights reserved.
+#  Copyright (c) 2020, ARM Ltd. All rights reserved.
 #  SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -427,12 +428,14 @@
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) /I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --source-code -l -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}. 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii 
 "$(ASL)" $(ASL_FLAGS) $(ASL_OUTFLAGS)${dst} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.
+-AmlToHex $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.aml
 
 
 Trim --asl-file --asl-deps -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i 
-i $(INC_LIST) ${src}
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) -I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --source-code -l -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}. 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii 
 "$(ASL)" $(ASL_FLAGS) $(ASL_OUTFLAGS)${dst} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.
+-AmlToHex $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.aml
 
 [C-Code-File.AcpiTable]
 
diff --git a/BaseTools/Source/Python/AmlToHex/AmlToHex.py 
b/BaseTools/Source/Python/AmlToHex/AmlToHex.py
new file mode 100644
index 
..643db2910e37acfdd80ac18d288c921320a79ce1
--- /dev/null
+++ b/BaseTools/Source/Python/AmlToHex/AmlToHex.py
@@ -0,0 +1,156 @@
+## @file
+#
+# Convert an AML file to a .hex file c

Re: [edk2-devel] [PATCH 0/3] BaseTools/Scripts: .mailmap improvements

[Bob Feng]

Pushed.

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Bob Feng
Sent: Wednesday, February 5, 2020 9:47 PM
To: Philippe Mathieu-Daude ; devel@edk2.groups.io
Cc: Gao, Liming 
Subject: Re: [edk2-devel] [PATCH 0/3] BaseTools/Scripts: .mailmap improvements

This patch set looks good to me.

Reviewed-by: Bob Feng 

-Original Message-
From: Philippe Mathieu-Daude [mailto:phi...@redhat.com] 
Sent: Wednesday, February 5, 2020 6:49 AM
To: devel@edk2.groups.io
Cc: Philippe Mathieu-Daude ; Feng, Bob C 
; Gao, Liming 
Subject: [PATCH 0/3] BaseTools/Scripts: .mailmap improvements

Hi,

This series improves PatchCheck.py so Mergify can catch the emails rewritten by 
the mailing list.
Also it enable mailmap usage by default in the git config.

Regards,

Phil.

Cc: Bob Feng 
Cc: Liming Gao 

Philippe Mathieu-Daudé (3):
  BaseTools/Scripts/PatchCheck.py: Do not use mailmap
  BaseTools/Scripts/PatchCheck.py: Detect emails rewritten by Groups.Io
  BaseTools/Scripts: Add log.mailmap to SetupGit.py

 BaseTools/Scripts/PatchCheck.py | 10 --
 BaseTools/Scripts/SetupGit.py   |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

--
2.21.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53863): https://edk2.groups.io/g/devel/message/53863
Mute This Topic: https://groups.io/mt/70984702/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH v4 1/1] BaseTools: Script for converting .aml to .hex

[Bob Feng]

Pushed. 

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Bob Feng
Sent: Thursday, February 6, 2020 6:09 PM
To: devel@edk2.groups.io; pierre.gond...@arm.com
Cc: ard.biesheu...@linaro.org; Gao, Liming ; 
sami.muja...@arm.com; n...@arm.com
Subject: Re: [edk2-devel] [PATCH v4 1/1] BaseTools: Script for converting .aml 
to .hex

Reviewed-by: Bob Feng 

-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Wednesday, February 5, 2020 10:52 PM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; ard.biesheu...@linaro.org; Feng, 
Bob C ; Gao, Liming ; 
sami.muja...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v4 1/1] BaseTools: Script for converting .aml to 
.hex

From: Pierre Gondois 

The "-tc" option of the iasl compiler allows to generate a .hex file containing 
a C array storing AML bytecode.

An online discussion suggested that this "-tc" option was specific to the iasl 
compiler and it shouldn't be relied on. This conversation is available at:
https://edk2.groups.io/g/devel/topic/39786201#49659

A way to address this issue is to implement a compiler independent script that 
takes an AML file as input, and generates a .hex file.

This patch implements a Python script that converts an AML file to a .hex file, 
containing a C array storing AML bytecode.
This scipt has been tested with the AML output from the following compilers 
supported by the EDKII implementation:
  * Intel ASL compiler
  * Microsoft ASL compiler

Signed-off-by: Pierre Gondois 
---

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/718_asl_to_hex_script_converter_v4

Notes:
v4:
 - Replacing warning message by info message [Pierre]

 BaseTools/BinWrappers/PosixLike/AmlToHex   |  14 ++
 BaseTools/BinWrappers/WindowsLike/AmlToHex.bat |   3 +
 BaseTools/Conf/build_rule.template |   3 +
 BaseTools/Source/Python/AmlToHex/AmlToHex.py   | 156 
 4 files changed, 176 insertions(+)

diff --git a/BaseTools/BinWrappers/PosixLike/AmlToHex 
b/BaseTools/BinWrappers/PosixLike/AmlToHex
new file mode 100755
index 
..1dd28e966288f6ea4fc52d42e2dc7b1f74226c23
--- /dev/null
+++ b/BaseTools/BinWrappers/PosixLike/AmlToHex
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#python `dirname $0`/RunToolFromSource.py `basename $0` $*
+
+# If a ${PYTHON_COMMAND} command is available, use it in preference to 
+python if command -v ${PYTHON_COMMAND} >/dev/null 2>&1; then
+python_exe=${PYTHON_COMMAND}
+fi
+
+full_cmd=${BASH_SOURCE:-$0} # see
+http://mywiki.wooledge.org/BashFAQ/028 for a discussion of why $0 is 
+not a good choice here dir=$(dirname "$full_cmd") exe=$(basename
+"$full_cmd")
+
+export PYTHONPATH="$dir/../../Source/Python${PYTHONPATH:+:"$PYTHONPATH"}"
+exec "${python_exe:-python}" "$dir/../../Source/Python/$exe/$exe.py" "$@"
diff --git a/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat 
b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
new file mode 100644
index 
..9616cd893bec9902451e6d8591f537cc408bd5e5
--- /dev/null
+++ b/BaseTools/BinWrappers/WindowsLike/AmlToHex.bat
@@ -0,0 +1,3 @@
+@setlocal
+@set ToolName=%~n0%
+@%PYTHON_COMMAND%
+%BASE_TOOLS_PATH%\Source\Python\%ToolName%\%ToolName%.py %*
diff --git a/BaseTools/Conf/build_rule.template 
b/BaseTools/Conf/build_rule.template
index 
51748bc0655a5c656258a3007b4db6b2dc941ea0..0822b681fcd9f61c6508e6f93ffc31fa70fd7059
 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -1,6 +1,7 @@
 #
 #  Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.  #  
Portions copyright (c) 2008 - 2010, Apple Inc. All rights reserved.
+#  Copyright (c) 2020, ARM Ltd. All rights reserved.
 #  SPDX-License-Identifier: BSD-2-Clause-Patent  #
 
@@ -427,12 +428,14 @@
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) /I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --source-code -l -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}. 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii 
 "$(ASL)" $(ASL_FLAGS) $(ASL_OUTFLAGS)${dst} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.
+-AmlToHex $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.aml
 
 
 Trim --asl-file --asl-deps -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i 
-i $(INC_LIST) ${src}
 "$(ASLPP)" $(DEPS_FLAGS) $(ASLPP_FLAGS) $(INC) -I${s_path} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.i > 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii
 Trim --source-code -l -o $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}. 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.iii 
 "$(ASL)" $(ASL_FLAGS) $(ASL_OUTFLAGS)${dst} 
$(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.
+-AmlToHex $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.aml
 
 [C-Code-Fil

Re: [edk2-devel] [Patch 02/11] BaseTools/Plugin: Add HostBasedUnitTestRunner plugin

[Bob Feng]

Acked-by: Bob Feng 

-Original Message-
From: Kinney, Michael D  
Sent: Friday, January 24, 2020 10:10 AM
To: devel@edk2.groups.io
Cc: Sean Brogan ; Bret Barkelew 
; Feng, Bob C ; Gao, Liming 

Subject: [Patch 02/11] BaseTools/Plugin: Add HostBasedUnitTestRunner plugin

Add plugin to BaseTools to run host based unit tests.

Cc: Sean Brogan 
Cc: Bret Barkelew 
Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Michael D Kinney 
---
 .../HostBasedUnitTestRunner.py| 115 ++
 .../HostBasedUnitTestRunner_plug_in.yaml  |  12 ++
 2 files changed, 127 insertions(+)
 create mode 100644 
BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py
 create mode 100644 
BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner_plug_in.yaml

diff --git 
a/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py 
b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.py
new file mode 100644
index 00..92426760ae
--- /dev/null
+++ b/BaseTools/Plugin/HostBasedUnitTestRunner/HostBasedUnitTestRunner.p
+++ y
@@ -0,0 +1,115 @@
+# @file HostBasedUnitTestRunner.py
+# Plugin to located any host-based unit tests in the output directory and 
execute them.
+##
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent # ## import os import 
+logging import glob import xml.etree.ElementTree from 
+edk2toolext.environment.plugintypes.uefi_build_plugin import 
+IUefiBuildPlugin from edk2toolext import edk2_logging import 
+edk2toollib.windows.locate_tools as locate_tools from 
+edk2toolext.environment import shell_environment from 
+edk2toollib.utility_functions import RunCmd
+
+
+class HostBasedUnitTestRunner(IUefiBuildPlugin):
+
+def do_pre_build(self, thebuilder):
+'''
+Works with the compiler (either the HostBasedCompilerPlugin or an 
other Builder) to set
+up the environment that will be needed to build host-based unit tests.
+
+EXPECTS:
+- Build Var 'CI_BUILD_TYPE' - If not set to 'host_unit_test', will not 
do anything.
+
+UPDATES:
+- Shell Var (Several) - Updates the shell with all vars listed in 
interesting_keys.
+- Shell Path - Updated from QueryVcVariables()
+- Shell Var 'CMOCKA_MESSAGE_OUTPUT'
+'''
+ci_type = thebuilder.env.GetValue('CI_BUILD_TYPE')
+if ci_type != 'host_unit_test':
+return 0
+
+shell_env = shell_environment.GetEnvironment()
+# Use the tools lib to determine the correct values for the vars that 
interest us.
+interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB", "LIBPATH", 
"UniversalCRTSdkDir",
+"UCRTVersion", "WindowsLibPath", 
"WindowsSdkBinPath", "WindowsSdkDir", "WindowsSdkVerBinPath",
+"WindowsSDKVersion", "VCToolsInstallDir"]
+vs_vars = locate_tools.QueryVcVariables(interesting_keys, "amd64")
+for (k, v) in vs_vars.items():
+if k.upper() == "PATH":
+shell_env.append_path(v)
+else:
+shell_env.set_shell_var(k, v)
+
+# Set up the reporting type for Cmocka.
+shell_env.set_shell_var('CMOCKA_MESSAGE_OUTPUT', 'xml')
+return 0
+
+def do_post_build(self, thebuilder):
+'''
+After a build, will automatically locate and run all host-based unit 
tests. Logs any
+failures with Warning severity and will return a count of the failures 
as the return code.
+
+EXPECTS:
+- Build Var 'CI_BUILD_TYPE' - If not set to 'host_unit_test', will not 
do anything.
+
+UPDATES:
+- Shell Var 'CMOCKA_XML_FILE'
+'''
+ci_type = thebuilder.env.GetValue('CI_BUILD_TYPE')
+if ci_type != 'host_unit_test':
+return 0
+
+shell_env = shell_environment.GetEnvironment()
+logging.log(edk2_logging.get_section_level(),
+"Run Host based Unit Tests")
+path = thebuilder.env.GetValue("BUILD_OUTPUT_BASE")
+
+failure_count = 0
+
+for arch in thebuilder.env.GetValue("TARGET_ARCH").split():
+logging.log(edk2_logging.get_subsection_level(),
+"Testing for architecture: " + arch)
+cp = os.path.join(path, arch)
+
+# If any old results XML files exist, clean them up.
+for old_result in glob.iglob(os.path.join(cp, "*.result.xml")):
+os.remove(old_result)
+
+# Determine whether any tests exist.
+testList = gl

Re: [edk2-devel] [Patch 2/4] BaseTools/PcdValueCommon: Fix 64-bit host compiler error

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Kinney, Michael D  
Sent: Friday, February 7, 2020 7:07 AM
To: devel@edk2.groups.io
Cc: Sean Brogan ; Feng, Bob C 
; Gao, Liming 
Subject: [Patch 2/4] BaseTools/PcdValueCommon: Fix 64-bit host compiler error

From: Sean Brogan 

https://bugzilla.tianocore.org/show_bug.cgi?id=2496

Cc: Sean Brogan 
Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Michael D Kinney 
---
 BaseTools/Source/C/Common/PcdValueCommon.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/C/Common/PcdValueCommon.c 
b/BaseTools/Source/C/Common/PcdValueCommon.c
index 42e3581707..98023e8786 100644
--- a/BaseTools/Source/C/Common/PcdValueCommon.c
+++ b/BaseTools/Source/C/Common/PcdValueCommon.c
@@ -478,7 +478,7 @@ Returns:
   //
   // Read all of the file contents.
   //
-  BytesRead = fread (*FileBuffer, sizeof (UINT8), *FileSize, InputFile);
+  BytesRead = (UINT32)fread (*FileBuffer, sizeof (UINT8), *FileSize, 
InputFile);
   if (BytesRead != *FileSize * sizeof (UINT8)) {
 fprintf (stderr, "Error reading the input file %s\n", InputFileName);
 fclose (InputFile);
-- 
2.21.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53985): https://edk2.groups.io/g/devel/message/53985
Mute This Topic: https://groups.io/mt/71036785/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [edk2-platform] FitGen: Support FV with the extension header

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Gao, Liming  
Sent: Wednesday, February 5, 2020 10:32 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C 
Subject: [edk2-platform] FitGen: Support FV with the extension header

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2086

To find Microcode FILE in FV image, need to skip FV header and FV extension 
header, then find the first RAW FFS file.

Cc: Bob Feng 
Signed-off-by: Liming Gao 
---
 Silicon/Intel/Tools/FitGen/FitGen.c | 40 +++--
 Silicon/Intel/Tools/FitGen/FitGen.h |  2 +-
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/Silicon/Intel/Tools/FitGen/FitGen.c 
b/Silicon/Intel/Tools/FitGen/FitGen.c
index 9f1db32a15..5f5638dc8b 100644
--- a/Silicon/Intel/Tools/FitGen/FitGen.c
+++ b/Silicon/Intel/Tools/FitGen/FitGen.c
@@ -747,6 +747,37 @@ CheckOverlap (
   }
 }
 
+UINT8 *
+GetMicrocodeBufferFromFv (
+  EFI_FIRMWARE_VOLUME_HEADER *FvHeader
+  )
+{
+  UINT8 *MicrocodeBuffer;
+  EFI_FFS_FILE_HEADER *FfsHeader;
+
+  MicrocodeBuffer = NULL;
+  //
+  // Skip FV header + FV extension header + FFS header  //  FfsHeader = 
+ (EFI_FFS_FILE_HEADER *)((UINT8 *) FvHeader + FvHeader->HeaderLength);
+ while ((UINT8 *) FfsHeader < (UINT8 *) FvHeader + FvHeader->FvLength) {
+if (FfsHeader->Type == EFI_FV_FILETYPE_RAW) {
+  //
+  // Find the first RAW ffs file as Microcode Buffer
+  //
+  MicrocodeBuffer = (UINT8 *)(FfsHeader + 1);
+  break;
+}
+if (GetFfsFileLength (FfsHeader) == 0xFF) {
+  // spare space is found, and exit
+  break;
+}
+FfsHeader = (EFI_FFS_FILE_HEADER *) ((UINT8 *) FfsHeader + 
+ ((GetFfsFileLength (FfsHeader)+7)&~7));  }
+
+  return MicrocodeBuffer;
+}
+
 UINT32
 GetFitEntryNumber (
   IN INTN   argc,
@@ -1047,8 +1078,7 @@ Returns:
 
 FvHeader = (EFI_FIRMWARE_VOLUME_HEADER *)MicrocodeFileBuffer;
 if (FvHeader->Signature == EFI_FVH_SIGNATURE) {
-  // Skip FV header + FFS header
-  MicrocodeBuffer = MicrocodeFileBuffer + 
sizeof(EFI_FIRMWARE_VOLUME_HEADER) + sizeof(EFI_FV_BLOCK_MAP_ENTRY) + 
sizeof(EFI_FFS_FILE_HEADER);
+  MicrocodeBuffer = GetMicrocodeBufferFromFv (FvHeader);
 } else {
   MicrocodeBuffer = MicrocodeFileBuffer;
 }
@@ -1388,8 +1418,7 @@ Returns:
 
 FvHeader = (EFI_FIRMWARE_VOLUME_HEADER *)MicrocodeFileBuffer;
 if (FvHeader->Signature == EFI_FVH_SIGNATURE) {
-  // Skip FV header + FFS header
-  MicrocodeBuffer = MicrocodeFileBuffer + 
sizeof(EFI_FIRMWARE_VOLUME_HEADER) + sizeof(EFI_FV_BLOCK_MAP_ENTRY) + 
sizeof(EFI_FFS_FILE_HEADER);
+  MicrocodeBuffer = GetMicrocodeBufferFromFv (FvHeader);
 } else {
   MicrocodeBuffer = MicrocodeFileBuffer;
 }
@@ -1401,8 +1430,7 @@ Returns:
 
 FvHeader = (EFI_FIRMWARE_VOLUME_HEADER *)MicrocodeFileBuffer;
 if (FvHeader->Signature == EFI_FVH_SIGNATURE) {
-  // Skip FV header + FFS header
-  MicrocodeBuffer = MicrocodeFileBuffer + 
sizeof(EFI_FIRMWARE_VOLUME_HEADER) + sizeof(EFI_FV_BLOCK_MAP_ENTRY) + 
sizeof(EFI_FFS_FILE_HEADER);
+  MicrocodeBuffer = GetMicrocodeBufferFromFv (FvHeader);
 } else {
   MicrocodeBuffer = MicrocodeFileBuffer;
 }
diff --git a/Silicon/Intel/Tools/FitGen/FitGen.h 
b/Silicon/Intel/Tools/FitGen/FitGen.h
index ecb5822d32..4d0a2dc6f8 100644
--- a/Silicon/Intel/Tools/FitGen/FitGen.h
+++ b/Silicon/Intel/Tools/FitGen/FitGen.h
@@ -31,7 +31,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent  // Utility 
version information  //  #define UTILITY_MAJOR_VERSION 0 -#define 
UTILITY_MINOR_VERSION 57
+#define UTILITY_MINOR_VERSION 58
 #define UTILITY_DATE  __DATE__
 
 //
--
2.13.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53988): https://edk2.groups.io/g/devel/message/53988
Mute This Topic: https://groups.io/mt/70996333/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 4/4] BaseTools/WindowsVsToolChain: Clean up Python source formatting

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Kinney, Michael D 
Sent: Friday, February 7, 2020 7:07 AM
To: devel@edk2.groups.io
Cc: Sean Brogan ; Feng, Bob C 
; Gao, Liming 
Subject: [Patch 4/4] BaseTools/WindowsVsToolChain: Clean up Python source 
formatting

From: Sean Brogan 

https://bugzilla.tianocore.org/show_bug.cgi?id=2495

Cc: Sean Brogan 
Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Michael D Kinney 
---
 .../WindowsVsToolChain/WindowsVsToolChain.py  | 45 +++
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py 
b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
index e3c4cc94a3..c9279e1c75 100644
--- a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
+++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
@@ -1,4 +1,4 @@
-## @file WindowsVsToolChain.py
+# @file WindowsVsToolChain.py
 # Plugin to configures paths for the VS2017 and VS2019 tool chain  ##  # This 
plugin works in conjuncture with the tools_def @@ -14,6 +14,7 @@ from 
edk2toollib.windows.locate_tools import FindWithVsWhere  from 
edk2toolext.environment import shell_environment  from edk2toolext.environment 
import version_aggregator
 
+
 class WindowsVsToolChain(IUefiBuildPlugin):
 
 def do_post_build(self, thebuilder):
@@ -28,9 +29,9 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 #
 # VS2017 - Follow VS2017 where there is potential for many versions of 
the tools.
 # If a specific version is required then the user must set both env 
variables:
-## VS150INSTALLPATH:  base install path on system to VC install dir.  
Here you will find the VC folder, etc
-## VS150TOOLVER:  version number for the VC compiler tools
-## VS2017_PREFIX: path to MSVC compiler folder with trailing slash 
(can be used instead of two vars above)
+# VS150INSTALLPATH:  base install path on system to VC install dir.  
Here you will find the VC folder, etc
+# VS150TOOLVER:  version number for the VC compiler tools
+# VS2017_PREFIX: path to MSVC compiler folder with trailing slash 
(can be used instead of two vars above)
 if thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS2017":
 
 # check to see if full path already configured @@ -38,11 +39,13 @@ 
class WindowsVsToolChain(IUefiBuildPlugin):
 self.Logger.info("VS2017_PREFIX is already set.")
 
 else:
-install_path = self._get_vs_install_path("VS2017".lower(), 
"VS150INSTALLPATH")
+install_path = self._get_vs_install_path(
+"VS2017".lower(), "VS150INSTALLPATH")
 vc_ver = self._get_vc_version(install_path, "VS150TOOLVER")
 
 if install_path is None or vc_ver is None:
-self.Logger.error("Failed to configure environment for 
VS2017")
+self.Logger.error(
+"Failed to configure environment for VS2017")
 return -1
 
 version_aggregator.GetVersionAggregator().ReportVersion(
@@ -50,8 +53,9 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 version_aggregator.GetVersionAggregator().ReportVersion(
 "VC Version", vc_ver, version_aggregator.VersionTypes.TOOL)
 
-#make VS2017_PREFIX to align with tools_def.txt
-prefix = os.path.join(install_path, "VC", "Tools", "MSVC", 
vc_ver)
+# make VS2017_PREFIX to align with tools_def.txt
+prefix = os.path.join(install_path, "VC",
+  "Tools", "MSVC", vc_ver)
 prefix = prefix + os.path.sep
 
shell_environment.GetEnvironment().set_shell_var("VS2017_PREFIX", prefix)
 
@@ -73,9 +77,9 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 #
 # VS2019 - Follow VS2019 where there is potential for many versions of 
the tools.
 # If a specific version is required then the user must set both env 
variables:
-## VS160INSTALLPATH:  base install path on system to VC install dir.  
Here you will find the VC folder, etc
-## VS160TOOLVER:  version number for the VC compiler tools
-## VS2019_PREFIX: path to MSVC compiler folder with trailing slash 
(can be used instead of two vars above)
+# VS160INSTALLPATH:  base install path on system to VC install dir.  
Here you will find the VC folder, etc
+# VS160TOOLVER:  version number for the VC compiler tools
+# VS2019_PREFIX: path to MSVC compiler folder with trailing slash 
(can be used instead of two vars above)
 elif thebuilder.env.GetValue("TOOL_CHAIN_TAG") == "VS20

Re: [edk2-devel] [Patch 3/4] BaseTools/WindowsVsToolChain: Setup VS2017/VS2019 env

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Kinney, Michael D 
Sent: Friday, February 7, 2020 7:07 AM
To: devel@edk2.groups.io
Cc: Sean Brogan ; Feng, Bob C 
; Gao, Liming 
Subject: [Patch 3/4] BaseTools/WindowsVsToolChain: Setup VS2017/VS2019 env

From: Sean Brogan 

https://bugzilla.tianocore.org/show_bug.cgi?id=2495

Update the WindowsVsToolChain plugin to setup the VS2017 or VS2019 development 
environment.  This is required to build BaseTools and Structured PCD host 
applications.

Cc: Sean Brogan 
Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Michael D Kinney 
---
 .../WindowsVsToolChain/WindowsVsToolChain.py  | 23 +++
 1 file changed, 23 insertions(+)

diff --git a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py 
b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
index a8202e5992..e3c4cc94a3 100644
--- a/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
+++ b/BaseTools/Plugin/WindowsVsToolChain/WindowsVsToolChain.py
@@ -21,6 +21,9 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 
 def do_pre_build(self, thebuilder):
 self.Logger = logging.getLogger("WindowsVsToolChain")
+interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB", "LIBPATH", 
"UniversalCRTSdkDir",
+"UCRTVersion", "WindowsLibPath", 
"WindowsSdkBinPath", "WindowsSdkDir", "WindowsSdkVerBinPath",
+"WindowsSDKVersion", "VCToolsInstallDir", 
+ "Path"]
 
 #
 # VS2017 - Follow VS2017 where there is potential for many versions of 
the tools.
@@ -52,6 +55,16 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 prefix = prefix + os.path.sep
 
shell_environment.GetEnvironment().set_shell_var("VS2017_PREFIX", prefix)
 
+shell_env = shell_environment.GetEnvironment()
+# Use the tools lib to determine the correct values for the 
vars that interest us.
+vs_vars = locate_tools.QueryVcVariables(
+interesting_keys, "amd64", vs_version="vs2017")
+for (k, v) in vs_vars.items():
+if k.upper() == "PATH":
+shell_env.insert_path(v)
+else:
+shell_env.set_shell_var(k, v)
+
 # now confirm it exists
 if not 
os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2017_PREFIX")):
 self.Logger.error("Path for VS2017 toolchain is invalid") @@ 
-87,6 +100,16 @@ class WindowsVsToolChain(IUefiBuildPlugin):
 prefix = prefix + os.path.sep
 
shell_environment.GetEnvironment().set_shell_var("VS2019_PREFIX", prefix)
 
+shell_env = shell_environment.GetEnvironment()
+# Use the tools lib to determine the correct values for the 
vars that interest us.
+vs_vars = locate_tools.QueryVcVariables(
+interesting_keys, "amd64", vs_version="vs2019")
+for (k, v) in vs_vars.items():
+if k.upper() == "PATH":
+shell_env.insert_path(v)
+else:
+shell_env.set_shell_var(k, v)
+
 # now confirm it exists
 if not 
os.path.exists(shell_environment.GetEnvironment().get_shell_var("VS2019_PREFIX")):
 self.Logger.error("Path for VS2019 toolchain is invalid")
--
2.21.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#53991): https://edk2.groups.io/g/devel/message/53991
Mute This Topic: https://groups.io/mt/71036786/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/4] BaseTools/Build: Do not use Common.lib in Structured PCD app

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Kinney, Michael D 
Sent: Friday, February 7, 2020 7:07 AM
To: devel@edk2.groups.io
Cc: Sean Brogan ; Feng, Bob C 
; Gao, Liming 
Subject: [Patch 1/4] BaseTools/Build: Do not use Common.lib in Structured PCD 
app

https://bugzilla.tianocore.org/show_bug.cgi?id=2496

Reduce the build and env dependencies for the Structured PCD application by 
removing the dependency on Common.lib that is only built when BaseTools is 
built which does not happen if pre-compiled BaseToools are used.  Change the 
makefile for the Structure PCD application to build all files from sources 
which adds PcdValueCommon.c to the makefile.  Also remove PcdValueCommon.c from 
Common.lib.

With the change to the makefile for the Structured PCD application, multiple C 
files are compiled.  Only PcdValueInit.c contains the extra information 
expected by the error/warning message parser.  Only parse the DSC line number 
into an error message if there is an error/warning in PcdValueInit.c.  
Errors/warnings in other files should be passed through.  This fixes a build 
failure with no useful log information that was observed when there was a 
compiler error in PcdValueCommon.c.

Cc: Sean Brogan 
Cc: Bob Feng 
Cc: Liming Gao 
Signed-off-by: Michael D Kinney 
---
 BaseTools/Source/C/Common/GNUmakefile |  3 +-
 BaseTools/Source/C/Common/Makefile|  3 +-
 .../Source/Python/Workspace/DscBuildData.py   | 42 ++-
 3 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/BaseTools/Source/C/Common/GNUmakefile 
b/BaseTools/Source/C/Common/GNUmakefile
index d09111bd64..b3eef7460f 100644
--- a/BaseTools/Source/C/Common/GNUmakefile
+++ b/BaseTools/Source/C/Common/GNUmakefile
@@ -28,7 +28,6 @@ OBJECTS = \
   PeCoffLoaderEx.o \
   SimpleFileParsing.o \
   StringFuncs.o \
-  TianoCompress.o \
-  PcdValueCommon.o
+  TianoCompress.o
 
 include $(MAKEROOT)/Makefiles/lib.makefile
diff --git a/BaseTools/Source/C/Common/Makefile 
b/BaseTools/Source/C/Common/Makefile
index ff35540dc8..ec61e45c81 100644
--- a/BaseTools/Source/C/Common/Makefile
+++ b/BaseTools/Source/C/Common/Makefile
@@ -28,8 +28,7 @@ OBJECTS = \
   PeCoffLoaderEx.obj \
   SimpleFileParsing.obj \
   StringFuncs.obj \
-  TianoCompress.obj \
-  PcdValueCommon.obj
+  TianoCompress.obj
 
 !INCLUDE ..\Makefiles\ms.lib
 
diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py 
b/BaseTools/Source/Python/Workspace/DscBuildData.py
index be6688dc75..476c7edaf9 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -91,9 +91,6 @@ WindowsCFLAGS = 'CFLAGS = $(CFLAGS) /wd4200 /wd4034 /wd4101 '
 LinuxCFLAGS = 'BUILD_CFLAGS += -Wno-pointer-to-int-cast -Wno-unused-variable '
 PcdMakefileEnd = '''
 !INCLUDE $(BASE_TOOLS_PATH)\Source\C\Makefiles\ms.common
-
-LIBS = $(LIB_PATH)\Common.lib
-
 !INCLUDE $(BASE_TOOLS_PATH)\Source\C\Makefiles\ms.app
 '''
 
@@ -2637,10 +2634,10 @@ class DscBuildData(PlatformBuildClassObject):
 
 MakeApp = PcdMakefileHeader
 if sys.platform == "win32":
-MakeApp = MakeApp + 'APPFILE = %s\%s.exe\n' % (self.OutputPath, 
PcdValueInitName) + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = 
%s\%s.obj\n' % (self.OutputPath, PcdValueInitName) + 'INC = '
+MakeApp = MakeApp + 'APPFILE = %s\%s.exe\n' % (self.OutputPath, 
PcdValueInitName) + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = 
%s\%s.obj %s.obj\n' % (self.OutputPath, PcdValueInitName, 
os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], 
"Source/C/Common/PcdValueCommon"))) + 'INC = '
 else:
 MakeApp = MakeApp + PcdGccMakefile
-MakeApp = MakeApp + 'APPFILE = %s/%s\n' % (self.OutputPath, 
PcdValueInitName) + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = 
%s/%s.o\n' % (self.OutputPath, PcdValueInitName) + \
+MakeApp = MakeApp + 'APPFILE = %s/%s\n' % (self.OutputPath, 
+ PcdValueInitName) + 'APPNAME = %s\n' % (PcdValueInitName) + 'OBJECTS = 
+ %s/%s.o %s.o\n' % (self.OutputPath, PcdValueInitName, 
+ os.path.normpath(mws.join(GlobalData.gGlobalDefines["EDK_TOOLS_PATH"], 
+ "Source/C/Common/PcdValueCommon"))) + \
   'include $(MAKEROOT)/Makefiles/app.makefile\n' + 
'INCLUDE +='
 
 IncSearchList = []
@@ -2739,8 +2736,8 @@ class DscBuildData(PlatformBuildClassObject):
 
IncludeFileFullPaths.append(os.path.normpath(includefullpath))
 break
 SearchPathList = []
-SearchPathList.append(os.path.normpath(mws.join(GlobalData.gWorkspace, 
"BaseTools/Source/C/Include")))
-SearchPathList.append(os.path.normpath(mws.join(G

Re: [edk2-devel] [PATCH v3 2/2] BaseTools: Remove caret in NASM_INC macro

[Bob Feng]

Hi Pierre,

This patch cause UefiCpuPkg build fail.
build -p UefiCpuPkg\UefiCpuPkg.dsc -a IA32 -t VS2015x86

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Thursday, February 6, 2020 11:52 PM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; Feng, Bob C 
; Gao, Liming ; 
sami.muja...@arm.com; pierre.gond...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v3 2/2] BaseTools: Remove caret in NASM_INC macro

From: Pierre Gondois 

NASM_INC contains the list of directory to include when using the nasm 
assembler.

In nmake makefiles, a trailing backslash escapes the newline char and replaces 
it with a space ' '. To have a literal trailing backslash, it must be escaped 
with a caret '^'. This is not necessary for GNU makefiles.

On windows platforms, for the NASM_INC macro, a caret escaping a trailing a 
backslash was appended to the last included folder regardless of the makefile 
type.
For instance, "/Include/" was replaced by "/Include/^\".

This is causing a build failure on windows platforms using GNU makefiles since 
the caret '^' doesn't escape any chars in GNU makefiles and is thus conserved.
"/Include^\" was replaced by "/Include\/" in nmake makefiles, but remained 
"/Include/^\" in GNU makefiles.

Escaping the trailing backslash in nmake makefiles on windows platforms is not 
needed because:
 * folder names don't require to end with a slash '/' or
   a backslash '\'.
 * a trailing backslash replaces the newline char by a space ' ',
   which is not difference from a newline char in macros.

Signed-off-by: Pierre Gondois 
---

 BaseTools/Source/Python/AutoGen/GenMake.py | 13 ++---
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 
b036f726db144c13e98b9e0fb021cb3855ec8e55..b2b27d43fe18e89ebdfdf8456f439c52759a4d9e
 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -608,19 +608,10 @@ cleanlib:
 IncludePathList = []
 asmsource = [item for item in MyAgo.SourceFileList if 
item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]
 if asmsource:
-for P in  MyAgo.IncludePathList:
+for P in MyAgo.IncludePathList:
 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, 
self.Macros)
-if IncludePath.endswith(os.sep):
-IncludePath = IncludePath.rstrip(os.sep)
-# When compiling .nasm files, need to add a literal backslash 
at each path
-# To specify a literal backslash at the end of the line, 
precede it with a caret (^)
-if P == MyAgo.IncludePathList[-1] and os.sep == '\\':
-IncludePath = ''.join([IncludePath, '^', os.sep])
-else:
-IncludePath = os.path.join(IncludePath, '')
-IncludePathList.append(IncludePath)
+IncludePathList.append(os.path.join(IncludePath, ''))
 
FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": 
"NASM_INC", "source_file": IncludePathList}))
-
 # Generate macros used to represent files containing list of input 
files
 for ListFileMacro in self.ListFileMacros:
 ListFileName = os.path.join(MyAgo.OutputDir, "%s.lst" % 
ListFileMacro.lower()[:len(ListFileMacro) - 5])
--
'Guid(CE165669-3EF3-493F-B85D-6190EE5B9759)'





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54004): https://edk2.groups.io/g/devel/message/54004
Mute This Topic: https://groups.io/mt/71025456/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fixed a Incremental build issue

[Bob Feng]

From: Bob Feng 

The .map file is not update to FFS_OUTPUT_DIR folder
in the incremental build.

Signed-off-by: Guo Dong 
Signed-off-by: Bob Feng 
Cc: Liming Gao 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index ba199c1aa73d..9ae09c47caf8 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -718,11 +718,11 @@ cleanlib:
 Src = self.ReplaceMacro(Src)
 Dst = self.ReplaceMacro(Dst)
 if Dst not in self.ResultFileList:
 self.ResultFileList.append(Dst)
 if '%s :' %(Dst) not in self.BuildTargetList:
-self.BuildTargetList.append("%s :" %(Dst))
+self.BuildTargetList.append("%s : %s" %(Dst,Src))
 self.BuildTargetList.append('\t' + 
self._CP_TEMPLATE_[self._FileType] %{'Src': Src, 'Dst': Dst})
 
 FfsCmdList = Cmd[0]
 for index, Str in enumerate(FfsCmdList):
 if '-o' == Str:
-- 
2.18.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54029): https://edk2.groups.io/g/devel/message/54029
Mute This Topic: https://groups.io/mt/71047523/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1] BaseTools: Fixed a Incremental build issue

[Bob Feng]

The .map file is not update to FFS_OUTPUT_DIR folder
in the incremental build.

Signed-off-by: Guo Dong 
Signed-off-by: Bob Feng 
Cc: Liming Gao 
---
 BaseTools/Source/Python/AutoGen/GenMake.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index ba199c1aa73d..9ae09c47caf8 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -718,11 +718,11 @@ cleanlib:
 Src = self.ReplaceMacro(Src)
 Dst = self.ReplaceMacro(Dst)
 if Dst not in self.ResultFileList:
 self.ResultFileList.append(Dst)
 if '%s :' %(Dst) not in self.BuildTargetList:
-self.BuildTargetList.append("%s :" %(Dst))
+self.BuildTargetList.append("%s : %s" %(Dst,Src))
 self.BuildTargetList.append('\t' + 
self._CP_TEMPLATE_[self._FileType] %{'Src': Src, 'Dst': Dst})
 
 FfsCmdList = Cmd[0]
 for index, Str in enumerate(FfsCmdList):
 if '-o' == Str:
-- 
2.18.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54030): https://edk2.groups.io/g/devel/message/54030
Mute This Topic: https://groups.io/mt/71047523/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH v3 2/2] BaseTools: Remove caret in NASM_INC macro

[Bob Feng]

Hi Pierre,

Thanks for updating the patch. I’ll trigger Intel internal CI to test this 
patch.

BTW, Since edk2 soft-freeze is coming, do you require this patch to be 
committed in this stable tag?

Thanks,
Bob

From: PierreGondois [mailto:pierre.gond...@arm.com]
Sent: Monday, February 10, 2020 9:29 PM
To: Feng, Bob C ; devel@edk2.groups.io
Subject: Re: [edk2-devel] [PATCH v3 2/2] BaseTools: Remove caret in NASM_INC 
macro

Hello Bob,

[From Bob]
This patch cause UefiCpuPkg build fail.
build -p UefiCpuPkg\UefiCpuPkg.dsc -a IA32 -t VS2015x86

As said in the cover letter of the v4 available at the link below, I was not 
able to reproduce the build fail that you pointed out. In the v4, the only 
thing that I am modifying is a condition on the makefile type. Hopefully this 
should only impact the build on Windows platforms using GNU makefiles. Thus 
there should not be any reason not to build on Windows platform using nmake 
makefiles. Feel free to modify this patch if you see what needs to be done.
V4 of "BaseTools: Remove caret in NASM_INC": 
https://edk2.groups.io/g/devel/message/54125

Regards,
Pierre

-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54204): https://edk2.groups.io/g/devel/message/54204
Mute This Topic: https://groups.io/mt/71025456/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH v4 0/2] BaseTools: Rationalise makefile generation

[Bob Feng]

This patch set is good to me.

Reviewed-by: Bob Feng 


-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Monday, February 10, 2020 6:49 PM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; Feng, Bob C 
; Gao, Liming ; 
sami.muja...@arm.com; pierre.gond...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v4 0/2] BaseTools: Rationalise makefile generation

v4:
  Remove caret in NASM_INC macro:
  I could not reproduce the build fail Bob Feng had when issuing
  the following command:
build -p UefiCpuPkg\UefiCpuPkg.dsc -a IA32 -t VS2015x86
  Nonetheless, this v4 should only have an impact on the builds
  made on Windows platforms using nmake.

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/720_BaseTools_Rationalise_makefile_generation_v4

Pierre Gondois (2):
  BaseTools: Rationalise makefile generation
  BaseTools: Remove caret in NASM_INC macro

 BaseTools/Source/Python/AutoGen/GenMake.py | 145 +++-
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py |  34 +++--  
BaseTools/Source/Python/AutoGen/PlatformAutoGen.py |  13 +-
 BaseTools/Source/Python/build/build.py |  25 ++--
 4 files changed, 120 insertions(+), 97 deletions(-)

--
'Guid(CE165669-3EF3-493F-B85D-6190EE5B9759)'





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54249): https://edk2.groups.io/g/devel/message/54249
Mute This Topic: https://groups.io/mt/71134119/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH v4 0/2] BaseTools: Rationalise makefile generation

[Bob Feng]

Pushed. 

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Bob Feng
Sent: Wednesday, February 12, 2020 10:07 AM
To: devel@edk2.groups.io; pierre.gond...@arm.com
Cc: Gao, Liming ; sami.muja...@arm.com; n...@arm.com
Subject: Re: [edk2-devel] [PATCH v4 0/2] BaseTools: Rationalise makefile 
generation

This patch set is good to me.

Reviewed-by: Bob Feng 


-Original Message-
From: devel@edk2.groups.io  On Behalf Of PierreGondois
Sent: Monday, February 10, 2020 6:49 PM
To: devel@edk2.groups.io
Cc: Pierre Gondois ; Feng, Bob C 
; Gao, Liming ; 
sami.muja...@arm.com; pierre.gond...@arm.com; n...@arm.com
Subject: [edk2-devel] [PATCH v4 0/2] BaseTools: Rationalise makefile generation

v4:
  Remove caret in NASM_INC macro:
  I could not reproduce the build fail Bob Feng had when issuing
  the following command:
build -p UefiCpuPkg\UefiCpuPkg.dsc -a IA32 -t VS2015x86
  Nonetheless, this v4 should only have an impact on the builds
  made on Windows platforms using nmake.

The changes can be seen at 
https://github.com/PierreARM/edk2/tree/720_BaseTools_Rationalise_makefile_generation_v4

Pierre Gondois (2):
  BaseTools: Rationalise makefile generation
  BaseTools: Remove caret in NASM_INC macro

 BaseTools/Source/Python/AutoGen/GenMake.py | 145 +++-
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py |  34 +++--  
BaseTools/Source/Python/AutoGen/PlatformAutoGen.py |  13 +-
 BaseTools/Source/Python/build/build.py |  25 ++--
 4 files changed, 120 insertions(+), 97 deletions(-)

--
'Guid(CE165669-3EF3-493F-B85D-6190EE5B9759)'








-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54251): https://edk2.groups.io/g/devel/message/54251
Mute This Topic: https://groups.io/mt/71134119/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch] BaseTools: Fixed build failure when using python38

[Bob Feng]

Reviewed-by: Bob C Feng 

-Original Message-
From: Fan, ZhijuX  
Sent: Wednesday, February 12, 2020 3:14 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [Patch] BaseTools: Fixed build failure when using python38

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2304

SyntaxWarning: "is" with a literal. Did you mean "=="?
Using "is" instead of "==" is an irregular syntax

Signed-off-by: Zhiju.Fan 
Cc: Bob C Feng 
Cc: Liming Gao 
---
 BaseTools/Source/Python/build/BuildReport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/build/BuildReport.py 
b/BaseTools/Source/Python/build/BuildReport.py
index 880459d367..8efa869162 100644
--- a/BaseTools/Source/Python/build/BuildReport.py
+++ b/BaseTools/Source/Python/build/BuildReport.py
@@ -2042,7 +2042,7 @@ class FdReport(object):
 self.VPDBaseAddress = 0
 self.VPDSize = 0
 for index, FdRegion in enumerate(Fd.RegionList):
-if str(FdRegion.RegionType) is 'FILE' and Wa.Platform.VpdToolGuid 
in str(FdRegion.RegionDataList):
+if str(FdRegion.RegionType) == 'FILE' and Wa.Platform.VpdToolGuid 
in str(FdRegion.RegionDataList):
 self.VPDBaseAddress = self.FdRegionList[index].BaseAddress
 self.VPDSize = self.FdRegionList[index].Size
 break
-- 
2.18.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54273): https://edk2.groups.io/g/devel/message/54273
Mute This Topic: https://groups.io/mt/71206432/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH V2] BaseTools:fix Ecc tool issue for check StructPcd

[Bob Feng]

Reviewed-by: Bob Feng < bob.c.f...@intel.com >

-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54274): https://edk2.groups.io/g/devel/message/54274
Mute This Topic: https://groups.io/mt/69769091/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/1] BaseTools:build failure in CLANGPDB tool chain

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Fan, ZhijuX
Sent: Wednesday, February 12, 2020 5:59 PM
To: devel@edk2.groups.io
Cc: Fan, ZhijuX ; Gao, Liming ; 
Feng, Bob C 
Subject: [edk2-devel] [Patch 1/1] BaseTools:build failure in CLANGPDB tool chain

REF:https://bugzilla.tianocore.org/show_bug.cgi?id=2520

Incremental build failure in CLANGPDB tool chain on Windows host The build 
failure is like below when do incremental build.
The root cause is in generated deps_target file. It has one line ":".

Signed-off-by: Zhiju.Fan 

Cc: Liming Gao 
Cc: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index ca9e02d19b4a..0a6314266f45 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -111,6 +111,8 @@ class IncludesAutoGen():
 continue
 dependency_file = item.strip(" \\\n")
 dependency_file = dependency_file.strip('''"''')
+if dependency_file == '':
+continue
 if os.path.normpath(dependency_file +".deps") == abspath:
 continue
 filename = os.path.basename(dependency_file).strip()
--
2.18.0.windows.1





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54329): https://edk2.groups.io/g/devel/message/54329
Mute This Topic: https://groups.io/mt/71208485/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [edk2-platform][patch] FitGen: Add FIT spec revision in FitGen tool help message

[Bob Feng]

Reviewed-by:  Bob Feng 

-Original Message-
From: Gao, Liming  
Sent: Friday, February 14, 2020 3:51 PM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Lohr, Paul A 
Subject: [edk2-platform][patch] FitGen: Add FIT spec revision in FitGen tool 
help message

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2483

Signed-off-by: Liming Gao 
Cc: Bob Feng 
Cc: Paul Lohr 
---
 Silicon/Intel/Tools/FitGen/FitGen.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Silicon/Intel/Tools/FitGen/FitGen.c 
b/Silicon/Intel/Tools/FitGen/FitGen.c
index 8122c10ebb..2ed636093a 100644
--- a/Silicon/Intel/Tools/FitGen/FitGen.c
+++ b/Silicon/Intel/Tools/FitGen/FitGen.c
@@ -294,7 +294,7 @@ Returns:
 --*/
 {
   printf (
-"%s - Tiano IA32/X64 FIT table generation Utility."" Version %i.%i\n\n",
+"%s - Tiano IA32/X64 FIT table generation Utility for FIT spec revision 
1.1."" Version %i.%i\n\n",
 UTILITY_NAME,
 UTILITY_MAJOR_VERSION,
 UTILITY_MINOR_VERSION
-- 
2.13.0.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54430): https://edk2.groups.io/g/devel/message/54430
Mute This Topic: https://groups.io/mt/71265173/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/1][edk2-stable202002]BaseTools: Fixed a regression issue in Makefile for incremental build

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2563

This patch is to fix a increametal build regression bug 
which happen when using nmake. That's introduced by 818283de3f6d.

If there is white space before !INCLUDE instruction, nmake will not
process it. Source code's dependent header files are listed in
${deps_file} file, if it's not included successfully, nmake will
not detect the change of those header file.

This patch has been verified in Windows with VS2015 and Linux with GCC5.
The header file add/modify/delete can trig the incremental build with this fix.
There is no impact on the clean build.

Cc: Andrew Fish 
Cc: Laszlo Ersek 
Cc: Leif Lindholm 
Cc: Michael D Kinney 
Cc: Pierre Gondois 
Signed-off-by: Bob Feng 
Reviewed-by: Liming Gao 
Tested-by: Liming Gao 
---
 .../Source/Python/AutoGen/IncludesAutoGen.py | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index 0a6314266f45..720d93395aaf 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -50,21 +50,21 @@ class IncludesAutoGen():
 MakePath = self.module_autogen.BuildOption.get('MAKE', {}).get('PATH')
 if not MakePath:
 EdkLogger.error("build", PARAMETER_MISSING, Message="No Make path 
available.")
 elif "nmake" in MakePath:
 _INCLUDE_DEPS_TEMPLATE = TemplateString('''
-${BEGIN}
-!IF EXIST(${deps_file})
-!INCLUDE ${deps_file}
-!ENDIF
-${END}
+${BEGIN}
+!IF EXIST(${deps_file})
+!INCLUDE ${deps_file}
+!ENDIF
+${END}
''')
 else:
 _INCLUDE_DEPS_TEMPLATE = TemplateString('''
-${BEGIN}
--include ${deps_file}
-${END}
+${BEGIN}
+-include ${deps_file}
+${END}
''')
 
 try:
 deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
 except Exception as e:
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#54992): https://edk2.groups.io/g/devel/message/54992
Mute This Topic: https://groups.io/mt/71583896/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch] BaseTools: Remove invalid leading space before !INCLUDE in Makefile

[Bob Feng]

REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2563

This patch is to fix a incremental build regression bug
which happen when using nmake. That's introduced by 818283de3f6d.

If there is white space before !INCLUDE instruction, nmake will not
process it. Source code's dependent header files are listed in
${deps_file} file, if it's not included successfully, nmake will
not detect the change of those header file.

This patch has been verified in Windows with VS2015 and Linux with GCC5.
The header file add/modify/delete can trig the incremental build with this fix.
There is no impact on the clean build.

Cc: Andrew Fish 
Cc: Laszlo Ersek 
Cc: Leif Lindholm 
Cc: Michael D Kinney 
Cc: Pierre Gondois 
Signed-off-by: Bob Feng 
Acked-by: Laszlo Ersek 
---
V2: 
1. update the subject
2. remove the Reviewed-by and tested-by
 .../Source/Python/AutoGen/IncludesAutoGen.py | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
index 0a6314266f..720d93395a 100644
--- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -50,21 +50,21 @@ class IncludesAutoGen():
 MakePath = self.module_autogen.BuildOption.get('MAKE', {}).get('PATH')
 if not MakePath:
 EdkLogger.error("build", PARAMETER_MISSING, Message="No Make path 
available.")
 elif "nmake" in MakePath:
 _INCLUDE_DEPS_TEMPLATE = TemplateString('''
-${BEGIN}
-!IF EXIST(${deps_file})
-!INCLUDE ${deps_file}
-!ENDIF
-${END}
+${BEGIN}
+!IF EXIST(${deps_file})
+!INCLUDE ${deps_file}
+!ENDIF
+${END}
''')
 else:
 _INCLUDE_DEPS_TEMPLATE = TemplateString('''
-${BEGIN}
--include ${deps_file}
-${END}
+${BEGIN}
+-include ${deps_file}
+${END}
''')
 
 try:
 deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
 except Exception as e:
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#55003): https://edk2.groups.io/g/devel/message/55003
Mute This Topic: https://groups.io/mt/71587619/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch 1/1][edk2-stable202002]BaseTools: Fixed a regression issue in Makefile for incremental build

[Bob Feng]

Hi Laszlo,

Thanks for your comments. I add comments inline.

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io  On Behalf Of Laszlo Ersek
Sent: Thursday, February 27, 2020 8:39 PM
To: Feng, Bob C ; Andrew Fish ; Leif 
Lindholm ; Kinney, Michael D ; 
Gao, Liming 
Cc: devel@edk2.groups.io; Pierre Gondois 
Subject: Re: [edk2-devel] [Patch 1/1][edk2-stable202002]BaseTools: Fixed a 
regression issue in Makefile for incremental build

On 02/27/20 10:47, Bob Feng wrote:
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2563
> 
> This patch is to fix a increametal build regression bug which happen 
> when using nmake. That's introduced by 818283de3f6d.
> 
> If there is white space before !INCLUDE instruction, nmake will not 
> process it. Source code's dependent header files are listed in 
> ${deps_file} file, if it's not included successfully, nmake will not 
> detect the change of those header file.
> 
> This patch has been verified in Windows with VS2015 and Linux with GCC5.
> The header file add/modify/delete can trig the incremental build with this 
> fix.
> There is no impact on the clean build.
> 
> Cc: Andrew Fish 
> Cc: Laszlo Ersek 
> Cc: Leif Lindholm 
> Cc: Michael D Kinney 
> Cc: Pierre Gondois 
> Signed-off-by: Bob Feng 
> Reviewed-by: Liming Gao 
> Tested-by: Liming Gao 
> ---
>  .../Source/Python/AutoGen/IncludesAutoGen.py | 16 
>  1 file changed, 8 insertions(+), 8 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
> b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> index 0a6314266f45..720d93395aaf 100644
> --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> @@ -50,21 +50,21 @@ class IncludesAutoGen():
>  MakePath = self.module_autogen.BuildOption.get('MAKE', 
> {}).get('PATH')
>  if not MakePath:
>  EdkLogger.error("build", PARAMETER_MISSING, Message="No Make 
> path available.")
>  elif "nmake" in MakePath:
>  _INCLUDE_DEPS_TEMPLATE = TemplateString('''
> -${BEGIN}
> -!IF EXIST(${deps_file})
> -!INCLUDE ${deps_file}
> -!ENDIF
> -${END}
> +${BEGIN}
> +!IF EXIST(${deps_file})
> +!INCLUDE ${deps_file}
> +!ENDIF
> +${END}
> ''')
>  else:
>  _INCLUDE_DEPS_TEMPLATE = TemplateString('''
> -${BEGIN}
> --include ${deps_file}
> -${END}
> +${BEGIN}
> +-include ${deps_file}
> +${END}
> ''')
>
>  try:
>  deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
>  except Exception as e:
> 

(1) I agree this should go into edk2-stable202002.

Acked-by: Laszlo Ersek 

(2) Andrew, Leif, Mike: should we extend the hard freeze by a few days?

(3) Bob, I'm sad that proper Bugzilla practices are not being followed.

- Commit 818283de3f6d never referenced TianoCore#2481. If the submitter forgets 
about adding such a reference, then it's the reviewer's / maintainer's 
responsibility to point it out!
[Bob] I agree reviewer/maintainer should remind the submitter to add BZ. 

- In the same vein, when commit 818283de3f6d was pushed, TianoCore#2481 was not 
closed. I've had to associate the BZ with the commit now (first:
I had to figure out the right BZ), and then close the BZ.
[Bob] I agree the BZ status should be update in time. I don't think BZ status 
update is the reviewer's/maintainer's responsibility, the BZ owner should be 
responsible for it.

NOTE: GitHub.com Pull Requests would not help *at all* in the face of such 
sloppiness; even on GitHub.com, people have to at least *name* issue numbers in 
commit messages.

- TianoCore#2563 (which tracks the regression) identifies *neither* the BZ for 
which the regression was introduced (2481), *nor* the faulty commit 
(818283de3f6d). You realize it's *completely useless* to file BZs with such 
negligence, right? It has no more information than "stuff broke, we need to fix 
it" -- but ain't that the general state of things, at all times? Are you only 
trying to fill a BZ quota?
[Bob] I don't agree this comments. 
I added the bug reproduce steps in BZ description. I think it's enough when I 
submit a new BZ.  I'll append the root cause and solution ( would be just patch 
review link) in its comments when I update the BZ status later. 
We found this critical bug in this afternoon (PRC time) and root cause and 
created patch very quickly. I don't think that I did not update the BZ in time 
is process violation.
I think the necessary information w

Re: [edk2-devel] [Patch 1/1][edk2-stable202002]BaseTools: Fixed a regression issue in Makefile for incremental build

[Bob Feng]

Laszlo, I agree BZ status should be update in time. You do a very good 
practices about that. I need to learn from you. And thank you that you have 
done a lot on BZ maintenance for me and other assignees. 

I think we need have a document to record these good practices on BZ 
maintenance so that people can clear to know what information should be 
published on BZ in each development stage.

Thanks,
Bob

-Original Message-
From: Laszlo Ersek [mailto:ler...@redhat.com] 
Sent: Friday, February 28, 2020 1:18 AM
To: Feng, Bob C ; devel@edk2.groups.io; Andrew Fish 
; Leif Lindholm ; Kinney, Michael D 
; Gao, Liming 
Cc: Pierre Gondois 
Subject: Re: [edk2-devel] [Patch 1/1][edk2-stable202002]BaseTools: Fixed a 
regression issue in Makefile for incremental build

On 02/27/20 16:53, Feng, Bob C wrote:

> [Bob] I agree the BZ status should be update in time. I don't think BZ status 
> update is the reviewer's/maintainer's responsibility, the BZ owner should be 
> responsible for it.

Agreed.

> 
> NOTE: GitHub.com Pull Requests would not help *at all* in the face of such 
> sloppiness; even on GitHub.com, people have to at least *name* issue numbers 
> in commit messages.
> 
> - TianoCore#2563 (which tracks the regression) identifies *neither* the BZ 
> for which the regression was introduced (2481), *nor* the faulty commit 
> (818283de3f6d). You realize it's *completely useless* to file BZs with such 
> negligence, right? It has no more information than "stuff broke, we need to 
> fix it" -- but ain't that the general state of things, at all times? Are you 
> only trying to fill a BZ quota?
> [Bob] I don't agree this comments. 
> I added the bug reproduce steps in BZ description. I think it's enough when I 
> submit a new BZ.  I'll append the root cause and solution ( would be just 
> patch review link) in its comments when I update the BZ status later. 

Yes, the patch explains the issue well. If the link had been in the BZ, I 
wouldn't have complained (as much).

> We found this critical bug in this afternoon (PRC time) and root cause and 
> created patch very quickly. I don't think that I did not update the BZ in 
> time is process violation.

It was not clear that you ever intended to add the link to the BZ.

> I think the necessary information was provided when the patch send out. The 
> bug description and reproduce steps are in BZ, root cause is in the patch 
> commit message, the solution is the patch itself, test result is in the 
> commit message.

Yes. There was no link from the BZ to the patch however. And it wasn't possible 
to determine whether you were going to add the link later. I tend to add the 
link immediately after posting, so I don't forget. My experience tell me that 
most patch submitters that don't add the link at once forget for good.

Yes, it was a generalization, sorry about that.

One thing I do have to admit (because I brought up GitHub.com before) is that, 
on GitHub.com, if you submit a pull request, and at least one of the commit 
messages references an Issue (like your commit message here references 
TianoCore#2563), then the issue automatically gets an update.
So in that regard GitHub.com does save some manual work.

Laszlo


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#55043): https://edk2.groups.io/g/devel/message/55043
Mute This Topic: https://groups.io/mt/71583896/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch] BaseTools: Remove invalid leading space before !INCLUDE in Makefile

[Bob Feng]

This patch was pushed.
https://github.com/tianocore/edk2/commit/2be4828af1c92a848af90429a9a0b44544c80553

Thanks,
Bob

-Original Message-
From: Gao, Liming 
Sent: Thursday, February 27, 2020 10:57 PM
To: devel@edk2.groups.io; Feng, Bob C 
Cc: Andrew Fish ; Laszlo Ersek ; Leif 
Lindholm ; Kinney, Michael D ; 
Pierre Gondois 
Subject: RE: [edk2-devel] [Patch] BaseTools: Remove invalid leading space 
before !INCLUDE in Makefile

Reviewed-by: Liming Gao 
Tested-by: Liming Gao  -Original Message-
> From: devel@edk2.groups.io  On Behalf Of Bob 
> Feng
> Sent: Thursday, February 27, 2020 10:25 PM
> To: devel@edk2.groups.io
> Cc: Andrew Fish ; Laszlo Ersek ; 
> Leif Lindholm ; Kinney, Michael D 
> ; Pierre Gondois 
> Subject: [edk2-devel] [Patch] BaseTools: Remove invalid leading space 
> before !INCLUDE in Makefile
> 
> REF: https://bugzilla.tianocore.org/show_bug.cgi?id=2563
> 
> This patch is to fix a incremental build regression bug which happen 
> when using nmake. That's introduced by 818283de3f6d.
> 
> If there is white space before !INCLUDE instruction, nmake will not 
> process it. Source code's dependent header files are listed in 
> ${deps_file} file, if it's not included successfully, nmake will not 
> detect the change of those header file.
> 
> This patch has been verified in Windows with VS2015 and Linux with GCC5.
> The header file add/modify/delete can trig the incremental build with this 
> fix.
> There is no impact on the clean build.
> 
> Cc: Andrew Fish 
> Cc: Laszlo Ersek 
> Cc: Leif Lindholm 
> Cc: Michael D Kinney 
> Cc: Pierre Gondois 
> Signed-off-by: Bob Feng 
> Acked-by: Laszlo Ersek 
> ---
> V2:
> 1. update the subject
> 2. remove the Reviewed-by and tested-by
>  .../Source/Python/AutoGen/IncludesAutoGen.py | 16 
>  1 file changed, 8 insertions(+), 8 deletions(-)
> 
> diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py 
> b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> index 0a6314266f..720d93395a 100644
> --- a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
> @@ -50,21 +50,21 @@ class IncludesAutoGen():
>  MakePath = self.module_autogen.BuildOption.get('MAKE', 
> {}).get('PATH')
>  if not MakePath:
>  EdkLogger.error("build", PARAMETER_MISSING, Message="No Make 
> path available.")
>  elif "nmake" in MakePath:
>  _INCLUDE_DEPS_TEMPLATE = TemplateString('''
> -${BEGIN}
> -!IF EXIST(${deps_file})
> -!INCLUDE ${deps_file}
> -!ENDIF
> -${END}
> +${BEGIN}
> +!IF EXIST(${deps_file})
> +!INCLUDE ${deps_file}
> +!ENDIF
> +${END}
> ''')
>  else:
>  _INCLUDE_DEPS_TEMPLATE = TemplateString('''
> -${BEGIN}
> --include ${deps_file}
> -${END}
> +${BEGIN}
> +-include ${deps_file}
> +${END}
> ''')
> 
>  try:
>  deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
>  except Exception as e:
> --
> 2.20.1.windows.1
> 
> 
> 


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#55132): https://edk2.groups.io/g/devel/message/55132
Mute This Topic: https://groups.io/mt/71587619/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH V2] BaseTools:fix regression issue for platform .map file

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Thursday, November 21, 2019 9:58 AM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH V2] BaseTools:fix regression issue for platform .map file

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2363

The line of IMAGE=*** is missing in platform .map file.For example, in 
Ovmf.map, there is no line of (IMAGE= ) under each of modules item.
This is a regression issue.

this patch is going to fix this issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/build/build.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index bcd832c525..07f1f21b5d 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -2267,6 +2267,10 @@ class Build():
 self.CreateAsBuiltInf()
 if GlobalData.gBinCacheDest:
 self.UpdateBuildCache()
+#
+# Get Module List
+#
+ModuleList = {ma.Guid.upper(): ma for ma in 
+ self.BuildModules}
 self.BuildModules = []
 self.MakeTime += int(round((time.time() - MakeContiue)))
 #
@@ -2285,10 +2289,6 @@ class Build():
 #
 if (Arch == 'IA32' or Arch == 'ARM') and 
self.LoadFixAddress != 0x and self.LoadFixAddress >= 
0x1:
 EdkLogger.error("build", PARAMETER_INVALID, 
"FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the 
platorm with IA32 or ARM arch modules")
-#
-# Get Module List
-#
-ModuleList = {ma.Guid.upper():ma for ma in 
self.BuildModules}
 
 #
 # Rebase module to the preferred memory address before 
GenFds
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#50998): https://edk2.groups.io/g/devel/message/50998
Mute This Topic: https://groups.io/mt/60997993/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:fixed Build failed issue for Non-English OS

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Wednesday, November 20, 2019 7:27 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH] BaseTools:fixed Build failed issue for Non-English OS

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2365

Build failed on Non-English OS if structurePcd is used in platform dsc file.
When the output of some functions is converted to code, Because different OS 
Character encoding form differently, there may be problems with some functions

The patch is going to fixed this issue

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/Workspace/DscBuildData.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py 
b/BaseTools/Source/Python/Workspace/DscBuildData.py
index 9192077f90..901d95a413 100644
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -1752,7 +1752,7 @@ class DscBuildData(PlatformBuildClassObject):
 except:
 EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute 
command: %s' % Command)
 Result = Process.communicate()
-return Process.returncode, Result[0].decode(), Result[1].decode()
+return Process.returncode, Result[0].decode(errors='ignore'), 
+ Result[1].decode(errors='ignore')
 
 @staticmethod
 def IntToCString(Value, ValueSize):
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#50999): https://edk2.groups.io/g/devel/message/50999
Mute This Topic: https://groups.io/mt/60828668/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH] BaseTools:fixed Build failed issue for Non-English OS

[Bob Feng]

Hi Laszlo,

I add some comments inline.

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Laszlo 
Ersek
Sent: Friday, November 22, 2019 1:53 AM
To: devel@edk2.groups.io; Fan, ZhijuX 
Cc: Gao, Liming ; Feng, Bob C ; 
Leif Lindholm 
Subject: Re: [edk2-devel] [PATCH] BaseTools:fixed Build failed issue for 
Non-English OS

(+Leif)

On 11/20/19 12:27, Fan, ZhijuX wrote:
> BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2365
>
> Build failed on Non-English OS if structurePcd is used in platform dsc 
> file.
> When the output of some functions is converted to code, Because 
> different OS Character encoding form differently, there may be 
> problems with some functions
>
> The patch is going to fixed this issue
>
> Cc: Liming Gao 
> Cc: Bob Feng 
> Signed-off-by: Zhiju.Fan 
> ---
>  BaseTools/Source/Python/Workspace/DscBuildData.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py 
> b/BaseTools/Source/Python/Workspace/DscBuildData.py
> index 9192077f90..901d95a413 100644
> --- a/BaseTools/Source/Python/Workspace/DscBuildData.py
> +++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
> @@ -1752,7 +1752,7 @@ class DscBuildData(PlatformBuildClassObject):
>  except:
>  EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute 
> command: %s' % Command)
>  Result = Process.communicate()
> -return Process.returncode, Result[0].decode(), Result[1].decode()
> +return Process.returncode, Result[0].decode(errors='ignore'), 
> + Result[1].decode(errors='ignore')
>
>  @staticmethod
>  def IntToCString(Value, ValueSize):

Let me quote the full method (pre-patch):


@staticmethod
def ExecuteCommand (Command):
try:
Process = subprocess.Popen(Command, stdout=subprocess.PIPE, 
stderr=subprocess.PIPE, shell=True)
except:
EdkLogger.error('Build', COMMAND_FAILURE, 'Can not execute command: 
%s' % Command)
Result = Process.communicate()
return Process.returncode, Result[0].decode(), Result[1].decode()

Relevant documentation:

- https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate
- https://docs.python.org/3/library/stdtypes.html#bytes.decode
- https://docs.python.org/3/library/codecs.html#error-handlers

So we launch a subprocess, read its stdout and stderr as byte arrays, and then 
decode those byte arrays (interpreted in UTF-8) to string (= sequences of 
unicode code points).

If the byte arrays turn out to be invalid (per UTF-8), we currently fail.

With the patch, we don't fail, but return garbage (or I guess "partial data" -- 
see the docs: "malformed data is ignored and encoding or decoding is continued 
without further notice").

[Bob] That's correct. I agree.

I think this patch is wrong. Here's why:

- I don't see why the symptom is inherently tied to structured PCDs.

[Bob] The build failure was found on a platform which uses Structure Pcds in 
dsc and the build machine is a Win10 with Korean language.
The present patch descripts the build failure case.

- "Non-English OS" is not precise; it comes down to the encoding of the
  byte stream that the subprocess produces. Even if the current locale
  specifies English-language messages, the problem can surface if the
  subprocess prints *symbols*, with non-UTF-8 encoding. Conversely, if
  the locale specifies non-English messages, but the subprocess prints
  them in correct UTF-8 encoding, the problem will not arise.

[Bob] I agree. "Non-English OS" is not precise. For this case, the build 
failure is because the visual studio c compiler output message includes 
localized string. 

- I think the code should attempt to decode the stdout / stderr byte
  arrays using the LC_CTYPE locale category (not fixed UTF-8), and if
  *even that* fails, we should use errors='replace' (or another
  replacement method).

[Bob] I think for this case build tool does not need to handle the non-ascii 
characters so 'Ignore' will be enough.

... Finally, please compare commit 8ddec24dea74 ("BaseTools:Updata the output 
encoding of the Popen function", 2019-08-01). At this point, I'm completely 
confused. Consider:

(a) Commit 8ddec24dea74 removed the "encoding='utf-8'" parameter --
which has no effect, as "encoding='utf-8'" is the default.

(b) It also removed "errors='ignore'". In effect, that change set
'errors="strict"'.

(c) Okay... so the change in said commit actually means "be strict about
UTF-8 decoding failures".  But then the *message* on commit
8ddec24dea7

Re: [edk2-devel] [PATCH] BaseTools:fixed Build failed issue for Non-English OS

[Bob Feng]

Hi Laszlo,

ExecuteCommand() name is so generic but this function is only used for 
Structure PCD. We may change that function name to eliminate the confusion.

Thanks,
Bob

-Original Message-
From: Laszlo Ersek [mailto:ler...@redhat.com] 
Sent: Saturday, November 23, 2019 12:47 AM
To: Feng, Bob C ; devel@edk2.groups.io; Fan, ZhijuX 

Cc: Gao, Liming ; Leif Lindholm 
Subject: Re: [edk2-devel] [PATCH] BaseTools:fixed Build failed issue for 
Non-English OS

Hi Bob,

On 11/22/19 07:39, Feng, Bob C wrote:

> [Bob] The build failure was found on a platform which uses Structure Pcds in 
> dsc and the build machine is a Win10 with Korean language.
> The present patch descripts the build failure case.

> [Bob] I agree. "Non-English OS" is not precise. For this case, the build 
> failure is because the visual studio c compiler output message includes 
> localized string. 

> [Bob] I think for this case build tool does not need to handle the non-ascii 
> characters so 'Ignore' will be enough.

> [Bob] I agree to mention the present patch is to revert part of commit 
> 8ddec24dea74, drop "non-English OS" language, but keep "structure PCD".

thanks for following up.

I'm happy if you agree to mention commit 8ddec24dea74.

Furthermore, I understand that Visual Studio can print localized strings. But, 
there are two things that remain unclear:

- Why are such messages tied to structure PCDs? Can Visual Studio *not* print 
the same kind of localized message in other cases? Like, what if you have a 
(VOID*) PCD and assign an L"..." string to it, in the platform DSC?

I mean I always encourage patch authors to include as many details about the 
failure scenario in the commit message as they feel comfortable about. But the 
current commit message suggests the issue is specific to structure PCDs *only*. 
That's the confusing part.

- You mention "I think for this case build tool does not need to handle the 
non-ascii characters so 'Ignore' will be enough." -- Sure, I guess for this 
particular case, "Ignore" could be OK -- but the method that's being modified 
bears the generic name "ExecuteCommand".

What *else* is ExecuteCommand() used for? I'm not convinced that ignoring 
decoding errors in the standard output / standard error of the subprocess is 
acceptable in *all* cases.

If there is no other use case for ExecuteCommand()'s standard output / standard 
error than to print them to the edk2 build log, then I agree "ignore" is 
tolerable. But I don't know if that's the only use case. If it is, it should be 
stated in the commit message.

Thanks
Laszlo


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51209): https://edk2.groups.io/g/devel/message/51209
Mute This Topic: https://groups.io/mt/60828668/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [PATCH V3] BaseTools:fix regression issue for platform .map file

[Bob Feng]

Reviewed-by: Bob Feng 

-Original Message-
From: Fan, ZhijuX 
Sent: Friday, November 22, 2019 6:55 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH V3] BaseTools:fix regression issue for platform .map file

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2363

This patch is to fix a build tool regression issue which was introduced by 
commit b8ac0b7f28.This issue caused map file lost the line of IMAGE=***.
For example,in Ovmf.map, there is no line of (IMAGE= ) under each 
of modules item.

The path to the efi file generated by each module is written on this line The 
purpose of this line is add the debug image full path.
there is no information about the module in the map file other than FVName, it 
allows us to quickly know which module this part corresponds to.

In commit b8ac0b7f28,add a line ("self.BuildModules = []") in function, but 
it's used to calculate the variable ModuleList in the following code.

Cc: Liming Gao 
Cc: Bob Feng 
Signed-off-by: Zhiju.Fan 
---
 BaseTools/Source/Python/build/build.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/BaseTools/Source/Python/build/build.py 
b/BaseTools/Source/Python/build/build.py
index bcd832c525..07f1f21b5d 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -2267,6 +2267,10 @@ class Build():
 self.CreateAsBuiltInf()
 if GlobalData.gBinCacheDest:
 self.UpdateBuildCache()
+#
+# Get Module List
+#
+ModuleList = {ma.Guid.upper(): ma for ma in 
+ self.BuildModules}
 self.BuildModules = []
 self.MakeTime += int(round((time.time() - MakeContiue)))
 #
@@ -2285,10 +2289,6 @@ class Build():
 #
 if (Arch == 'IA32' or Arch == 'ARM') and 
self.LoadFixAddress != 0x and self.LoadFixAddress >= 
0x1:
 EdkLogger.error("build", PARAMETER_INVALID, 
"FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the 
platorm with IA32 or ARM arch modules")
-#
-# Get Module List
-#
-ModuleList = {ma.Guid.upper():ma for ma in 
self.BuildModules}
 
 #
 # Rebase module to the preferred memory address before 
GenFds
--
2.14.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51212): https://edk2.groups.io/g/devel/message/51212
Mute This Topic: https://groups.io/mt/61631498/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch] Document: Update DSC spec to add Packages section

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2383

Update DSC spec to add Packages section.

The purpose of this section is to involve the packages,
which are not included by any component, into the platform.
In this way, more FeatureFlagPcd or FixedAtBuildPcd
can be used in platform DSC file or FDF file,
even if those PCDs are not used in any component.

Signed-off-by: Bob Feng 
Cc: Liming Gao 
---
 .../213_[packages]_section_processing.md  | 70 ++
 .../313_[packages]_section.md | 93 +++
 README.md |  1 +
 SUMMARY.md|  2 +
 4 files changed, 166 insertions(+)
 create mode 100644 2_dsc_overview/213_[packages]_section_processing.md
 create mode 100644 3_edk_ii_dsc_file_format/313_[packages]_section.md

diff --git a/2_dsc_overview/213_[packages]_section_processing.md 
b/2_dsc_overview/213_[packages]_section_processing.md
new file mode 100644
index 000..7a73031
--- /dev/null
+++ b/2_dsc_overview/213_[packages]_section_processing.md
@@ -0,0 +1,70 @@
+
+
+## 2.13 [Packages] Section Processing
+
+The `[Packages]` section lists the EDK II declaration files. The purpose of 
this
+section is to involve the packages, which are not included by any component, 
into
+the platform. In this way, more FeatureFlagPcd or FixedAtBuildPcd can be used 
in
+platform DSC file or FDF file, even if those PCDs are not used in any 
component.
+
+Packages listed in architectural sections must not be listed in common
+`[Packages]` sections. The architectural section modifier is used as a
+restriction to mask items from architectures that are not applicable. The
+locations of the packages listed in this section will be used for build tool to
+get the PCD declaration. The packages must be listed
+in the order that resolves any include dependencies.
+
+This section uses one of the following section definitions:
+
+```ini
+[Packages]
+[Packages.common]
+[Packages.IA32]
+[Packages.X64]
+[Packages.EBC]
+```
+
+The path must include the DEC file name and the name of the directory that
+contains the DEC file.
+
+```
+MdeModulePkg/MdeModulePkg.dec  # MdeModulePkg
+MdePkg/MdePkg.dec  # MdePkg
+```
+
+The following is an example of a packages section:
+
+```ini
+[Packages]
+  MdeModulePkg/MdeModulePkg.dec
+  MdePkg/MdePkg.dec
+```
diff --git a/3_edk_ii_dsc_file_format/313_[packages]_section.md 
b/3_edk_ii_dsc_file_format/313_[packages]_section.md
new file mode 100644
index 000..0594048
--- /dev/null
+++ b/3_edk_ii_dsc_file_format/313_[packages]_section.md
@@ -0,0 +1,93 @@
+
+
+## 3.13 [Packages] Section
+
+These are optional sections. The purpose of this section is to involve the 
packages,
+which are not included by any component, into the platform. In this way, more
+FeatureFlagPcd or FixedAtBuildPcd can be used in platform DSC file or FDF file,
+even if those PCDs are not used in any component.
+
+ Summary
+
+Defines the `[Packages]` section tag that is used in EDK II platform DSC files.
+Each entry in this section contains a directory name, forward slash character
+and the name of the DEC file contained in the directory name.
+
+Each package filename must be listed only once per section. Package filenames
+listed in architectural sections are not permitted to be listed in the common
+architectural section.
+
+The `"common"` architecture modifier in a section tag must not be combined with
+other architecture type; doing so will result in a build break.
+
+Packages listed under the `"common"` architecture section must not be listed in
+sections that have other architecture modifiers.
+
+ Prototype
+
+```c
+   ::= "[Packages" [] "]"  *
+  ::= {".common"} {}
+  ::=  [","  "Packages" ]
+  ::= "." 
+ ::= {} {}
+  ::=   [] 
+ ::=  
+ ::= 
+```
+
+ Parameters
+
+**_Filename_**
+
+Paths listed in the `[Packages]` section contain a directory name, forward
+slash character and the name of the DEC file contained in the directory name.
+Use of "..", "." and "../" in the directory path is not permitted. Use of an
+absolute path is prohibited.
+
+**_FeatureFlagExpress_**
+
+When present, the feature flag expression determines whether the entry line is
+valid. If the feature flag expression evaluates to FALSE, this entry will be
+ignored by the EDK II build tools.
+
+ Example
+
+```ini
+[Packages]
+  MdePkg/MdePkg.dec
+  MdeModulePkg/MdeModulePkg.dec
+
+[Packages.IA32]
+  DEFINE CPUS = IA32FamilyCpuPkg
+  $(CPUS)/DualCore/DualCore.dec
+```
diff --git a/README.md b/README.md
index 42febaf..4ec9105 100644
--- a/README.md
+++ b/README.md
@@ -194,5 +194,6 @@ Copyright (c) 2006-

Re: [edk2-devel] Patch List for 201911 stable tag

[Bob Feng]

Patch "fix regression issue for platform .map file" was pushed.

More comments inline.

Thanks,
Bob
-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Laszlo 
Ersek
Sent: Friday, November 22, 2019 1:26 AM
To: Leif Lindholm ; Gao, Liming 
Cc: devel@edk2.groups.io; Kinney, Michael D ; 
'af...@apple.com' 
Subject: Re: [edk2-devel] Patch List for 201911 stable tag

On 11/21/19 11:37, Leif Lindholm wrote:
> On Thu, Nov 21, 2019 at 08:57:12AM +, Gao, Liming wrote:
>> Hi Stewards and all:
>>   New bugs are for 201911 stable tag. Can you give the comments for them?
>>
>> Bug List (those all have pass code review):
>> https://edk2.groups.io/g/devel/message/50931 [PATCH] MdeModulePkg: 
>> LzmaCustomDecompressLib.inf don't support EBC anymore
> 
> This can wait until after the stable tag *unless* that causes serious 
> issues for CI. (And I don't consider non-zero build failures for EBC 
> to be serious.) The fix only addresses building the component 
> standalone as part of MdeModulePkg, so it affects no real platforms.

I would have beem more permissive with this patch, but I'm OK with the above 
resolution too.

> 
> I have some comments with regards to the implementation as well, but 
> I'll give those on the actual patch email.
> 
>> https://edk2.groups.io/g/devel/message/50990 [PATCH V1 1/1] 
>> MdeModulePkg/Variable: Initialize local variable
> 
> Those two bugs are poster children for why long functions should be 
> avoided.
> 
> OK for me to include in stable tag.

Agreed.

> 
>> https://edk2.groups.io/g/devel/message/50989 [PATCH V2] BaseTools:fix 
>> regression issue for platform .map file
> 
> OK for me to include *if* the commit message (and bugzilla) are 
> updated to clarify the issue. "is missing" is passive language, and 
> hence fails to convey what is causing the problem. Does this mean that 
> the 'build' command fails to generate these lines?
> 
> Also, a comment on why this change (of moving this hunk 18 lines up) 
> resolves the issue should be included.

Agreed; please add a reminder to the commit message (even on push) about what 
the specific (missing) lines in the map files are good for in the first place.

[Bob] The commit message was updated 
https://edk2.groups.io/g/devel/message/51212
And the patch was pushed.

> 
>> https://edk2.groups.io/g/devel/message/50936 [PATCH] BaseTools:fixed 
>> Build failed issue for Non-English OS
> 
> I agree that a fix for this bug must be included in the stable tag.
> But both the BZ and the commit message need improving.
> And seeing a fix that consists solely of adding "errors='ignore'"
> worries me somewhat.
> 
> Please explain *what* goes wrong when using Non-English OS (and which 
> OS), as well as why "errors='ignore'" does not cause further problems.

I think this patch is plain wrong. I'll comment on it in its own thread.

Thanks
Laszlo





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51214): https://edk2.groups.io/g/devel/message/51214
Mute This Topic: https://groups.io/mt/60556595/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch] GenBiosId: Enable GenBiosId to set timestamp as zero

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2384

We need eliminate the effect of timestamp to verify
the reproducible build.

This patch is to add the support for GenBiosId to set
timestamp as zero.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 Platform/Intel/Tools/GenBiosId/GenBiosId.py | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/Platform/Intel/Tools/GenBiosId/GenBiosId.py 
b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
index 31abb24d31..1b25621104 100644
--- a/Platform/Intel/Tools/GenBiosId/GenBiosId.py
+++ b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
@@ -100,26 +100,28 @@ def MyOptionParser():
 parser.add_argument('-v', '--version', action='version', 
version=__version__,
 help="show program's version number and exit")
 parser.add_argument('-i', '--int', metavar='FILENAME', dest='InputFile', 
help="Input Config file")
 parser.add_argument('-o', '--out', metavar='FILENAME', dest='OutputFile', 
help="Output file")
 parser.add_argument('-ot', '--text', metavar='FILENAME', 
dest='OutputTextFile', help="Output Text file")
+parser.add_argument('-NT', '--NoTimeStamp', dest='NoTimestamp', 
action='store_true', default=False, help="Set timestamp to zero")
 Options = parser.parse_args()
 return Options
 
 
 # Check the Tool for missing variables
 def CheckOptions(Options):
-if len(sys.argv) != 5 and not (len(sys.argv) == 7 and 
Options.OutputTextFile):
+if len(sys.argv) not in [5,6] and not (len(sys.argv) not in [7,8] and 
Options.OutputTextFile):
 EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
 elif not Options.InputFile or not Options.OutputFile:
 EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
 InputFile = Options.InputFile
 OutputFile = Options.OutputFile
 OutputTextFile = Options.OutputTextFile
+NoTimestamp = Options.NoTimestamp
 if not os.path.exists(InputFile):
 EdkLogger("GenBiosId", FILE_NOT_FOUND, ExtraData="Input file not 
found")
-return InputFile, OutputFile, OutputTextFile
+return InputFile, OutputFile, OutputTextFile,NoTimestamp
 
 # Read input file and get config
 def ReadInputFile(InputFile):
 InputDict = OrderedDict()
 with open(InputFile) as File:
@@ -132,23 +134,25 @@ def ReadInputFile(InputFile):
 InputDict[Key.strip()] = Value.strip()
 return InputDict
 
 
 # Parse the input file and extract the information
-def ParserInputFile(InputDict):
+def ParserInputFile(InputDict,NoTimestamp):
 for Item in InputDict:
 if Item not in _ConfigItem:
 EdkLogger("GenBiosId", FORMAT_INVALID, 
ExtraData=_ConfigItemInvalid % Item)
 _ConfigItem[Item]['Value'] = InputDict[Item]
 if len(_ConfigItem[Item]['Value']) != _ConfigItem[Item]['Length']:
 EdkLogger("GenBiosId", FORMAT_INVALID, ExtraData=_ConfigLenInvalid 
% Item)
 for Item in _ConfigItem:
 if not _ConfigItem[Item]['Value']:
 EdkLogger("GenBiosId", FORMAT_UNKNOWN_ERROR, ExtraData="Item %s is 
missing" % Item)
 utcnow = datetime.datetime.utcnow()
-TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
-
+if NoTimestamp:
+TimeStamp = "\0\0\0\0\0\0\0\0\0\0"
+else:
+TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
 Id_Str = _ConfigItem['BOARD_ID']['Value'] + 
_ConfigItem['BOARD_REV']['Value'] + '.' + _ConfigItem['BOARD_EXT'][
 'Value'] + '.' + _ConfigItem['VERSION_MAJOR']['Value'] + \
  '.' + _ConfigItem["BUILD_TYPE"]['Value'] + 
_ConfigItem['VERSION_MINOR']['Value'] + '.' + TimeStamp
 return Id_Str
 
@@ -169,13 +173,13 @@ def PrintOutputFile(OutputFile, OutputTextFile, Id_Str):
 
 
 # Tool entrance method
 def Main():
 Options = MyOptionParser()
-InputFile, OutputFile, OutputTextFile = CheckOptions(Options)
+InputFile, OutputFile, OutputTextFile,NoTimestamp = CheckOptions(Options)
 InputDict = ReadInputFile(InputFile)
-Id_Str = ParserInputFile(InputDict)
+Id_Str = ParserInputFile(InputDict,NoTimestamp)
 PrintOutputFile(OutputFile, OutputTextFile, Id_Str)
 return 0
 
 
 if __name__ == '__main__':
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51215): https://edk2.groups.io/g/devel/message/51215
Mute This Topic: https://groups.io/mt/61908239/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] [Patch] GenBiosId: Enable GenBiosId to set timestamp as zero

[Bob Feng]

I agree.  Thanks for your comments. I'll update it in patch v2.

-Original Message-
From: Philippe Mathieu-Daudé [mailto:phi...@redhat.com] 
Sent: Monday, November 25, 2019 6:56 PM
To: devel@edk2.groups.io; Feng, Bob C 
Cc: Gao, Liming ; Shi, Steven 
Subject: Re: [edk2-devel] [Patch] GenBiosId: Enable GenBiosId to set timestamp 
as zero

Hi Bob,

On 11/25/19 9:05 AM, Bob Feng via Groups.Io wrote:
> BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2384
> 
> We need eliminate the effect of timestamp to verify the reproducible 
> build.
> 
> This patch is to add the support for GenBiosId to set timestamp as 
> zero.
> 
> Cc: Liming Gao 
> Cc: Steven Shi 
> Signed-off-by: Bob Feng 
> ---
>   Platform/Intel/Tools/GenBiosId/GenBiosId.py | 18 +++---
>   1 file changed, 11 insertions(+), 7 deletions(-)
> 
> diff --git a/Platform/Intel/Tools/GenBiosId/GenBiosId.py 
> b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
> index 31abb24d31..1b25621104 100644
> --- a/Platform/Intel/Tools/GenBiosId/GenBiosId.py
> +++ b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
> @@ -100,26 +100,28 @@ def MyOptionParser():
>   parser.add_argument('-v', '--version', action='version', 
> version=__version__,
>   help="show program's version number and exit")
>   parser.add_argument('-i', '--int', metavar='FILENAME', 
> dest='InputFile', help="Input Config file")
>   parser.add_argument('-o', '--out', metavar='FILENAME', 
> dest='OutputFile', help="Output file")
>   parser.add_argument('-ot', '--text', metavar='FILENAME', 
> dest='OutputTextFile', help="Output Text file")
> +parser.add_argument('-NT', '--NoTimeStamp', dest='NoTimestamp', 
> + action='store_true', default=False, help="Set timestamp to zero")

All other options are in lowercase, why use CamelCase here? Can we use '-nt', 
'--notimestamp' to keep the style?

Two minor style comment below; except the style comments the patch looks good.

>   Options = parser.parse_args()
>   return Options
>   
>   
>   # Check the Tool for missing variables
>   def CheckOptions(Options):
> -if len(sys.argv) != 5 and not (len(sys.argv) == 7 and 
> Options.OutputTextFile):
> +if len(sys.argv) not in [5,6] and not (len(sys.argv) not in [7,8] and 
> Options.OutputTextFile):
>   EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
>   elif not Options.InputFile or not Options.OutputFile:
>   EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
>   InputFile = Options.InputFile
>   OutputFile = Options.OutputFile
>   OutputTextFile = Options.OutputTextFile
> +NoTimestamp = Options.NoTimestamp
>   if not os.path.exists(InputFile):
>   EdkLogger("GenBiosId", FILE_NOT_FOUND, ExtraData="Input file not 
> found")
> -return InputFile, OutputFile, OutputTextFile
> +return InputFile, OutputFile, OutputTextFile,NoTimestamp

Missing space before NoTimestamp.

>   
>   # Read input file and get config
>   def ReadInputFile(InputFile):
>   InputDict = OrderedDict()
>   with open(InputFile) as File:
> @@ -132,23 +134,25 @@ def ReadInputFile(InputFile):
>   InputDict[Key.strip()] = Value.strip()
>   return InputDict
>   
>   
>   # Parse the input file and extract the information -def 
> ParserInputFile(InputDict):
> +def ParserInputFile(InputDict,NoTimestamp):
>   for Item in InputDict:
>   if Item not in _ConfigItem:
>   EdkLogger("GenBiosId", FORMAT_INVALID, 
> ExtraData=_ConfigItemInvalid % Item)
>   _ConfigItem[Item]['Value'] = InputDict[Item]
>   if len(_ConfigItem[Item]['Value']) != _ConfigItem[Item]['Length']:
>   EdkLogger("GenBiosId", FORMAT_INVALID, 
> ExtraData=_ConfigLenInvalid % Item)
>   for Item in _ConfigItem:
>   if not _ConfigItem[Item]['Value']:
>   EdkLogger("GenBiosId", FORMAT_UNKNOWN_ERROR, ExtraData="Item %s 
> is missing" % Item)
>   utcnow = datetime.datetime.utcnow()
> -TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
> -
> +if NoTimestamp:
> +TimeStamp = "\0\0\0\0\0\0\0\0\0\0"
> +else:
> +TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
>   Id_Str = _ConfigItem['BOARD_ID']['Value'] + 
>

[edk2-devel] [Patch V2] GenBiosId: Enable GenBiosId to set timestamp as zero

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2384

We need eliminate the effect of timestamp to verify
the reproducible build.

This patch is to add the support for GenBiosId to set
timestamp as zero.

Cc: Liming Gao 
Cc: Steven Shi 
Cc: Philippe Mathieu-Daude 
Signed-off-by: Bob Feng 
---
V2: 
1. Change command line argument to lowercase.
2. Add white space before NoTimestamp
 Platform/Intel/Tools/GenBiosId/GenBiosId.py | 18 +++---
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/Platform/Intel/Tools/GenBiosId/GenBiosId.py 
b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
index 31abb24d31..8cecb1c76f 100644
--- a/Platform/Intel/Tools/GenBiosId/GenBiosId.py
+++ b/Platform/Intel/Tools/GenBiosId/GenBiosId.py
@@ -100,26 +100,28 @@ def MyOptionParser():
 parser.add_argument('-v', '--version', action='version', 
version=__version__,
 help="show program's version number and exit")
 parser.add_argument('-i', '--int', metavar='FILENAME', dest='InputFile', 
help="Input Config file")
 parser.add_argument('-o', '--out', metavar='FILENAME', dest='OutputFile', 
help="Output file")
 parser.add_argument('-ot', '--text', metavar='FILENAME', 
dest='OutputTextFile', help="Output Text file")
+parser.add_argument('-nt', '--notimestamp', dest='NoTimestamp', 
action='store_true', default=False, help="Set timestamp to zero")
 Options = parser.parse_args()
 return Options
 
 
 # Check the Tool for missing variables
 def CheckOptions(Options):
-if len(sys.argv) != 5 and not (len(sys.argv) == 7 and 
Options.OutputTextFile):
+if len(sys.argv) not in [5,6] and not (len(sys.argv) not in [7,8] and 
Options.OutputTextFile):
 EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
 elif not Options.InputFile or not Options.OutputFile:
 EdkLogger("GenBiosId", OPTION_MISSING, ExtraData=_Usage)
 InputFile = Options.InputFile
 OutputFile = Options.OutputFile
 OutputTextFile = Options.OutputTextFile
+NoTimestamp = Options.NoTimestamp
 if not os.path.exists(InputFile):
 EdkLogger("GenBiosId", FILE_NOT_FOUND, ExtraData="Input file not 
found")
-return InputFile, OutputFile, OutputTextFile
+return InputFile, OutputFile, OutputTextFile, NoTimestamp
 
 # Read input file and get config
 def ReadInputFile(InputFile):
 InputDict = OrderedDict()
 with open(InputFile) as File:
@@ -132,23 +134,25 @@ def ReadInputFile(InputFile):
 InputDict[Key.strip()] = Value.strip()
 return InputDict
 
 
 # Parse the input file and extract the information
-def ParserInputFile(InputDict):
+def ParserInputFile(InputDict, NoTimestamp):
 for Item in InputDict:
 if Item not in _ConfigItem:
 EdkLogger("GenBiosId", FORMAT_INVALID, 
ExtraData=_ConfigItemInvalid % Item)
 _ConfigItem[Item]['Value'] = InputDict[Item]
 if len(_ConfigItem[Item]['Value']) != _ConfigItem[Item]['Length']:
 EdkLogger("GenBiosId", FORMAT_INVALID, ExtraData=_ConfigLenInvalid 
% Item)
 for Item in _ConfigItem:
 if not _ConfigItem[Item]['Value']:
 EdkLogger("GenBiosId", FORMAT_UNKNOWN_ERROR, ExtraData="Item %s is 
missing" % Item)
 utcnow = datetime.datetime.utcnow()
-TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
-
+if NoTimestamp:
+TimeStamp = "\0\0\0\0\0\0\0\0\0\0"
+else:
+TimeStamp = time.strftime("%y%m%d%H%M", utcnow.timetuple())
 Id_Str = _ConfigItem['BOARD_ID']['Value'] + 
_ConfigItem['BOARD_REV']['Value'] + '.' + _ConfigItem['BOARD_EXT'][
 'Value'] + '.' + _ConfigItem['VERSION_MAJOR']['Value'] + \
  '.' + _ConfigItem["BUILD_TYPE"]['Value'] + 
_ConfigItem['VERSION_MINOR']['Value'] + '.' + TimeStamp
 return Id_Str
 
@@ -169,13 +173,13 @@ def PrintOutputFile(OutputFile, OutputTextFile, Id_Str):
 
 
 # Tool entrance method
 def Main():
 Options = MyOptionParser()
-InputFile, OutputFile, OutputTextFile = CheckOptions(Options)
+InputFile, OutputFile, OutputTextFile, NoTimestamp = CheckOptions(Options)
 InputDict = ReadInputFile(InputFile)
-Id_Str = ParserInputFile(InputDict)
+Id_Str = ParserInputFile(InputDict, NoTimestamp)
 PrintOutputFile(OutputFile, OutputTextFile, Id_Str)
 return 0
 
 
 if __name__ == '__main__':
-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51268): https://edk2.groups.io/g/devel/message/51268
Mute This Topic: https://groups.io/mt/61956815/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [edk2-devel] Seeking feedback: example of building OvmfPkg with Pytools

[Bob Feng]

Hi Jeremiah,

I have some questions for this change.
1. Will there be a PlatformBuild.py in any platform as your design? For example 
there will be a PlatformBuild.py in MdeModulePkg, EmulatorPkg and etc. And this 
PlatformBuild.py can be seen as a template?
2. The original build options can be set either from PlatformBuild.py 
CommonPlatform class or set from command line, is it right?
3. There is another build scripts edk2-platforms\Platform\Intel\build_bios.py 
for build Min-Platforms. Can PlatformBuild.py work together with it? 
4. Dose the pytools support clang on windows?
5. I tried this patch on my local machines, but build failed with pytools and 
your change. 
1) One windows machine only installed VS2015,  vswhere can't detect VS2015.
2) Another windows machine installed VS2015 and VS2017, vswhere can find 
VS2017, but build failed with the messages
build.py...
INFO -  : error 7000: Failed to execute command
INFO -  Vc\bin\nmake.exe

3) I have a Ubuntu 18.04 machine, build failed at the step of "stuart_update -c 
OvmfPkg/PlatformBuild.py" with the error messages:
File 
"/home/bobfeng/PythonVenv1/lib/python3.6/site-packages/edk2toolext/environment/extdeptypes/nuget_dependency.py",
 line 180, in fetch
if self._fetch_from_cache(package_name):
  File 
"/home/bobfeng/PythonVenv1/lib/python3.6/site-packages/edk2toolext/environment/extdeptypes/nuget_dependency.py",
 line 113, in _fetch_from_cache
if not os.path.isdir(NugetDependency.global_cache_path):
  File "/usr/lib/python3.6/genericpath.py", line 42, in isdir
st = os.stat(s)

Thanks,
Bob

-Original Message-
From: devel@edk2.groups.io [mailto:devel@edk2.groups.io] On Behalf Of Laszlo 
Ersek
Sent: Wednesday, November 27, 2019 5:29 PM
To: devel@edk2.groups.io; jere...@microsoft.com
Subject: Re: [edk2-devel] Seeking feedback: example of building OvmfPkg with 
Pytools

Hi Jeremiah,

On 11/27/19 01:03, Jeremiah Cox via Groups.Io wrote:
> All,
> I created an example of how to build OvmfPkg using the Pytools build 
> environment ( https://github.com/tianocore/edk2-pytool-extensions ).
> It is available for your feedback on GitHub:
> https://github.com/out0xb2/edk2/pull/3/files
> 
> Looking for your thoughts on integrating this example into EDK2.

This patch set seems to target the edk2 repository, and at least in part 
OvmfPkg. Please submit the patch set to the list for review.

Thanks
Laszlo





-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51482): https://edk2.groups.io/g/devel/message/51482
Mute This Topic: https://groups.io/mt/62059595/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 4/4 V3] BaseTools: Enhance Basetool for incremental build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py|  83 ++
 .../Source/Python/AutoGen/IncludesAutoGen.py  | 258 ++
 .../Source/Python/AutoGen/ModuleAutoGen.py|  23 ++
 BaseTools/Source/Python/build/build.py|  63 -
 4 files changed, 354 insertions(+), 73 deletions(-)
 create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 59a01a7f243a..fe94f9a4c232 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -183,10 +183,16 @@ class BuildFile(object):
 EdkLogger.error("build", PARAMETER_INVALID, "Invalid build type 
[%s]" % FileType,
 ExtraData="[%s]" % str(self._AutoGenObject))
 self._FileType = FileType
 FileContent = self._TEMPLATE_.Replace(self._TemplateDict)
 FileName = self._FILE_NAME_[FileType]
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt"),"w+") as fd:
+fd.write("")
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency"),"w+") as fd:
+fd.write("")
 return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, 
FileName), FileContent, False)
 
 ## Return a list of directory creation command string
 #
 #   @param  DirList The list of directory to be created
@@ -302,13 +308,10 @@ MAKE_FILE = ${makefile_path}
 # Build Macro
 #
 ${BEGIN}${file_macro}
 ${END}
 
-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\
-  ${END}
-
 #
 # Overridable Target Macro Definitions
 #
 FORCE_REBUILD = force_build
 INIT_TARGET = init
@@ -380,10 +383,12 @@ gen_libs:
 #
 gen_fds:
 \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds
 \t@cd $(MODULE_BUILD_DIR)
 
+${INCLUDETAG}
+
 #
 # Individual Object Build Targets
 #
 ${BEGIN}${file_build_target}
 ${END}
@@ -513,13 +518,10 @@ cleanlib:
 if Tool == "MAKE":
 continue
 # Remove duplicated include path, if any
 if Attr == "FLAGS":
 Value = RemoveDupOption(Value, IncPrefix, 
MyAgo.IncludePathList)
-if self._AutoGenObject.BuildRuleFamily == 
TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:
-Value = Value.replace(' /MP', '')
-MyAgo.BuildOption[Tool][Attr] = Value
 if Tool == "OPTROM" and PCI_COMPRESS_Flag:
 ValueList = Value.split()
 if ValueList:
 for i, v in enumerate(ValueList):
 if '-e' == v:
@@ -538,11 +540,11 @@ cleanlib:
 RespFile = os.path.join(MyAgo.OutputDir, str(Resp).lower() + 
'.txt')
 StrList = RespDict[Resp].split(' ')
 UnexpandMacro = []
 NewStr = []
 for Str in StrList:
-if '$' in Str:
+if '$' in Str or '-MMD' in Str or '-MF' in Str:
 UnexpandMacro.append(Str)
 else:
 NewStr.append(Str)
 UnexpandMacroStr = ' '.join(UnexpandMacro)
 NewRespStr = ' '.join(NewStr)
@@ -588,14 +590,13 @@ cleanlib:
 "source_file" : 
IncludePathList
 }
 )
 FileMacroList.append(FileMacro)
 # Add support when compiling .nasm source files
-for File in self.FileCache.keys():
-if not str(File).endswith('.nasm'):
-continue
-IncludePathList = []
+IncludePathList = []
+asmsource = [item for item in MyAgo.SourceFileList if 
item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]
+if asmsource:
 for P in  MyAgo.IncludePathList:
 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, 
self.Macros)
 if IncludePath.endswith

[edk2-devel] [Patch 2/4 V3] BaseTools: Generate dependent files for ASL and ASM files

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Implement the function in Trim tool to get the included
file list for ASL and ASM file.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/Trim/Trim.py | 115 ++-
 1 file changed, 95 insertions(+), 20 deletions(-)

diff --git a/BaseTools/Source/Python/Trim/Trim.py 
b/BaseTools/Source/Python/Trim/Trim.py
index 24c3fafa76f9..c5638376e41a 100644
--- a/BaseTools/Source/Python/Trim/Trim.py
+++ b/BaseTools/Source/Python/Trim/Trim.py
@@ -54,10 +54,14 @@ gLongNumberPattern = 
re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?L
 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", 
re.MULTILINE)
 ## Regular expression for matching C style #include "XXX.asl" in asl file
 gAslCIncludePattern = 
re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)
 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
 
+## Regular expression for finding header file inclusions
+gIncludePattern = re.compile(r"^[ \t]*[%]?[ \t]*include(?:[ 
\t]*(?:\\(?:\r\n|\r|\n))*[ \t]*)*(?:\(?[\"<]?[ \t]*)([-\w.\\/() \t]+)(?:[ 
\t]*[\">]?\)?)", re.MULTILINE | re.UNICODE | re.IGNORECASE)
+
+
 ## file cache to avoid circular include in ASL file
 gIncludedAslFile = []
 
 ## Trim preprocessed source code
 #
@@ -251,13 +255,14 @@ def TrimPreprocessedVfr(Source, Target):
 # @param  IncludePathList   The list of external include file
 # @param  LocalSearchPath   If LocalSearchPath is specified, this path will be 
searched
 #   first for the included file; otherwise, only the 
path specified
 #   in the IncludePathList will be searched.
 #
-def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
+def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None, 
IncludeFileList = None, filetype=None):
 NewFileContent = []
-
+if IncludeFileList is None:
+IncludeFileList = []
 try:
 #
 # Search LocalSearchPath first if it is specified.
 #
 if LocalSearchPath:
@@ -286,28 +291,41 @@ def DoInclude(Source, Indent='', IncludePathList=[], 
LocalSearchPath=None):
 if IncludeFile in gIncludedAslFile:
 EdkLogger.warn("Trim", "Circular include",
ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), 
IncludeFile))
 return []
 gIncludedAslFile.append(IncludeFile)
-
+IncludeFileList.append(IncludeFile.strip())
 for Line in F:
 LocalSearchPath = None
-Result = gAslIncludePattern.findall(Line)
-if len(Result) == 0:
-Result = gAslCIncludePattern.findall(Line)
-if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() 
not in [".asl", ".asi"]:
+if filetype == "ASL":
+Result = gAslIncludePattern.findall(Line)
+if len(Result) == 0:
+Result = gAslCIncludePattern.findall(Line)
+if len(Result) == 0 or 
os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
+NewFileContent.append("%s%s" % (Indent, Line))
+continue
+#
+# We should first search the local directory if current file 
are using pattern #include "XXX"
+#
+if Result[0][2] == '"':
+LocalSearchPath = os.path.dirname(IncludeFile)
+CurrentIndent = Indent + Result[0][0]
+IncludedFile = Result[0][1]
+NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, 
IncludePathList, LocalSearchPath,IncludeFileList,filetype))
+NewFileContent.append("\n")
+elif filetype == "ASM":
+Result = gIncludePattern.findall(Line)
+if len(Result) == 0:
 NewFileContent.append("%s%s" % (Indent, Line))
 continue
-#
-# We should first search the local directory if current file are 
using pattern #include "XXX"
-#
-if Result[0][2] == '"':
-LocalSearchPath = os.path.dirname(IncludeFile)
-CurrentIndent = Indent + Result[0][0]
-IncludedFile = Result[0][1]
-NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, 
IncludePathList, LocalSearchPath))
-NewFileContent.append("\n")
+
+IncludedFile = Result[0]
+
+IncludedFile = IncludedFile.strip()
+IncludedFile = os.path.normpath(IncludedFile)
+NewFileContent.extend(DoInclude(IncludedFile, '', IncludePathList, 
LocalS

[edk2-devel] [Patch 3/4 V3] BaseTools: Update build_rule.txt to generate dependent files.

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Enable the dependent files generation function for compilers
and Trim tool.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Conf/build_rule.template | 94 +-
 1 file changed, 52 insertions(+), 42 deletions(-)

diff --git a/BaseTools/Conf/build_rule.template 
b/BaseTools/Conf/build_rule.template
index 84d8426e7d80..04bcbb06db36 100755
--- a/BaseTools/Conf/build_rule.template
+++ b/BaseTools/Conf/build_rule.template
@@ -121,18 +121,18 @@
 
 
 $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
 
 
-"$(CC)" /Fo${dst} $(CC_FLAGS) $(INC) ${src}
+"$(CC)" /Fo${dst} $(DEPS_FLAGS) $(CC_FLAGS) $(INC) ${src}
 
 
 # For RVCTCYGWIN CC_FLAGS must be first to work around pathing issues
-"$(CC)" $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
+"$(CC)" $(DEPS_FLAGS) $(CC_FLAGS) -c -o ${dst} $(INC) ${src}
 
 
-"$(CC)" $(CC_FLAGS) -o ${dst} $(INC) ${src}
+"$(CC)" $(DEPS_FLAGS) $(CC_FLAGS) -o ${dst} $(INC) ${src}
 
 
[C-Code-File.BASE.AARCH64,C-Code-File.SEC.AARCH64,C-Code-File.PEI_CORE.AARCH64,C-Code-File.PEIM.AARCH64,C-Code-File.BASE.ARM,C-Code-File.SEC.ARM,C-Code-File.PEI_CORE.ARM,C-Code-File.PEIM.ARM]
 
 ?.c
 
@@ -165,19 +165,21 @@
 
 
 $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
 
 
-"$(PP)" $(PP_FLAGS) $(INC) ${src} > ${d_path}(+)${s_base}.i
-Trim --source-code --convert-hex --trim-long -o 
${d_path}(+)${s_base}.iii ${d_path}(+)${s_base}.i
-"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.iii
+Trim --asm-file -o ${d_path}(+)${s_base}.i -i $(INC_LIST) ${src}
+"$(PP)" $(DEPS_FLAGS) $(PP_FLAGS) $(INC) ${src} > 
${d_path}(+)${s_base}.ii
+Trim --source-code --convert-hex --trim-long -o 
${d_path}(+)${s_base}. ${d_path}(+)${s_base}.ii
+"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.
 
 
-"$(PP)" $(PP_FLAGS) $(INC) ${src} > ${d_path}(+)${s_base}.i
-Trim --trim-long --source-code -o ${d_path}(+)${s_base}.iii 
${d_path}(+)${s_base}.i
+Trim --asm-file -o ${d_path}(+)${s_base}.i -i $(INC_LIST) ${src}
+"$(PP)" $(DEPS_FLAGS) $(PP_FLAGS) $(INC) ${src} > 
${d_path}(+)${s_base}.ii
+Trim --trim-long --source-code -o ${d_path}(+)${s_base}. 
${d_path}(+)${s_base}.ii
 # For RVCTCYGWIN ASM_FLAGS must be first to work around pathing issues
-"$(ASM)" $(ASM_FLAGS) -o ${dst} $(INC) ${d_path}(+)${s_base}.iii
+"$(ASM)" $(ASM_FLAGS) -o ${dst} $(INC) ${d_path}(+)${s_base}.
 
 [Assembly-Code-File.COMMON.ARM,Assembly-Code-File.COMMON.AARCH64]
 # Remove --convert-hex for ARM as it breaks MSFT assemblers
 
 ?.asm, ?.Asm, ?.ASM
@@ -190,24 +192,27 @@
 
 
 $(OUTPUT_DIR)(+)${s_dir}(+)${s_base}.obj
 
 
-"$(PP)" $(PP_FLAGS) $(INC) ${src} > ${d_path}(+)${s_base}.i
-Trim --source-code --convert-hex --trim-long -o 
${d_path}(+)${s_base}.iii ${d_path}(+)${s_base}.i
-"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.iii
+Trim --asm-file -o ${d_path}(+)${s_base}.i -i $(INC_LIST) ${src}
+"$(PP)" $(DEPS_FLAGS) $(PP_FLAGS) $(INC) ${src} > 
${d_path}(+)${s_base}.ii
+Trim --source-code --convert-hex --trim-long -o 
${d_path}(+)${s_base}. ${d_path}(+)${s_base}.ii
+"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.
 
 
-"$(PP)" $(PP_FLAGS) $(INC) ${src} > ${d_path}(+)${s_base}.i
-Trim --source-code --trim-long -o ${d_path}(+)${s_base}.iii 
${d_path}(+)${s_base}.i
-"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.iii
+Trim --asm-file  -o ${d_path}(+)${s_base}.i -i $(INC_LIST) ${src}
+"$(PP)" $(DEPS_FLAGS) $(PP_FLAGS) $(INC) ${src} > 
${d_path}(+)${s_base}.ii
+Trim --source-code --trim-long -o ${d_path}(+)${s_base}. 
${d_path}(+)${s_base}.ii
+"$(ASM)" /Fo${dst} $(ASM_FLAGS) /I${s_path} $(INC) 
${d_path}(+)${s_base}.
 
 
-"$(PP)" $(PP_FLAGS) $(INC) ${src} > ${d_path}(+)${s_base}.i
-Trim --trim-long --source-code -o ${d_path}(+)${s_base}.iii 
${d_path}(+)${s_base}.i
+Trim --asm-file -o ${d_path}(+)${s_base}.i -i $(INC_LIST) ${src}
+"$(PP)" $(DEPS_FLAGS) $(PP_FLAGS) $(INC) ${src} > 
${d_path}(+)${s_base}.ii
+Trim --trim-long --source-code -o ${d_path}(+)${s_base}. 
${d_path}(+)${s_base}.ii
 # For RVCTCYGWIN ASM_FLAGS must be first to work around pathing issues
-"$(ASM)" $(ASM_FLAGS) -o ${dst} $(INC) ${d_path}(+)${s_b

[edk2-devel] [Patch 0/4 V3] Enhance Incremental Build

[Bob Feng]

V3: Change CLANG9 to CLANGPDB according to commit 14672c34bd 
V2: Fixed a bug in patch 4/4.

Incremental build reduces the build time by only building
the module that need to update. Edk2 Build system is a Makefile
based build system. The incrememtal build ability is provided by
the Make program. But Edk2 build tool need to generate correct makefile
to have Make program do incremental build correctly.

The current solution in build tool to support incremental build is that build
tool find out the include file list for each source file of a module, and in 
module's
makefile, build tool add the include file list as the source file's dependency. 
In this way Make program can decide if it need to rebuild a source code by 
checking
its dependency. This solution has 2 shortcommings, one is the process of finding
include list is slow, the other is this method can't handle case that 
a MACRO in #include statement so the related source file is always built. 

This patch provides another method to support incremental build. That is to use
c preprocessor and trim tool to generate dependency files for the source file.
This method will save much time in AutoGen phase and handle MACRO in #include 
correctly.

For C files:
1. MSVS.
cl.exe has a build option /showIncludes to display include files on 
stdout. Build tool captures
that messages and generate dependency files, .deps files.
2. CLANG and GCC
-MMD -MF build option are used to generate dependency files by 
preprocessor. Build tool updates the
   .deps files.

For ASL files:
1. Trim find out all the included files, which are asl specific include 
format, and generate .trim.deps file.
2. ASL PP use c preprocessor to find out all included files with #include 
format and generate a .deps file
3. build tool updates the .deps file

For ASM files (.asm, .s or .nasm):
1. Trim find out all the included files, which are asm specific include 
format, and generate .trim.deps file.
2. ASM PP use c preprocessor to find out all included files with #include 
format and generate a deps file
3. build tool updates the .deps file

Build tool add "include" instruction for those deps files in the Makefile.

This patch does not support RVCT tool chain for the BZ 
https://bugzilla.tianocore.org/show_bug.cgi?id=1750

Feng, Bob C (4):
  BaseTools: Add build option for dependency file generation
  BaseTools: Generate dependent files for ASL and ASM files
  BaseTools: Update build_rule.txt to generate dependent files.
  BaseTools: Enhance Basetool for incremental build

 BaseTools/Conf/build_rule.template|  94 ---
 BaseTools/Conf/tools_def.template | 173 ++--
 BaseTools/Source/Python/AutoGen/GenMake.py|  83 ++
 .../Source/Python/AutoGen/IncludesAutoGen.py  | 255 ++
 .../Source/Python/AutoGen/ModuleAutoGen.py|  23 ++
 BaseTools/Source/Python/Trim/Trim.py  | 115 ++--
 BaseTools/Source/Python/build/build.py|  63 -
 7 files changed, 588 insertions(+), 218 deletions(-)
 create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51575): https://edk2.groups.io/g/devel/message/51575
Mute This Topic: https://groups.io/mt/65527687/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 1/4 V3] BaseTools: Add build option for dependency file generation

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Add /showIncludes for msvc and -MMD -MF $@.deps
for GCC and CLANG

Remove /MP for msvc since /MP does not work with
/showIncludes

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
V3: Change CLANG9 to CLANGPDB according to commit 14672c34bd

 BaseTools/Conf/tools_def.template | 173 --
 1 file changed, 90 insertions(+), 83 deletions(-)

diff --git a/BaseTools/Conf/tools_def.template 
b/BaseTools/Conf/tools_def.template
index ca0b122dbb6c..8206e97054c7 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -145,10 +145,11 @@ DEFINE IASL_OUTFLAGS   = -p
 
 DEFINE DEFAULT_WIN_ASL_BIN  = DEF(WIN_IASL_BIN)
 DEFINE DEFAULT_WIN_ASL_FLAGS= DEF(IASL_FLAGS)
 DEFINE DEFAULT_WIN_ASL_OUTFLAGS = DEF(IASL_OUTFLAGS)
 
+DEFINE MSFT_DEPS_FLAGS = /showIncludes
 DEFINE MSFT_ASLPP_FLAGS= /nologo /E /C /FIAutoGen.h
 DEFINE MSFT_ASLCC_FLAGS= /nologo /c /FIAutoGen.h /TC 
/Dmain=ReferenceAcpiTable
 DEFINE MSFT_ASLDLINK_FLAGS = /NODEFAULTLIB /ENTRY:ReferenceAcpiTable 
/SUBSYSTEM:CONSOLE
 
 DEFINE IPHONE_TOOLS= 
/Developer/Platforms/iPhoneOS.platform/Developer
@@ -413,11 +414,11 @@ DEFINE DTC_BIN = ENV(DTC_PREFIX)dtc
 
 *_VS2008_*_SLINK_FLAGS= /NOLOGO /LTCG
 *_VS2008_*_APP_FLAGS  = /nologo /E /TC
 *_VS2008_*_PP_FLAGS   = /nologo /E /TC /FIAutoGen.h
 *_VS2008_*_VFRPP_FLAGS= /nologo /E /TC /DVFRCOMPILE 
/FI$(MODULE_NAME)StrDefs.h
-
+*_VS2008_*_DEPS_FLAGS= DEF(MSFT_DEPS_FLAGS)
 *_VS2008_*_ASM16_PATH = DEF(VS2008_BIN)\ml.exe
 
 ##
 # ASL definitions
 ##
@@ -476,13 +477,13 @@ NOOPT_VS2008_IA32_DLINK_FLAGS = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF
 *_VS2008_X64_DLINK_PATH= DEF(VS2008_BINX64)\link.exe
 *_VS2008_X64_ASLCC_PATH= DEF(VS2008_BINX64)\cl.exe
 *_VS2008_X64_ASLPP_PATH= DEF(VS2008_BINX64)\cl.exe
 *_VS2008_X64_ASLDLINK_PATH = DEF(VS2008_BINX64)\link.exe
 
-  DEBUG_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /MP
-RELEASE_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /MP
-NOOPT_VS2008_X64_CC_FLAGS   = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /Od /MP
+  DEBUG_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7
+RELEASE_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF
+NOOPT_VS2008_X64_CC_FLAGS   = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /Od
 
   DEBUG_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd /Zi
 RELEASE_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd
 NOOPT_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd /Zi
 
@@ -529,11 +530,11 @@ NOOPT_VS2008_X64_DLINK_FLAGS  = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF /OPT
 *_VS2008x86_*_MAKE_FLAGS  = /nologo
 *_VS2008x86_*_SLINK_FLAGS = /NOLOGO /LTCG
 *_VS2008x86_*_APP_FLAGS   = /nologo /E /TC
 *_VS2008x86_*_PP_FLAGS= /nologo /E /TC /FIAutoGen.h
 *_VS2008x86_*_VFRPP_FLAGS = /nologo /E /TC /DVFRCOMPILE 
/FI$(MODULE_NAME)StrDefs.h
-
+*_VS2008x86_*_DEPS_FLAGS  = DEF(MSFT_DEPS_FLAGS)
 *_VS2008x86_*_ASM16_PATH  = DEF(VS2008x86_BIN)\ml.exe
 
 ##
 # ASL definitions
 ##
@@ -560,13 +561,13 @@ NOOPT_VS2008_X64_DLINK_FLAGS  = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF /OPT
 *_VS2008x86_IA32_APP_PATH = DEF(VS2008x86_BIN)\cl.exe
 *_VS2008x86_IA32_PP_PATH  = DEF(VS2008x86_BIN)\cl.exe
 *_VS2008x86_IA32_ASM_PATH = DEF(VS2008x86_BIN)\ml.exe
 
   *_VS2008x86_IA32_MAKE_FLAGS  = /nologo
-  DEBUG_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /MP
-RELEASE_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /MP
-NOOPT_VS2008x86_IA32_CC_FLAGS  = /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /Od /MP
+  DEBUG_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7
+RELEASE_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF
+NOOPT_VS2008x86_IA32_CC_FLAGS  = /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /Od
 
   DEBUG_VS2008x86_IA32_ASM_FLAGS   = /nologo /c /WX /W3 /Cx /coff /Zd /Zi
 RELEASE_VS2008x86_IA32_ASM_FLAGS   = /nologo /c /WX /W3 /Cx /coff /Zd
 NOOPT_VS2008x86_IA32_ASM_FLAGS = /nologo /c /WX /W3 /Cx /coff /Zd /Zi
 
@@ -592,14 +593,13 @@ NOOPT_VS2008x86_IA32_DLINK_FLAGS

Re: [edk2-devel] [PATCH v2 0/4] BaseTools: Leverage compiler output dependency files for binary cache

[Bob Feng]

For this patch set, Reviewed-by: Bob Feng 

-Original Message-
From: Shi, Steven 
Sent: Tuesday, December 3, 2019 1:17 PM
To: devel@edk2.groups.io
Cc: Gao, Liming ; Feng, Bob C 
Subject: [PATCH v2 0/4] BaseTools: Leverage compiler output dependency files 
for binary cache

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2353

This patch is based on patch set: https://edk2.groups.io/g/devel/message/50360
You can directly try this patch set in this branch: 
https://github.com/shijunjing/edk2/tree/incrmtl_fix_v8

A previous patch set (https://edk2.groups.io/g/devel/message/50360)
enhances Incremental Build, which is to use c preprocessor and trim tool to 
generate dependency files for the source file.
This patch set is enhance the binary cache to follow the new method of 
dependency files generation through compiler, and redesign the binary cache 
intermediate meta files format.

V2:
Rebase the change on the latest edk2 to fix the patch set git apply failure.

V1:
Initial patch set

Shi, Steven (4):
  BaseTools: store more complete output files in binary cache
  BaseTools: enhance the CacheCopyFile method arg names
  BaseTools: Leverage compiler output to optimize binary cache
  BaseTools: Remove redundant binary cache file

 .../Source/Python/AutoGen/AutoGenWorker.py|   77 +-
 BaseTools/Source/Python/AutoGen/CacheIR.py|   29 -
 BaseTools/Source/Python/AutoGen/DataPipe.py   |2 +
 .../Source/Python/AutoGen/ModuleAutoGen.py| 1108 +++--
 .../Source/Python/AutoGen/WorkspaceAutoGen.py |   64 +-
 BaseTools/Source/Python/Common/GlobalData.py  |   35 +-
 BaseTools/Source/Python/build/build.py|  276 ++--
 7 files changed, 721 insertions(+), 870 deletions(-)  delete mode 100755 
BaseTools/Source/Python/AutoGen/CacheIR.py

--
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51580): https://edk2.groups.io/g/devel/message/51580
Mute This Topic: https://groups.io/mt/65521102/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [EXTERNAL] RE: [edk2-devel] Seeking feedback: example of building OvmfPkg with Pytools

[Bob Feng]

Hi Matthew and Jeremiah,

Thanks for your answers.


1.   Cool. So platformBuild.py is intent to handle the additional pre-build 
and post-build tasks. I think it would be great if there is a document record 
how to create a PlatformBuild.py.

2.   So the command-line arguments for build.py can also work for 
PlatformBuild.py. right?

3.   Where would FamilyBuild.py be located?

4.   Cool. Thanks.

For 5.2 the error message in BUILD.txt is:

PROGRESS - Running Pre Build
INFO - Cmd to run is: 
e:\pythonvenv1\lib\site-packages\edk2toollib\bin\vswhere.exe -latest -nologo 
-all -property installationPath -products *
INFO - 
INFO - --Cmd Output Starting---
INFO - 
INFO - C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional
INFO - 
INFO - --Cmd Output Finished---
INFO - - Running Time (mm:ss): 00:00 --
INFO - 
DEBUG - Calling 'C:\Program Files (x86)\Microsoft Visual 
Studio\2017\Professional\VC\Auxiliary\Build\vcvarsall.bat amd64'
DEBUG - Getting host info for host: uname_result(system='Windows', 
node='sh1gapp1005', release='10', version='10.0.14393', machine='AMD64', 
processor='Intel64 Family 6 Model 85 Stepping 4, GenuineIntel')
DEBUG - Getting host info for host: uname_result(system='Windows', 
node='sh1gapp1005', release='10', version='10.0.14393', machine='AMD64', 
processor='Intel64 Family 6 Model 85 Stepping 4, GenuineIntel')
DEBUG - Getting host info for host: uname_result(system='Windows', 
node='sh1gapp1005', release='10', version='10.0.14393', machine='AMD64', 
processor='Intel64 Family 6 Model 85 Stepping 4, GenuineIntel')
DEBUG - Plugin Success: Windows RC Path Support
DEBUG - Plugin Success: Windows Visual Studio Tool Chain Support
INFO - Writing BuildToolsReports to 
E:\BobFeng\edk2\Build\Ovmf3264\DEBUG_VS2015x86\BUILD_TOOLS_REPORT
DEBUG - Plugin Success: Build Tools Report Generator
PROGRESS - Running Build DEBUG
DEBUG - Getting all build keys for build type DEBUG
INFO - Cmd to run is: build -p OvmfPkg/OvmfPkgIa32X64.dsc -b DEBUG -t VS2015x86 
-a IA32 -a X64
INFO - 
INFO - --Cmd Output Starting---
INFO - 
INFO - Build environment: Windows-10-10.0.14393-SP0
INFO - Build start time: 14:32:35, Dec.03 2019
INFO -
INFO - WORKSPACE= e:\bobfeng\edk2
INFO - EDK_TOOLS_PATH   = e:\bobfeng\edk2\basetools
INFO - CONF_PATH= E:\BobFeng\edk2\Conf
INFO - PYTHON_COMMAND   = e:\pythonvenv1\scripts\python.exe
INFO -
INFO -
INFO - Processing meta-data .
INFO - Architecture(s)  = IA32 X64
INFO - Build target = DEBUG
INFO - Toolchain= VS2015x86
INFO -
INFO - Active Platform  = e:\bobfeng\edk2\OvmfPkg\OvmfPkgIa32X64.dsc
INFO -  done!
INFO - Building ... e:\bobfeng\edk2\MdePkg\Library\UefiLib\UefiLib.inf [X64]
INFO - The system cannot find the path specified.
INFO -
INFO -
INFO - build.py...
INFO -  : error 7000: Failed to execute command
INFO -   Vc\bin\nmake.exe /nologo tbuild 
[e:\bobfeng\edk2\Build\Ovmf3264\DEBUG_VS2015x86\X64\MdePkg\Library\UefiLib\UefiLib]
INFO -
INFO -
INFO - build.py...
INFO -  : error F002: Failed to build module
INFO -   e:\bobfeng\edk2\MdePkg\Library\UefiLib\UefiLib.inf [X64, VS2015x86, 
DEBUG]
INFO -
INFO - - Failed -
INFO - Build end time: 14:32:46, Dec.03 2019
INFO - Build total time: 00:00:11
INFO -
INFO - 
INFO - --Cmd Output Finished---
INFO - - Running Time (mm:ss): 00:11 --
INFO - 
ERROR - Compiler #7000 from :   Failed to execute command
ERROR - EDK2 #002 from :   Failed to build module
CRITICAL - Build failed
PROGRESS - End time: 2019-12-03 14:32:47.018063   Total time Elapsed: 
0:00:15
SECTION - Log file is located at: E:\BobFeng\edk2\Build\BUILDLOG_OvmfPkg.txt
SECTION - Summary
PROGRESS - Error

For 5.3, the error message in UPDATE.txt is:

INFO - Cmd to run is: mono 
/home/bobfeng/.local/lib/python3.6/site-packages/edk2toolext/bin/NuGet.exe 
locals global-packages -list
INFO - 
INFO - --Cmd Output Starting---
INFO - 
INFO - Could not load file or assembly or one of its dependencies.
INFO -   Could not load file or assembly or one of its dependencies.
INFO -   Could not load file or assembly or one of its dependencies.
INFO -   Could not load file or assembly or one of its dependencies.
INFO -   Could not load file or assembly or one of its dependencies.
INFO - System.AggregateException: One or more errors occurred. ---> 
System.IO.FileNotFoundException: Could not load file or ass

[edk2-devel] [Patch 2/4 V4] BaseTools: Generate dependent files for ASL and ASM files

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Implement the function in Trim tool to get the included
file list for ASL and ASM file.

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/Trim/Trim.py | 115 ++-
 1 file changed, 95 insertions(+), 20 deletions(-)

diff --git a/BaseTools/Source/Python/Trim/Trim.py 
b/BaseTools/Source/Python/Trim/Trim.py
index 24c3fafa76..c5638376e4 100644
--- a/BaseTools/Source/Python/Trim/Trim.py
+++ b/BaseTools/Source/Python/Trim/Trim.py
@@ -54,10 +54,14 @@ gLongNumberPattern = 
re.compile("(?<=[^a-zA-Z0-9_])(0[xX][0-9a-fA-F]+|[0-9]+)U?L
 gAslIncludePattern = re.compile("^(\s*)[iI]nclude\s*\(\"?([^\"\(\)]+)\"\)", 
re.MULTILINE)
 ## Regular expression for matching C style #include "XXX.asl" in asl file
 gAslCIncludePattern = 
re.compile(r'^(\s*)#include\s*[<"]\s*([-\\/\w.]+)\s*([>"])', re.MULTILINE)
 ## Patterns used to convert EDK conventions to EDK2 ECP conventions
 
+## Regular expression for finding header file inclusions
+gIncludePattern = re.compile(r"^[ \t]*[%]?[ \t]*include(?:[ 
\t]*(?:\\(?:\r\n|\r|\n))*[ \t]*)*(?:\(?[\"<]?[ \t]*)([-\w.\\/() \t]+)(?:[ 
\t]*[\">]?\)?)", re.MULTILINE | re.UNICODE | re.IGNORECASE)
+
+
 ## file cache to avoid circular include in ASL file
 gIncludedAslFile = []
 
 ## Trim preprocessed source code
 #
@@ -251,13 +255,14 @@ def TrimPreprocessedVfr(Source, Target):
 # @param  IncludePathList   The list of external include file
 # @param  LocalSearchPath   If LocalSearchPath is specified, this path will be 
searched
 #   first for the included file; otherwise, only the 
path specified
 #   in the IncludePathList will be searched.
 #
-def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None):
+def DoInclude(Source, Indent='', IncludePathList=[], LocalSearchPath=None, 
IncludeFileList = None, filetype=None):
 NewFileContent = []
-
+if IncludeFileList is None:
+IncludeFileList = []
 try:
 #
 # Search LocalSearchPath first if it is specified.
 #
 if LocalSearchPath:
@@ -286,28 +291,41 @@ def DoInclude(Source, Indent='', IncludePathList=[], 
LocalSearchPath=None):
 if IncludeFile in gIncludedAslFile:
 EdkLogger.warn("Trim", "Circular include",
ExtraData= "%s -> %s" % (" -> ".join(gIncludedAslFile), 
IncludeFile))
 return []
 gIncludedAslFile.append(IncludeFile)
-
+IncludeFileList.append(IncludeFile.strip())
 for Line in F:
 LocalSearchPath = None
-Result = gAslIncludePattern.findall(Line)
-if len(Result) == 0:
-Result = gAslCIncludePattern.findall(Line)
-if len(Result) == 0 or os.path.splitext(Result[0][1])[1].lower() 
not in [".asl", ".asi"]:
+if filetype == "ASL":
+Result = gAslIncludePattern.findall(Line)
+if len(Result) == 0:
+Result = gAslCIncludePattern.findall(Line)
+if len(Result) == 0 or 
os.path.splitext(Result[0][1])[1].lower() not in [".asl", ".asi"]:
+NewFileContent.append("%s%s" % (Indent, Line))
+continue
+#
+# We should first search the local directory if current file 
are using pattern #include "XXX"
+#
+if Result[0][2] == '"':
+LocalSearchPath = os.path.dirname(IncludeFile)
+CurrentIndent = Indent + Result[0][0]
+IncludedFile = Result[0][1]
+NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, 
IncludePathList, LocalSearchPath,IncludeFileList,filetype))
+NewFileContent.append("\n")
+elif filetype == "ASM":
+Result = gIncludePattern.findall(Line)
+if len(Result) == 0:
 NewFileContent.append("%s%s" % (Indent, Line))
 continue
-#
-# We should first search the local directory if current file are 
using pattern #include "XXX"
-#
-if Result[0][2] == '"':
-LocalSearchPath = os.path.dirname(IncludeFile)
-CurrentIndent = Indent + Result[0][0]
-IncludedFile = Result[0][1]
-NewFileContent.extend(DoInclude(IncludedFile, CurrentIndent, 
IncludePathList, LocalSearchPath))
-NewFileContent.append("\n")
+
+IncludedFile = Result[0]
+
+IncludedFile = IncludedFile.strip()
+IncludedFile = os.path.normpath(IncludedFile)
+NewFileContent.extend(DoInclude(IncludedFile, '', IncludePathList, 
LocalS

[edk2-devel] [Patch 0/4 V4] Enhance Incremental Build

[Bob Feng]

V4: Add $(DEPS_FLAGS) for rule [C-Code-File.BASE.AARCH64 ...]
V3: Change CLANG9 to CLANGPDB according to commit 14672c34bd 
V2: Fixed a bug in patch 4/4.

Incremental build reduces the build time by only building
the module that need to update. Edk2 Build system is a Makefile
based build system. The incrememtal build ability is provided by
the Make program. But Edk2 build tool need to generate correct makefile
to have Make program do incremental build correctly.

The current solution in build tool to support incremental build is that build
tool find out the include file list for each source file of a module, and in 
module's
makefile, build tool add the include file list as the source file's dependency. 
In this way Make program can decide if it need to rebuild a source code by 
checking
its dependency. This solution has 2 shortcommings, one is the process of finding
include list is slow, the other is this method can't handle case that 
a MACRO in #include statement so the related source file is always built. 

This patch provides another method to support incremental build. That is to use
c preprocessor and trim tool to generate dependency files for the source file.
This method will save much time in AutoGen phase and handle MACRO in #include 
correctly.

For C files:
1. MSVS.
cl.exe has a build option /showIncludes to display include files on 
stdout. Build tool captures
that messages and generate dependency files, .deps files.
2. CLANG and GCC
-MMD -MF build option are used to generate dependency files by 
preprocessor. Build tool updates the
   .deps files.

For ASL files:
1. Trim find out all the included files, which are asl specific include 
format, and generate .trim.deps file.
2. ASL PP use c preprocessor to find out all included files with #include 
format and generate a .deps file
3. build tool updates the .deps file

For ASM files (.asm, .s or .nasm):
1. Trim find out all the included files, which are asm specific include 
format, and generate .trim.deps file.
2. ASM PP use c preprocessor to find out all included files with #include 
format and generate a deps file
3. build tool updates the .deps file

Build tool add "include" instruction for those deps files in the Makefile.

This patch does not support RVCT tool chain for the BZ 
https://bugzilla.tianocore.org/show_bug.cgi?id=1750

Feng, Bob C (4):
  BaseTools: Add build option for dependency file generation
  BaseTools: Generate dependent files for ASL and ASM files
  BaseTools: Update build_rule.txt to generate dependent files.
  BaseTools: Enhance Basetool for incremental build

 BaseTools/Conf/build_rule.template|  94 ---
 BaseTools/Conf/tools_def.template | 173 ++--
 BaseTools/Source/Python/AutoGen/GenMake.py|  83 ++
 .../Source/Python/AutoGen/IncludesAutoGen.py  | 255 ++
 .../Source/Python/AutoGen/ModuleAutoGen.py|  23 ++
 BaseTools/Source/Python/Trim/Trim.py  | 115 ++--
 BaseTools/Source/Python/build/build.py|  63 -
 7 files changed, 588 insertions(+), 218 deletions(-)
 create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

-- 
2.20.1.windows.1


-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.

View/Reply Online (#51588): https://edk2.groups.io/g/devel/message/51588
Mute This Topic: https://groups.io/mt/65666140/21656
Group Owner: devel+ow...@edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub  [arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[edk2-devel] [Patch 4/4 V4] BaseTools: Enhance Basetool for incremental build

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Source/Python/AutoGen/GenMake.py|  83 ++
 .../Source/Python/AutoGen/IncludesAutoGen.py  | 258 ++
 .../Source/Python/AutoGen/ModuleAutoGen.py|  23 ++
 BaseTools/Source/Python/build/build.py|  63 -
 4 files changed, 354 insertions(+), 73 deletions(-)
 create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py 
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 59a01a7f24..fe94f9a4c2 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -183,10 +183,16 @@ class BuildFile(object):
 EdkLogger.error("build", PARAMETER_INVALID, "Invalid build type 
[%s]" % FileType,
 ExtraData="[%s]" % str(self._AutoGenObject))
 self._FileType = FileType
 FileContent = self._TEMPLATE_.Replace(self._TemplateDict)
 FileName = self._FILE_NAME_[FileType]
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"deps.txt"),"w+") as fd:
+fd.write("")
+if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency")):
+with open(os.path.join(self._AutoGenObject.MakeFileDir, 
"dependency"),"w+") as fd:
+fd.write("")
 return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, 
FileName), FileContent, False)
 
 ## Return a list of directory creation command string
 #
 #   @param  DirList The list of directory to be created
@@ -302,13 +308,10 @@ MAKE_FILE = ${makefile_path}
 # Build Macro
 #
 ${BEGIN}${file_macro}
 ${END}
 
-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\
-  ${END}
-
 #
 # Overridable Target Macro Definitions
 #
 FORCE_REBUILD = force_build
 INIT_TARGET = init
@@ -380,10 +383,12 @@ gen_libs:
 #
 gen_fds:
 \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds
 \t@cd $(MODULE_BUILD_DIR)
 
+${INCLUDETAG}
+
 #
 # Individual Object Build Targets
 #
 ${BEGIN}${file_build_target}
 ${END}
@@ -513,13 +518,10 @@ cleanlib:
 if Tool == "MAKE":
 continue
 # Remove duplicated include path, if any
 if Attr == "FLAGS":
 Value = RemoveDupOption(Value, IncPrefix, 
MyAgo.IncludePathList)
-if self._AutoGenObject.BuildRuleFamily == 
TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:
-Value = Value.replace(' /MP', '')
-MyAgo.BuildOption[Tool][Attr] = Value
 if Tool == "OPTROM" and PCI_COMPRESS_Flag:
 ValueList = Value.split()
 if ValueList:
 for i, v in enumerate(ValueList):
 if '-e' == v:
@@ -538,11 +540,11 @@ cleanlib:
 RespFile = os.path.join(MyAgo.OutputDir, str(Resp).lower() + 
'.txt')
 StrList = RespDict[Resp].split(' ')
 UnexpandMacro = []
 NewStr = []
 for Str in StrList:
-if '$' in Str:
+if '$' in Str or '-MMD' in Str or '-MF' in Str:
 UnexpandMacro.append(Str)
 else:
 NewStr.append(Str)
 UnexpandMacroStr = ' '.join(UnexpandMacro)
 NewRespStr = ' '.join(NewStr)
@@ -588,14 +590,13 @@ cleanlib:
 "source_file" : 
IncludePathList
 }
 )
 FileMacroList.append(FileMacro)
 # Add support when compiling .nasm source files
-for File in self.FileCache.keys():
-if not str(File).endswith('.nasm'):
-continue
-IncludePathList = []
+IncludePathList = []
+asmsource = [item for item in MyAgo.SourceFileList if 
item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]
+if asmsource:
 for P in  MyAgo.IncludePathList:
 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, 
self.Macros)
 if IncludePath.endswith

[edk2-devel] [Patch 1/4 V4] BaseTools: Add build option for dependency file generation

[Bob Feng]

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Add /showIncludes for msvc and -MMD -MF $@.deps
for GCC and CLANG

Remove /MP for msvc since /MP does not work with
/showIncludes

Cc: Liming Gao 
Cc: Steven Shi 
Signed-off-by: Bob Feng 
---
 BaseTools/Conf/tools_def.template | 173 --
 1 file changed, 90 insertions(+), 83 deletions(-)

diff --git a/BaseTools/Conf/tools_def.template 
b/BaseTools/Conf/tools_def.template
index ca0b122dbb..8206e97054 100755
--- a/BaseTools/Conf/tools_def.template
+++ b/BaseTools/Conf/tools_def.template
@@ -145,10 +145,11 @@ DEFINE IASL_OUTFLAGS   = -p
 
 DEFINE DEFAULT_WIN_ASL_BIN  = DEF(WIN_IASL_BIN)
 DEFINE DEFAULT_WIN_ASL_FLAGS= DEF(IASL_FLAGS)
 DEFINE DEFAULT_WIN_ASL_OUTFLAGS = DEF(IASL_OUTFLAGS)
 
+DEFINE MSFT_DEPS_FLAGS = /showIncludes
 DEFINE MSFT_ASLPP_FLAGS= /nologo /E /C /FIAutoGen.h
 DEFINE MSFT_ASLCC_FLAGS= /nologo /c /FIAutoGen.h /TC 
/Dmain=ReferenceAcpiTable
 DEFINE MSFT_ASLDLINK_FLAGS = /NODEFAULTLIB /ENTRY:ReferenceAcpiTable 
/SUBSYSTEM:CONSOLE
 
 DEFINE IPHONE_TOOLS= 
/Developer/Platforms/iPhoneOS.platform/Developer
@@ -413,11 +414,11 @@ DEFINE DTC_BIN = ENV(DTC_PREFIX)dtc
 
 *_VS2008_*_SLINK_FLAGS= /NOLOGO /LTCG
 *_VS2008_*_APP_FLAGS  = /nologo /E /TC
 *_VS2008_*_PP_FLAGS   = /nologo /E /TC /FIAutoGen.h
 *_VS2008_*_VFRPP_FLAGS= /nologo /E /TC /DVFRCOMPILE 
/FI$(MODULE_NAME)StrDefs.h
-
+*_VS2008_*_DEPS_FLAGS= DEF(MSFT_DEPS_FLAGS)
 *_VS2008_*_ASM16_PATH = DEF(VS2008_BIN)\ml.exe
 
 ##
 # ASL definitions
 ##
@@ -476,13 +477,13 @@ NOOPT_VS2008_IA32_DLINK_FLAGS = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF
 *_VS2008_X64_DLINK_PATH= DEF(VS2008_BINX64)\link.exe
 *_VS2008_X64_ASLCC_PATH= DEF(VS2008_BINX64)\cl.exe
 *_VS2008_X64_ASLPP_PATH= DEF(VS2008_BINX64)\cl.exe
 *_VS2008_X64_ASLDLINK_PATH = DEF(VS2008_BINX64)\link.exe
 
-  DEBUG_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /MP
-RELEASE_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /MP
-NOOPT_VS2008_X64_CC_FLAGS   = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /Od /MP
+  DEBUG_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7
+RELEASE_VS2008_X64_CC_FLAGS = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/O1b2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF
+NOOPT_VS2008_X64_CC_FLAGS   = /nologo /c /WX /GS- /W4 /Gs32768 /D UNICODE 
/Gy /FIAutoGen.h /EHs-c- /GR- /GF /Z7 /Od
 
   DEBUG_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd /Zi
 RELEASE_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd
 NOOPT_VS2008_X64_ASM_FLAGS= /nologo /c /WX /W3 /Cx /Zd /Zi
 
@@ -529,11 +530,11 @@ NOOPT_VS2008_X64_DLINK_FLAGS  = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF /OPT
 *_VS2008x86_*_MAKE_FLAGS  = /nologo
 *_VS2008x86_*_SLINK_FLAGS = /NOLOGO /LTCG
 *_VS2008x86_*_APP_FLAGS   = /nologo /E /TC
 *_VS2008x86_*_PP_FLAGS= /nologo /E /TC /FIAutoGen.h
 *_VS2008x86_*_VFRPP_FLAGS = /nologo /E /TC /DVFRCOMPILE 
/FI$(MODULE_NAME)StrDefs.h
-
+*_VS2008x86_*_DEPS_FLAGS  = DEF(MSFT_DEPS_FLAGS)
 *_VS2008x86_*_ASM16_PATH  = DEF(VS2008x86_BIN)\ml.exe
 
 ##
 # ASL definitions
 ##
@@ -560,13 +561,13 @@ NOOPT_VS2008_X64_DLINK_FLAGS  = /NOLOGO /NODEFAULTLIB 
/IGNORE:4001 /OPT:REF /OPT
 *_VS2008x86_IA32_APP_PATH = DEF(VS2008x86_BIN)\cl.exe
 *_VS2008x86_IA32_PP_PATH  = DEF(VS2008x86_BIN)\cl.exe
 *_VS2008x86_IA32_ASM_PATH = DEF(VS2008x86_BIN)\ml.exe
 
   *_VS2008x86_IA32_MAKE_FLAGS  = /nologo
-  DEBUG_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /MP
-RELEASE_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /MP
-NOOPT_VS2008x86_IA32_CC_FLAGS  = /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /Od /MP
+  DEBUG_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7
+RELEASE_VS2008x86_IA32_CC_FLAGS= /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /O1b2 /GL /FIAutoGen.h /EHs-c- /GR- /GF
+NOOPT_VS2008x86_IA32_CC_FLAGS  = /nologo /c /WX /GS- /W4 /Gs32768 /D 
UNICODE /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Z7 /Od
 
   DEBUG_VS2008x86_IA32_ASM_FLAGS   = /nologo /c /WX /W3 /Cx /coff /Zd /Zi
 RELEASE_VS2008x86_IA32_ASM_FLAGS   = /nologo /c /WX /W3 /Cx /coff /Zd
 NOOPT_VS2008x86_IA32_ASM_FLAGS = /nologo /c /WX /W3 /Cx /coff /Zd /Zi
 
@@ -592,14 +593,13 @@ NOOPT_VS2008x86_IA32_DLINK_FLAGS   = /NOLOGO 
/NODEFAULTLIB /IGNORE:4001 /OPT:REF