BaseTools/Source/Python/AutoGen/AutoGen.py | 161 +++++++++++++++++++++++++-- BaseTools/Source/Python/Common/GlobalData.py | 7 ++ BaseTools/Source/Python/build/build.py | 30 +++++ 3 files changed, 189 insertions(+), 9 deletions(-)
Add three new options:
--hash enables hash-based caching during build process. when --hash is
enabled, build tool will base on the module hash value to do the
incremental build, without --hash, build tool will base on the
timestamp to do the incremental build. --hash option use md5 method to
get every hash value, DSC/FDF, tools_def.txt, build_rule.txt and build
command are calculated as global hash value, Package DEC and its
include header files are calculated as package hash value, Module
source files and its INF file are calculated as module hash value.
Library hash value will combine the global hash value and its dependent
package hash value. Driver hash value will combine the global hash
value, its dependent package hash value and its linked library hash
value.
When --hash and --bindest are specified, build tool will copy the
generated binary files for each module into the directory specified by
bindest at the build phase. Bindest caches all generated binary files.
When --hash and --binsource are specified, build tool will try to get
the binary files from the binary source directory at the build phase.
If the cached binary has the same hash value, it will be directly used.
Otherwise, build tool will compile the source files and generate the
binary files.
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yonghong Zhu <yonghong.zhu@intel.com>
---
BaseTools/Source/Python/AutoGen/AutoGen.py | 161 +++++++++++++++++++++++++--
BaseTools/Source/Python/Common/GlobalData.py | 7 ++
BaseTools/Source/Python/build/build.py | 30 +++++
3 files changed, 189 insertions(+), 9 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/Python/AutoGen/AutoGen.py
index 70e2e62..246bfec 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -41,10 +41,11 @@ import Common.VpdInfoFile as VpdInfoFile
from GenPcdDb import CreatePcdDatabaseCode
from Workspace.MetaFileCommentParser import UsageList
from Common.MultipleWorkspace import MultipleWorkspace as mws
import InfSectionParser
import datetime
+import hashlib
## Regular expression for splitting Dependency Expression string into tokens
gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
#
@@ -263,10 +264,14 @@ class WorkspaceAutoGen(AutoGen):
self.FdfFile = FlashDefinitionFile
self.FdTargetList = Fds
self.FvTargetList = Fvs
self.CapTargetList = Caps
self.AutoGenObjectList = []
+ self._BuildDir = None
+ self._FvDir = None
+ self._MakeFileDir = None
+ self._BuildCommand = None
# there's many relative directory operations, so ...
os.chdir(self.WorkspaceDir)
#
@@ -642,10 +647,18 @@ class WorkspaceAutoGen(AutoGen):
#
Pa.CollectPlatformDynamicPcds()
Pa.CollectFixedAtBuildPcds()
self.AutoGenObjectList.append(Pa)
+ #
+ # Generate Package level hash value
+ #
+ GlobalData.gPackageHash[Arch] = {}
+ if GlobalData.gUseHashCache:
+ for Pkg in Pkgs:
+ self._GenPkgLevelHash(Pkg)
+
#
# Check PCDs token value conflict in each DEC file.
#
self._CheckAllPcdsTokenValueConflict()
@@ -655,15 +668,10 @@ class WorkspaceAutoGen(AutoGen):
self._CheckPcdDefineAndType()
# if self.FdfFile:
# self._CheckDuplicateInFV(Fdf)
- self._BuildDir = None
- self._FvDir = None
- self._MakeFileDir = None
- self._BuildCommand = None
-
#
# Create BuildOptions Macro & PCD metafile, also add the Active Platform and FDF file.
#
content = 'gCommandLineDefines: '
content += str(GlobalData.gCommandLineDefines)
@@ -675,10 +683,14 @@ class WorkspaceAutoGen(AutoGen):
content += str(self.Platform)
content += os.linesep
if self.FdfFile:
content += 'Flash Image Definition: '
content += str(self.FdfFile)
+ content += os.linesep
+ if GlobalData.gBinCacheDest:
+ content += 'Cache of .efi location: '
+ content += str(GlobalData.gBinCacheDest)
SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content, False)
#
# Create PcdToken Number file for Dynamic/DynamicEx Pcd.
#
@@ -704,10 +716,22 @@ class WorkspaceAutoGen(AutoGen):
for f in AllWorkSpaceMetaFiles:
if os.stat(f)[8] > SrcTimeStamp:
SrcTimeStamp = os.stat(f)[8]
self._SrcTimeStamp = SrcTimeStamp
+ if GlobalData.gUseHashCache:
+ m = hashlib.md5()
+ for files in AllWorkSpaceMetaFiles:
+ if files.endswith('.dec'):
+ continue
+ f = open(files, 'r')
+ Content = f.read()
+ f.close()
+ m.update(Content)
+ SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), True)
+ GlobalData.gPlatformHash = m.hexdigest()
+
#
# Write metafile list to build directory
#
AutoGenFilePath = os.path.join(self.BuildDir, 'AutoGen')
if os.path.exists (AutoGenFilePath):
@@ -717,10 +741,33 @@ class WorkspaceAutoGen(AutoGen):
with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
for f in AllWorkSpaceMetaFiles:
print >> file, f
return True
+ def _GenPkgLevelHash(self, Pkg):
+ PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
+ CreateDirectory(PkgDir)
+ HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')
+ m = hashlib.md5()
+ # Get .dec file's hash value
+ f = open(Pkg.MetaFile.Path, 'r')
+ Content = f.read()
+ f.close()
+ m.update(Content)
+ # Get include files hash value
+ if Pkg.Includes:
+ for inc in Pkg.Includes:
+ for Root, Dirs, Files in os.walk(str(inc)):
+ for File in Files:
+ File_Path = os.path.join(Root, File)
+ f = open(File_Path, 'r')
+ Content = f.read()
+ f.close()
+ m.update(Content)
+ SaveFileOnChange(HashFile, m.hexdigest(), True)
+ if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]:
+ GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] = m.hexdigest()
def _GetMetaFiles(self, Target, Toolchain, Arch):
AllWorkSpaceMetaFiles = set()
#
# add fdf
@@ -954,11 +1001,12 @@ class WorkspaceAutoGen(AutoGen):
self._FvDir = path.join(self.BuildDir, 'FV')
return self._FvDir
## Return the directory to store all intermediate and final files built
def _GetBuildDir(self):
- return self.AutoGenObjectList[0].BuildDir
+ if self._BuildDir == None:
+ return self.AutoGenObjectList[0].BuildDir
## Return the build output directory platform specifies
def _GetOutputDir(self):
return self.Platform.OutputDirectory
@@ -3898,37 +3946,45 @@ class ModuleAutoGen(AutoGen):
AsBuiltInfDict['module_uefi_specification_version'] += [self.Specification['UEFI_SPECIFICATION_VERSION']]
if 'PI_SPECIFICATION_VERSION' in self.Specification:
AsBuiltInfDict['module_pi_specification_version'] += [self.Specification['PI_SPECIFICATION_VERSION']]
OutputDir = self.OutputDir.replace('\\', '/').strip('/')
-
+ self.OutputFile = []
for Item in self.CodaTargetList:
File = Item.Target.Path.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/')
+ if File not in self.OutputFile:
+ self.OutputFile.append(File)
if Item.Target.Ext.lower() == '.aml':
AsBuiltInfDict['binary_item'] += ['ASL|' + File]
elif Item.Target.Ext.lower() == '.acpi':
AsBuiltInfDict['binary_item'] += ['ACPI|' + File]
elif Item.Target.Ext.lower() == '.efi':
AsBuiltInfDict['binary_item'] += ['PE32|' + self.Name + '.efi']
else:
AsBuiltInfDict['binary_item'] += ['BIN|' + File]
if self.DepexGenerated:
+ if self.Name + '.depex' not in self.OutputFile:
+ self.OutputFile.append(self.Name + '.depex')
if self.ModuleType in ['PEIM']:
AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + '.depex']
if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER', 'DXE_SAL_DRIVER', 'UEFI_DRIVER']:
AsBuiltInfDict['binary_item'] += ['DXE_DEPEX|' + self.Name + '.depex']
if self.ModuleType in ['DXE_SMM_DRIVER']:
AsBuiltInfDict['binary_item'] += ['SMM_DEPEX|' + self.Name + '.depex']
Bin = self._GenOffsetBin()
if Bin:
AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin]
+ if Bin not in self.OutputFile:
+ self.OutputFile.append(Bin)
for Root, Dirs, Files in os.walk(OutputDir):
for File in Files:
if File.lower().endswith('.pdb'):
AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File]
+ if File not in self.OutputFile:
+ self.OutputFile.append(File)
HeaderComments = self.Module.HeaderComments
StartPos = 0
for Index in range(len(HeaderComments)):
if HeaderComments[Index].find('@BinaryHeader') != -1:
HeaderComments[Index] = HeaderComments[Index].replace('@BinaryHeader', '@file')
@@ -4105,11 +4161,52 @@ class ModuleAutoGen(AutoGen):
AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict))
SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False)
self.IsAsBuiltInfCreated = True
-
+ if GlobalData.gBinCacheDest:
+ self.CopyModuleToCache()
+
+ def CopyModuleToCache(self):
+ FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, self.SourceDir, self.MetaFile.BaseName)
+ CreateDirectory (FileDir)
+ HashFile = path.join(self.BuildDir, self.Name + '.hash')
+ ModuleFile = path.join(self.OutputDir, self.Name + '.inf')
+ if os.path.exists(HashFile):
+ shutil.copy2(HashFile, FileDir)
+ if os.path.exists(ModuleFile):
+ shutil.copy2(ModuleFile, FileDir)
+ if self.OutputFile:
+ for File in self.OutputFile:
+ if not os.path.isabs(File):
+ File = os.path.join(self.OutputDir, File)
+ if os.path.exists(File):
+ shutil.copy2(File, FileDir)
+
+ def AttemptModuleCacheCopy(self):
+ if self.IsBinaryModule:
+ return False
+ FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, self.SourceDir, self.MetaFile.BaseName)
+ HashFile = path.join(FileDir, self.Name + '.hash')
+ if os.path.exists(HashFile):
+ f = open(HashFile, 'r')
+ CacheHash = f.read()
+ f.close()
+ if GlobalData.gModuleHash[self.Arch][self.Name]:
+ if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:
+ for root, dir, files in os.walk(FileDir):
+ for f in files:
+ if self.Name + '.hash' in f:
+ shutil.copy2(HashFile, self.BuildDir)
+ else:
+ File = path.join(root, f)
+ shutil.copy2(File, self.OutputDir)
+ if self.Name == "PcdPeim" or self.Name == "PcdDxe":
+ CreatePcdDatabaseCode(self, TemplateString(), TemplateString())
+ return True
+ return False
+
## Create makefile for the module and its dependent libraries
#
# @param CreateLibraryMakeFile Flag indicating if or not the makefiles of
# dependent libraries will be created
#
@@ -4233,12 +4330,58 @@ class ModuleAutoGen(AutoGen):
self._LibraryAutoGenList.append(La)
for Lib in La.CodaTargetList:
self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE)
return self._LibraryAutoGenList
+ def GenModuleHash(self):
+ if self.Arch not in GlobalData.gModuleHash:
+ GlobalData.gModuleHash[self.Arch] = {}
+ m = hashlib.md5()
+ # Add Platform level hash
+ m.update(GlobalData.gPlatformHash)
+ # Add Package level hash
+ if self.DependentPackageList:
+ for Pkg in self.DependentPackageList:
+ if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]:
+ m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName])
+
+ # Add Library hash
+ if self.LibraryAutoGenList:
+ for Lib in self.LibraryAutoGenList:
+ if Lib.Name not in GlobalData.gModuleHash[self.Arch]:
+ Lib.GenModuleHash()
+ m.update(GlobalData.gModuleHash[self.Arch][Lib.Name])
+
+ # Add Module self
+ f = open(str(self.MetaFile), 'r')
+ Content = f.read()
+ f.close()
+ m.update(Content)
+ # Add Module's source files
+ if self.SourceFileList:
+ for File in self.SourceFileList:
+ f = open(str(File), 'r')
+ Content = f.read()
+ f.close()
+ m.update(Content)
+
+ ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")
+ if self.Name not in GlobalData.gModuleHash[self.Arch]:
+ GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()
+ if GlobalData.gBinCacheSource:
+ CacheValid = self.AttemptModuleCacheCopy()
+ if CacheValid:
+ return False
+ return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True)
+
+ ## Decide whether we can skip the ModuleAutoGen process
+ def CanSkipbyHash(self):
+ if GlobalData.gUseHashCache:
+ return not self.GenModuleHash()
+
## Decide whether we can skip the ModuleAutoGen process
- # If any source file is newer than the modeule than we cannot skip
+ # If any source file is newer than the module than we cannot skip
#
def CanSkip(self):
if not os.path.exists(self.GetTimeStampPath()):
return False
#last creation time of the module
diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py
index 667877e..45e7ea0 100644
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -85,5 +85,12 @@ BuildOptionPcd = []
#
MixedPcd = {}
# Pcd name for the Pcd which used in the Conditional directives
gConditionalPcds = []
+
+gUseHashCache = None
+gBinCacheDest = None
+gBinCacheSource = None
+gPlatformHash = None
+gPackageHash = {}
+gModuleHash = {}
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 7436453..bb34b87 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -764,10 +764,34 @@ class Build():
self.TargetTxt = TargetTxtClassObject()
self.ToolDef = ToolDefClassObject()
GlobalData.BuildOptionPcd = BuildOptions.OptionPcd
#Set global flag for build mode
GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
+ GlobalData.gUseHashCache = BuildOptions.UseHashCache
+ GlobalData.gBinCacheDest = BuildOptions.BinCacheDest
+ GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
+
+ if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--bindest must be used together with --hashcache.")
+
+ if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--binsource must be used together with --hashcache.")
+
+ if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--bindest can not be used together with --binsource.")
+
+ if GlobalData.gBinCacheSource:
+ BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
+ if not os.path.isabs(BinCacheSource):
+ BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
+ GlobalData.gBinCacheSource = BinCacheSource
+
+ if GlobalData.gBinCacheDest:
+ BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
+ if not os.path.isabs(BinCacheDest):
+ BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
+ GlobalData.gBinCacheDest = BinCacheDest
if self.ConfDirectory:
# Get alternate Conf location, if it is absolute, then just use the absolute directory name
ConfDirectoryPath = os.path.normpath(self.ConfDirectory)
@@ -1908,10 +1932,13 @@ class Build():
# Get ModuleAutoGen object to generate C code file and makefile
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile)
if Ma == None:
continue
+ if Ma.CanSkipbyHash():
+ continue
+
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
if not self.SkipAutoGen or self.Target == 'genc':
Ma.CreateCodeFile(True)
@@ -2213,10 +2240,13 @@ def MyOptionParser():
Parser.add_option("--conf", action="store", type="string", dest="ConfDirectory", help="Specify the customized Conf directory.")
Parser.add_option("--check-usage", action="store_true", dest="CheckUsage", default=False, help="Check usage content of entries listed in INF file.")
Parser.add_option("--ignore-sources", action="store_true", dest="IgnoreSources", default=False, help="Focus to a binary build and ignore all source files")
Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set PCD value by command line. Format: \"PcdName=Value\" ")
Parser.add_option("-l", "--cmd-len", action="store", type="int", dest="CommandLength", help="Specify the maximum line length of build command. Default is 4096.")
+ Parser.add_option("--hash", action="store_true", dest="UseHashCache", default=False, help="Enable hash-based caching during build process.")
+ Parser.add_option("--bindest", action="store", type="string", dest="BinCacheDest", help="Generate a cache of binary files in the specified directory.")
+ Parser.add_option("--binsource", action="store", type="string", dest="BinCacheSource", help="Consume a cache of binary files from the specified directory.")
(Opt, Args) = Parser.parse_args()
return (Opt, Args)
## Tool entrance method
--
2.6.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
Reviewed-by: Liming Gao <liming.gao@intel.com>
>-----Original Message-----
>From: Zhu, Yonghong
>Sent: Monday, September 04, 2017 4:44 PM
>To: edk2-devel@lists.01.org
>Cc: Gao, Liming <liming.gao@intel.com>
>Subject: [Patch] BaseTools: add support for BIOS build with binary cache
>
>Add three new options:
>--hash enables hash-based caching during build process. when --hash is
>enabled, build tool will base on the module hash value to do the
>incremental build, without --hash, build tool will base on the
>timestamp to do the incremental build. --hash option use md5 method to
>get every hash value, DSC/FDF, tools_def.txt, build_rule.txt and build
>command are calculated as global hash value, Package DEC and its
>include header files are calculated as package hash value, Module
>source files and its INF file are calculated as module hash value.
>Library hash value will combine the global hash value and its dependent
>package hash value. Driver hash value will combine the global hash
>value, its dependent package hash value and its linked library hash
>value.
>When --hash and --bindest are specified, build tool will copy the
>generated binary files for each module into the directory specified by
>bindest at the build phase. Bindest caches all generated binary files.
>When --hash and --binsource are specified, build tool will try to get
>the binary files from the binary source directory at the build phase.
>If the cached binary has the same hash value, it will be directly used.
>Otherwise, build tool will compile the source files and generate the
>binary files.
>
>Cc: Liming Gao <liming.gao@intel.com>
>Contributed-under: TianoCore Contribution Agreement 1.1
>Signed-off-by: Yonghong Zhu <yonghong.zhu@intel.com>
>---
> BaseTools/Source/Python/AutoGen/AutoGen.py | 161
>+++++++++++++++++++++++++--
> BaseTools/Source/Python/Common/GlobalData.py | 7 ++
> BaseTools/Source/Python/build/build.py | 30 +++++
> 3 files changed, 189 insertions(+), 9 deletions(-)
>
>diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
>b/BaseTools/Source/Python/AutoGen/AutoGen.py
>index 70e2e62..246bfec 100644
>--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
>+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
>@@ -41,10 +41,11 @@ import Common.VpdInfoFile as VpdInfoFile
> from GenPcdDb import CreatePcdDatabaseCode
> from Workspace.MetaFileCommentParser import UsageList
> from Common.MultipleWorkspace import MultipleWorkspace as mws
> import InfSectionParser
> import datetime
>+import hashlib
>
> ## Regular expression for splitting Dependency Expression string into tokens
> gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
>
> #
>@@ -263,10 +264,14 @@ class WorkspaceAutoGen(AutoGen):
> self.FdfFile = FlashDefinitionFile
> self.FdTargetList = Fds
> self.FvTargetList = Fvs
> self.CapTargetList = Caps
> self.AutoGenObjectList = []
>+ self._BuildDir = None
>+ self._FvDir = None
>+ self._MakeFileDir = None
>+ self._BuildCommand = None
>
> # there's many relative directory operations, so ...
> os.chdir(self.WorkspaceDir)
>
> #
>@@ -642,10 +647,18 @@ class WorkspaceAutoGen(AutoGen):
> #
> Pa.CollectPlatformDynamicPcds()
> Pa.CollectFixedAtBuildPcds()
> self.AutoGenObjectList.append(Pa)
>
>+ #
>+ # Generate Package level hash value
>+ #
>+ GlobalData.gPackageHash[Arch] = {}
>+ if GlobalData.gUseHashCache:
>+ for Pkg in Pkgs:
>+ self._GenPkgLevelHash(Pkg)
>+
> #
> # Check PCDs token value conflict in each DEC file.
> #
> self._CheckAllPcdsTokenValueConflict()
>
>@@ -655,15 +668,10 @@ class WorkspaceAutoGen(AutoGen):
> self._CheckPcdDefineAndType()
>
> # if self.FdfFile:
> # self._CheckDuplicateInFV(Fdf)
>
>- self._BuildDir = None
>- self._FvDir = None
>- self._MakeFileDir = None
>- self._BuildCommand = None
>-
> #
> # Create BuildOptions Macro & PCD metafile, also add the Active Platform
>and FDF file.
> #
> content = 'gCommandLineDefines: '
> content += str(GlobalData.gCommandLineDefines)
>@@ -675,10 +683,14 @@ class WorkspaceAutoGen(AutoGen):
> content += str(self.Platform)
> content += os.linesep
> if self.FdfFile:
> content += 'Flash Image Definition: '
> content += str(self.FdfFile)
>+ content += os.linesep
>+ if GlobalData.gBinCacheDest:
>+ content += 'Cache of .efi location: '
>+ content += str(GlobalData.gBinCacheDest)
> SaveFileOnChange(os.path.join(self.BuildDir, 'BuildOptions'), content,
>False)
>
> #
> # Create PcdToken Number file for Dynamic/DynamicEx Pcd.
> #
>@@ -704,10 +716,22 @@ class WorkspaceAutoGen(AutoGen):
> for f in AllWorkSpaceMetaFiles:
> if os.stat(f)[8] > SrcTimeStamp:
> SrcTimeStamp = os.stat(f)[8]
> self._SrcTimeStamp = SrcTimeStamp
>
>+ if GlobalData.gUseHashCache:
>+ m = hashlib.md5()
>+ for files in AllWorkSpaceMetaFiles:
>+ if files.endswith('.dec'):
>+ continue
>+ f = open(files, 'r')
>+ Content = f.read()
>+ f.close()
>+ m.update(Content)
>+ SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'),
>m.hexdigest(), True)
>+ GlobalData.gPlatformHash = m.hexdigest()
>+
> #
> # Write metafile list to build directory
> #
> AutoGenFilePath = os.path.join(self.BuildDir, 'AutoGen')
> if os.path.exists (AutoGenFilePath):
>@@ -717,10 +741,33 @@ class WorkspaceAutoGen(AutoGen):
> with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
> for f in AllWorkSpaceMetaFiles:
> print >> file, f
> return True
>
>+ def _GenPkgLevelHash(self, Pkg):
>+ PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
>+ CreateDirectory(PkgDir)
>+ HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')
>+ m = hashlib.md5()
>+ # Get .dec file's hash value
>+ f = open(Pkg.MetaFile.Path, 'r')
>+ Content = f.read()
>+ f.close()
>+ m.update(Content)
>+ # Get include files hash value
>+ if Pkg.Includes:
>+ for inc in Pkg.Includes:
>+ for Root, Dirs, Files in os.walk(str(inc)):
>+ for File in Files:
>+ File_Path = os.path.join(Root, File)
>+ f = open(File_Path, 'r')
>+ Content = f.read()
>+ f.close()
>+ m.update(Content)
>+ SaveFileOnChange(HashFile, m.hexdigest(), True)
>+ if Pkg.PackageName not in GlobalData.gPackageHash[Pkg.Arch]:
>+ GlobalData.gPackageHash[Pkg.Arch][Pkg.PackageName] =
>m.hexdigest()
>
> def _GetMetaFiles(self, Target, Toolchain, Arch):
> AllWorkSpaceMetaFiles = set()
> #
> # add fdf
>@@ -954,11 +1001,12 @@ class WorkspaceAutoGen(AutoGen):
> self._FvDir = path.join(self.BuildDir, 'FV')
> return self._FvDir
>
> ## Return the directory to store all intermediate and final files built
> def _GetBuildDir(self):
>- return self.AutoGenObjectList[0].BuildDir
>+ if self._BuildDir == None:
>+ return self.AutoGenObjectList[0].BuildDir
>
> ## Return the build output directory platform specifies
> def _GetOutputDir(self):
> return self.Platform.OutputDirectory
>
>@@ -3898,37 +3946,45 @@ class ModuleAutoGen(AutoGen):
> AsBuiltInfDict['module_uefi_specification_version'] +=
>[self.Specification['UEFI_SPECIFICATION_VERSION']]
> if 'PI_SPECIFICATION_VERSION' in self.Specification:
> AsBuiltInfDict['module_pi_specification_version'] +=
>[self.Specification['PI_SPECIFICATION_VERSION']]
>
> OutputDir = self.OutputDir.replace('\\', '/').strip('/')
>-
>+ self.OutputFile = []
> for Item in self.CodaTargetList:
> File = Item.Target.Path.replace('\\', '/').strip('/').replace(OutputDir,
>'').strip('/')
>+ if File not in self.OutputFile:
>+ self.OutputFile.append(File)
> if Item.Target.Ext.lower() == '.aml':
> AsBuiltInfDict['binary_item'] += ['ASL|' + File]
> elif Item.Target.Ext.lower() == '.acpi':
> AsBuiltInfDict['binary_item'] += ['ACPI|' + File]
> elif Item.Target.Ext.lower() == '.efi':
> AsBuiltInfDict['binary_item'] += ['PE32|' + self.Name + '.efi']
> else:
> AsBuiltInfDict['binary_item'] += ['BIN|' + File]
> if self.DepexGenerated:
>+ if self.Name + '.depex' not in self.OutputFile:
>+ self.OutputFile.append(self.Name + '.depex')
> if self.ModuleType in ['PEIM']:
> AsBuiltInfDict['binary_item'] += ['PEI_DEPEX|' + self.Name + '.depex']
> if self.ModuleType in ['DXE_DRIVER', 'DXE_RUNTIME_DRIVER',
>'DXE_SAL_DRIVER', 'UEFI_DRIVER']:
> AsBuiltInfDict['binary_item'] += ['DXE_DEPEX|' + self.Name + '.depex']
> if self.ModuleType in ['DXE_SMM_DRIVER']:
> AsBuiltInfDict['binary_item'] += ['SMM_DEPEX|' + self.Name +
>'.depex']
>
> Bin = self._GenOffsetBin()
> if Bin:
> AsBuiltInfDict['binary_item'] += ['BIN|%s' % Bin]
>+ if Bin not in self.OutputFile:
>+ self.OutputFile.append(Bin)
>
> for Root, Dirs, Files in os.walk(OutputDir):
> for File in Files:
> if File.lower().endswith('.pdb'):
> AsBuiltInfDict['binary_item'] += ['DISPOSABLE|' + File]
>+ if File not in self.OutputFile:
>+ self.OutputFile.append(File)
> HeaderComments = self.Module.HeaderComments
> StartPos = 0
> for Index in range(len(HeaderComments)):
> if HeaderComments[Index].find('@BinaryHeader') != -1:
> HeaderComments[Index] =
>HeaderComments[Index].replace('@BinaryHeader', '@file')
>@@ -4105,11 +4161,52 @@ class ModuleAutoGen(AutoGen):
> AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict))
>
> SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'),
>str(AsBuiltInf), False)
>
> self.IsAsBuiltInfCreated = True
>-
>+ if GlobalData.gBinCacheDest:
>+ self.CopyModuleToCache()
>+
>+ def CopyModuleToCache(self):
>+ FileDir = path.join(GlobalData.gBinCacheDest, self.Arch, self.SourceDir,
>self.MetaFile.BaseName)
>+ CreateDirectory (FileDir)
>+ HashFile = path.join(self.BuildDir, self.Name + '.hash')
>+ ModuleFile = path.join(self.OutputDir, self.Name + '.inf')
>+ if os.path.exists(HashFile):
>+ shutil.copy2(HashFile, FileDir)
>+ if os.path.exists(ModuleFile):
>+ shutil.copy2(ModuleFile, FileDir)
>+ if self.OutputFile:
>+ for File in self.OutputFile:
>+ if not os.path.isabs(File):
>+ File = os.path.join(self.OutputDir, File)
>+ if os.path.exists(File):
>+ shutil.copy2(File, FileDir)
>+
>+ def AttemptModuleCacheCopy(self):
>+ if self.IsBinaryModule:
>+ return False
>+ FileDir = path.join(GlobalData.gBinCacheSource, self.Arch, self.SourceDir,
>self.MetaFile.BaseName)
>+ HashFile = path.join(FileDir, self.Name + '.hash')
>+ if os.path.exists(HashFile):
>+ f = open(HashFile, 'r')
>+ CacheHash = f.read()
>+ f.close()
>+ if GlobalData.gModuleHash[self.Arch][self.Name]:
>+ if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]:
>+ for root, dir, files in os.walk(FileDir):
>+ for f in files:
>+ if self.Name + '.hash' in f:
>+ shutil.copy2(HashFile, self.BuildDir)
>+ else:
>+ File = path.join(root, f)
>+ shutil.copy2(File, self.OutputDir)
>+ if self.Name == "PcdPeim" or self.Name == "PcdDxe":
>+ CreatePcdDatabaseCode(self, TemplateString(),
>TemplateString())
>+ return True
>+ return False
>+
> ## Create makefile for the module and its dependent libraries
> #
> # @param CreateLibraryMakeFile Flag indicating if or not the makefiles
>of
> # dependent libraries will be created
> #
>@@ -4233,12 +4330,58 @@ class ModuleAutoGen(AutoGen):
> self._LibraryAutoGenList.append(La)
> for Lib in La.CodaTargetList:
> self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE)
> return self._LibraryAutoGenList
>
>+ def GenModuleHash(self):
>+ if self.Arch not in GlobalData.gModuleHash:
>+ GlobalData.gModuleHash[self.Arch] = {}
>+ m = hashlib.md5()
>+ # Add Platform level hash
>+ m.update(GlobalData.gPlatformHash)
>+ # Add Package level hash
>+ if self.DependentPackageList:
>+ for Pkg in self.DependentPackageList:
>+ if Pkg.PackageName in GlobalData.gPackageHash[self.Arch]:
>+
>m.update(GlobalData.gPackageHash[self.Arch][Pkg.PackageName])
>+
>+ # Add Library hash
>+ if self.LibraryAutoGenList:
>+ for Lib in self.LibraryAutoGenList:
>+ if Lib.Name not in GlobalData.gModuleHash[self.Arch]:
>+ Lib.GenModuleHash()
>+ m.update(GlobalData.gModuleHash[self.Arch][Lib.Name])
>+
>+ # Add Module self
>+ f = open(str(self.MetaFile), 'r')
>+ Content = f.read()
>+ f.close()
>+ m.update(Content)
>+ # Add Module's source files
>+ if self.SourceFileList:
>+ for File in self.SourceFileList:
>+ f = open(str(File), 'r')
>+ Content = f.read()
>+ f.close()
>+ m.update(Content)
>+
>+ ModuleHashFile = path.join(self.BuildDir, self.Name + ".hash")
>+ if self.Name not in GlobalData.gModuleHash[self.Arch]:
>+ GlobalData.gModuleHash[self.Arch][self.Name] = m.hexdigest()
>+ if GlobalData.gBinCacheSource:
>+ CacheValid = self.AttemptModuleCacheCopy()
>+ if CacheValid:
>+ return False
>+ return SaveFileOnChange(ModuleHashFile, m.hexdigest(), True)
>+
>+ ## Decide whether we can skip the ModuleAutoGen process
>+ def CanSkipbyHash(self):
>+ if GlobalData.gUseHashCache:
>+ return not self.GenModuleHash()
>+
> ## Decide whether we can skip the ModuleAutoGen process
>- # If any source file is newer than the modeule than we cannot skip
>+ # If any source file is newer than the module than we cannot skip
> #
> def CanSkip(self):
> if not os.path.exists(self.GetTimeStampPath()):
> return False
> #last creation time of the module
>diff --git a/BaseTools/Source/Python/Common/GlobalData.py
>b/BaseTools/Source/Python/Common/GlobalData.py
>index 667877e..45e7ea0 100644
>--- a/BaseTools/Source/Python/Common/GlobalData.py
>+++ b/BaseTools/Source/Python/Common/GlobalData.py
>@@ -85,5 +85,12 @@ BuildOptionPcd = []
> #
> MixedPcd = {}
>
> # Pcd name for the Pcd which used in the Conditional directives
> gConditionalPcds = []
>+
>+gUseHashCache = None
>+gBinCacheDest = None
>+gBinCacheSource = None
>+gPlatformHash = None
>+gPackageHash = {}
>+gModuleHash = {}
>diff --git a/BaseTools/Source/Python/build/build.py
>b/BaseTools/Source/Python/build/build.py
>index 7436453..bb34b87 100644
>--- a/BaseTools/Source/Python/build/build.py
>+++ b/BaseTools/Source/Python/build/build.py
>@@ -764,10 +764,34 @@ class Build():
> self.TargetTxt = TargetTxtClassObject()
> self.ToolDef = ToolDefClassObject()
> GlobalData.BuildOptionPcd = BuildOptions.OptionPcd
> #Set global flag for build mode
> GlobalData.gIgnoreSource = BuildOptions.IgnoreSources
>+ GlobalData.gUseHashCache = BuildOptions.UseHashCache
>+ GlobalData.gBinCacheDest = BuildOptions.BinCacheDest
>+ GlobalData.gBinCacheSource = BuildOptions.BinCacheSource
>+
>+ if GlobalData.gBinCacheDest and not GlobalData.gUseHashCache:
>+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--
>bindest must be used together with --hashcache.")
>+
>+ if GlobalData.gBinCacheSource and not GlobalData.gUseHashCache:
>+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--
>binsource must be used together with --hashcache.")
>+
>+ if GlobalData.gBinCacheDest and GlobalData.gBinCacheSource:
>+ EdkLogger.error("build", OPTION_NOT_SUPPORTED, ExtraData="--
>bindest can not be used together with --binsource.")
>+
>+ if GlobalData.gBinCacheSource:
>+ BinCacheSource = os.path.normpath(GlobalData.gBinCacheSource)
>+ if not os.path.isabs(BinCacheSource):
>+ BinCacheSource = mws.join(self.WorkspaceDir, BinCacheSource)
>+ GlobalData.gBinCacheSource = BinCacheSource
>+
>+ if GlobalData.gBinCacheDest:
>+ BinCacheDest = os.path.normpath(GlobalData.gBinCacheDest)
>+ if not os.path.isabs(BinCacheDest):
>+ BinCacheDest = mws.join(self.WorkspaceDir, BinCacheDest)
>+ GlobalData.gBinCacheDest = BinCacheDest
>
> if self.ConfDirectory:
> # Get alternate Conf location, if it is absolute, then just use the absolute
>directory name
> ConfDirectoryPath = os.path.normpath(self.ConfDirectory)
>
>@@ -1908,10 +1932,13 @@ class Build():
> # Get ModuleAutoGen object to generate C code file and
>makefile
> Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch,
>self.PlatformFile)
>
> if Ma == None:
> continue
>+ if Ma.CanSkipbyHash():
>+ continue
>+
> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
> # for target which must generate AutoGen code and makefile
> if not self.SkipAutoGen or self.Target == 'genc':
> Ma.CreateCodeFile(True)
>@@ -2213,10 +2240,13 @@ def MyOptionParser():
> Parser.add_option("--conf", action="store", type="string",
>dest="ConfDirectory", help="Specify the customized Conf directory.")
> Parser.add_option("--check-usage", action="store_true",
>dest="CheckUsage", default=False, help="Check usage content of entries
>listed in INF file.")
> Parser.add_option("--ignore-sources", action="store_true",
>dest="IgnoreSources", default=False, help="Focus to a binary build and ignore
>all source files")
> Parser.add_option("--pcd", action="append", dest="OptionPcd", help="Set
>PCD value by command line. Format: \"PcdName=Value\" ")
> Parser.add_option("-l", "--cmd-len", action="store", type="int",
>dest="CommandLength", help="Specify the maximum line length of build
>command. Default is 4096.")
>+ Parser.add_option("--hash", action="store_true", dest="UseHashCache",
>default=False, help="Enable hash-based caching during build process.")
>+ Parser.add_option("--bindest", action="store", type="string",
>dest="BinCacheDest", help="Generate a cache of binary files in the specified
>directory.")
>+ Parser.add_option("--binsource", action="store", type="string",
>dest="BinCacheSource", help="Consume a cache of binary files from the
>specified directory.")
>
> (Opt, Args) = Parser.parse_args()
> return (Opt, Args)
>
> ## Tool entrance method
>--
>2.6.1.windows.1
_______________________________________________
edk2-devel mailing list
edk2-devel@lists.01.org
https://lists.01.org/mailman/listinfo/edk2-devel
© 2016 - 2024 Red Hat, Inc.