From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: intel.com, ip: 192.55.52.115, mailfrom: bob.c.feng@intel.com) Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by groups.io with SMTP; Mon, 17 Jun 2019 01:55:36 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 17 Jun 2019 01:55:35 -0700 X-ExtLoop1: 1 Received: from fmsmsx103.amr.corp.intel.com ([10.18.124.201]) by orsmga008.jf.intel.com with ESMTP; 17 Jun 2019 01:55:35 -0700 Received: from fmsmsx126.amr.corp.intel.com (10.18.125.43) by FMSMSX103.amr.corp.intel.com (10.18.124.201) with Microsoft SMTP Server (TLS) id 14.3.439.0; Mon, 17 Jun 2019 01:55:35 -0700 Received: from shsmsx107.ccr.corp.intel.com (10.239.4.96) by FMSMSX126.amr.corp.intel.com (10.18.125.43) with Microsoft SMTP Server (TLS) id 14.3.439.0; Mon, 17 Jun 2019 01:55:35 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.104]) by SHSMSX107.ccr.corp.intel.com ([169.254.9.173]) with mapi id 14.03.0439.000; Mon, 17 Jun 2019 16:55:33 +0800 From: "Bob Feng" To: "Shi, Steven" , "devel@edk2.groups.io" CC: "Gao, Liming" , "Rodriguez, Christian" Subject: Re: [PATCH v2 1/1] BaseTools:Introduce CopyFileOnChange() function to copy cache files Thread-Topic: [PATCH v2 1/1] BaseTools:Introduce CopyFileOnChange() function to copy cache files Thread-Index: AQHVJOjUzf4xfeoHiE+2nd4ui+2C1qafivUw Date: Mon, 17 Jun 2019 08:55:32 +0000 Message-ID: <08650203BA1BD64D8AD9B6D5D74A85D1601527ED@SHSMSX101.ccr.corp.intel.com> References: <20190617084358.6168-1-steven.shi@intel.com> <20190617084358.6168-2-steven.shi@intel.com> In-Reply-To: <20190617084358.6168-2-steven.shi@intel.com> Accept-Language: zh-CN, en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Return-Path: bob.c.feng@intel.com Content-Language: en-US Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable Reviewed-by: Bob Feng -----Original Message----- From: Shi, Steven=20 Sent: Monday, June 17, 2019 4:44 PM To: devel@edk2.groups.io Cc: Gao, Liming ; Feng, Bob C ;= Rodriguez, Christian Subject: [PATCH v2 1/1] BaseTools:Introduce CopyFileOnChange() function to = copy cache files BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=3D1894 Basetool need a CopyFileOnChange() function to avoid cache file writing rac= e in multi-thread build. Some platforms build fail with file IO writing rac= e issue when the build cache is enabled to store cache files in multi-threa= ds. This is because common same library cache files (e.g. some libs in MdePkg) = can be stored by many different driver modules' build threads at same time. Current build cache need a function to check wh= ether the same cache file already exist, and only copy source file if it is= different from the destination file. This patch introduces an atomic copy function to avoid duplicated cache fil= es copy. Cc: Liming Gao Cc: Bob Feng Cc: Christian Rodriguez Signed-off-by: Steven Shi --- BaseTools/Source/Python/AutoGen/AutoGen.py | 10 +++++----- BaseTools/Source/Python/Common/LongFilePathOs.py | 4 ++++ BaseTools/Source/Python/Common/Misc.py | 55 ++++++++++++++++++++= +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py b/BaseTools/Source/= Python/AutoGen/AutoGen.py index 3f41fbb507..46c8fbe181 100644 --- a/BaseTools/Source/Python/AutoGen/AutoGen.py +++ b/BaseTools/Source/Python/AutoGen/AutoGen.py @@ -3901,11 +3901,11 @@ class ModuleAutoGen(AutoGen): CreateDirectory (FileDir) HashFile =3D path.join(self.BuildDir, self.Name + '.hash') if os.path.exists(HashFile): - shutil.copy2(HashFile, FileDir) + CopyFileOnChange(HashFile, FileDir) if not self.IsLibrary: ModuleFile =3D path.join(self.OutputDir, self.Name + '.inf') if os.path.exists(ModuleFile): - shutil.copy2(ModuleFile, FileDir) + CopyFileOnChange(ModuleFile, FileDir) else: OutputDir =3D self.OutputDir.replace('\\', '/').strip('/') DebugDir =3D self.DebugDir.replace('\\', '/').strip('/') @@ -3= 925,7 +3925,7 @@ class ModuleAutoGen(AutoGen): destination_file =3D os.path.join(FileDir, sub_dir) destination_dir =3D os.path.dirname(destination_file) CreateDirectory(destination_dir) - shutil.copy2(File, destination_dir) + CopyFileOnChange(File, destination_dir) =20 def AttemptModuleCacheCopy(self): # If library or Module is binary do not skip by hash @@ -3947,14 += 3947,14 @@ class ModuleAutoGen(AutoGen): for root, dir, files in os.walk(FileDir): for f in files: if self.Name + '.hash' in f: - shutil.copy(HashFile, self.BuildDir) + CopyFileOnChange(HashFile,=20 + self.BuildDir) else: File =3D path.join(root, f) sub_dir =3D os.path.relpath(File, FileDir) destination_file =3D os.path.join(self.Out= putDir, sub_dir) destination_dir =3D os.path.dirname(destin= ation_file) CreateDirectory(destination_dir) - shutil.copy(File, destination_dir) + CopyFileOnChange(File, destination_dir) if self.Name =3D=3D "PcdPeim" or self.Name =3D=3D "Pcd= Dxe": CreatePcdDatabaseCode(self, TemplateString(), Temp= lateString()) return True diff --git a/BaseTools/Source/Python/Common/LongFilePathOs.py b/BaseTools/S= ource/Python/Common/LongFilePathOs.py index ae8a68e4ad..190f36d7ec 100644 --- a/BaseTools/Source/Python/Common/LongFilePathOs.py +++ b/BaseTools/Source/Python/Common/LongFilePathOs.py @@ -60,6 +60,10 @@ def listdir(path): List.append(Item) return List =20 +if hasattr(os, 'replace'): + def replace(src, dst): + return os.replace(LongFilePath(src), LongFilePath(dst)) + environ =3D os.environ getcwd =3D os.getcwd chdir =3D os.chdir diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Pyth= on/Common/Misc.py index d082c58bef..9a63463913 100644 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -18,6 +18,7 @@ import re import pickle import array import shutil +import filecmp from random import sample from struct import pack import uuid @@ -502,6 +503,60 @@ def SaveFileOnChange(File, Content, IsBinaryFile=3DTru= e): =20 return True =20 +## Copy source file only if it is different from the destination file #=20 +# This method is used to copy file only if the source file and=20 +destination # file content are different. This is quite useful to=20 +avoid duplicated # file writing. +# +# @param SrcFile The path of source file +# @param Dst The path of destination file or folder +# +# @retval True The two files content are different and the file= is copied +# @retval False No copy really happen +# +def CopyFileOnChange(SrcFile, Dst): + if not os.path.exists(SrcFile): + return False + + if os.path.isdir(Dst): + DstFile =3D os.path.join(Dst, os.path.basename(SrcFile)) + else: + DstFile =3D Dst + + if os.path.exists(DstFile) and filecmp.cmp(SrcFile, DstFile, shallow= =3DFalse): + return False + + DirName =3D os.path.dirname(DstFile) + if not CreateDirectory(DirName): + EdkLogger.error(None, FILE_CREATE_FAILURE, "Could not create direc= tory %s" % DirName) + else: + if DirName =3D=3D '': + DirName =3D os.getcwd() + if not os.access(DirName, os.W_OK): + EdkLogger.error(None, PERMISSION_FAILURE, "Do not have=20 + write permission on directory %s" % DirName) + + # os.replace and os.rename are the atomic operations in python 3 and 2= . + # we use these two atomic operations to ensure the file copy is atomic= : + # copy the src to a temp file in the dst same folder firstly, then + # replace or rename the temp file to the destination file. + with tempfile.NamedTemporaryFile(dir=3DDirName, delete=3DFalse) as tf: + shutil.copy(SrcFile, tf.name) + tempname =3D tf.name + try: + if hasattr(os, 'replace'): + os.replace(tempname, DstFile) + else: + # os.rename reqire to remove the dst on Windows, otherwise OSE= rror will be raised. + if GlobalData.gIsWindows and os.path.exists(DstFile): + os.remove(DstFile) + os.rename(tempname, DstFile) + + except IOError as X: + EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData=3D'IOError %s'= =20 + % X) + + return True + ## Retrieve and cache the real path name in file system # # @param Root The root directory of path relative to -- 2.17.1.windows.2