From mboxrd@z Thu Jan 1 00:00:00 1970 Authentication-Results: mx.groups.io; dkim=missing; spf=fail (domain: intel.com, ip: , mailfrom: steven.shi@intel.com) Received: from mga09.intel.com (mga09.intel.com []) by groups.io with SMTP; Tue, 13 Aug 2019 01:51:09 -0700 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 13 Aug 2019 01:51:09 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,380,1559545200"; d="scan'208";a="200414618" Received: from jshi19-mobl.ccr.corp.intel.com ([10.254.211.145]) by fmsmga004.fm.intel.com with ESMTP; 13 Aug 2019 01:51:08 -0700 From: "Steven Shi" To: devel@edk2.groups.io Cc: liming.gao@intel.com, bob.c.feng@intel.com, christian.rodriguez@intel.com, michael.johnson@intel.com, "Shi, Steven" Subject: [PATCH v3 5/5] BaseTools: Improve the file saving and copying reliability Date: Tue, 13 Aug 2019 16:50:55 +0800 Message-Id: <20190813085055.23208-6-steven.shi@intel.com> X-Mailer: git-send-email 2.17.1.windows.2 In-Reply-To: <20190813085055.23208-1-steven.shi@intel.com> References: <20190813085055.23208-1-steven.shi@intel.com> From: "Shi, Steven" BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2079 The Basetool CopyFileOnChange() and SaveFileOnChange() functions might raise the IOError occasionally when build in Windows with multi-process and build cache enabled. The CopyFileOnChange() and SaveFileOnChange() might be invoked in multiple sub-processes simultaneously, and this patch adds a global lock to sync these functions invoking which can harden their reliability. Cc: Liming Gao Cc: Bob Feng Signed-off-by: Steven Shi --- BaseTools/Source/Python/AutoGen/GenC.py | 2 +- BaseTools/Source/Python/AutoGen/GenMake.py | 2 +- BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 2 +- BaseTools/Source/Python/Common/Misc.py | 18 +++++++++++++----- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py old mode 100644 new mode 100755 index 910c8fe370..1bb3f42b46 --- a/BaseTools/Source/Python/AutoGen/GenC.py +++ b/BaseTools/Source/Python/AutoGen/GenC.py @@ -2103,5 +2103,5 @@ def CreateCode(Info, AutoGenC, AutoGenH, StringH, UniGenCFlag, UniGenBinBuffer, # @retval False If the file exists and the content is not changed # def Generate(FilePath, Content, IsBinaryFile): - return SaveFileOnChange(FilePath, Content, IsBinaryFile) + return SaveFileOnChange(FilePath, Content, IsBinaryFile, GlobalData.file_lock) diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py index de820eeb2f..e159782306 100755 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -185,7 +185,7 @@ class BuildFile(object): self._FileType = FileType FileContent = self._TEMPLATE_.Replace(self._TemplateDict) FileName = self._FILE_NAME_[FileType] - return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False) + return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False, GlobalData.file_lock) ## Return a list of directory creation command string # diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index ee8518e19c..0d16d8ded7 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1612,7 +1612,7 @@ class ModuleAutoGen(AutoGen): destination_dir = os.path.dirname(destination_file) CreateDirectory(destination_dir) try: - CopyFileOnChange(File, destination_dir) + CopyFileOnChange(File, destination_dir, GlobalData.file_lock) except: EdkLogger.quiet("[cache warning]: fail to copy file:%s to folder:%s" % (File, destination_dir)) return diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py old mode 100644 new mode 100755 index 26d149c270..db2c3c9589 --- a/BaseTools/Source/Python/Common/Misc.py +++ b/BaseTools/Source/Python/Common/Misc.py @@ -448,7 +448,7 @@ def RemoveDirectory(Directory, Recursively=False): # @retval True If the file content is changed and the file is renewed # @retval False If the file content is the same # -def SaveFileOnChange(File, Content, IsBinaryFile=True): +def SaveFileOnChange(File, Content, IsBinaryFile=True, FileLock=None): if os.path.exists(File): if IsBinaryFile: @@ -486,8 +486,12 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True): tf.write(Content) tempname = tf.name try: - os.rename(tempname, File) - except: + if FileLock: + with FileLock: + os.rename(tempname, File) + else: + os.rename(tempname, File) + except IOError as X: EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X) else: try: @@ -510,7 +514,7 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True): # @retval True The two files content are different and the file is copied # @retval False No copy really happen # -def CopyFileOnChange(SrcFile, Dst): +def CopyFileOnChange(SrcFile, Dst, FileLock=None): if not os.path.exists(SrcFile): return False @@ -545,7 +549,11 @@ def CopyFileOnChange(SrcFile, Dst): # os.rename reqire to remove the dst on Windows, otherwise OSError will be raised. if GlobalData.gIsWindows and os.path.exists(DstFile): os.remove(DstFile) - os.rename(tempname, DstFile) + if FileLock: + with FileLock: + os.rename(tempname, DstFile) + else: + os.rename(tempname, DstFile) except IOError as X: EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X) -- 2.17.1