* [PATCH 1/3] build: Fix python3.10 threading DeprecationWarnings
2021-07-23 20:02 [PATCH 0/3] BaseTools: fix some python DeprecationWarnings Cole
@ 2021-07-23 20:02 ` Cole
2021-07-23 20:02 ` [PATCH 2/3] python: Replace distutils.utils.split_quotes with shlex.split Cole
` (3 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Cole @ 2021-07-23 20:02 UTC (permalink / raw)
To: devel; +Cc: Cole Robinson
threading camelCase functions have preferred alternatives since
python2.6. python3.10 has started emitting DeprecationWarnings
for them
Signed-off-by: Cole Robinson <crobinso@redhat.com>
---
BaseTools/Source/Python/build/build.py | 48 +++++++++++++-------------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 3e4d83409f..02b4898924 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -197,7 +197,7 @@ def ReadMessage(From, To, ExitFlag,MemTo=None):
To(LineStr)
else:
break
- if ExitFlag.isSet():
+ if ExitFlag.is_set():
break
class MakeSubProc(Popen):
@@ -241,8 +241,8 @@ def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
EndOfProcedure.clear()
if Proc.stdout:
StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
- StdOutThread.setName("STDOUT-Redirector")
- StdOutThread.setDaemon(False)
+ StdOutThread.name = "STDOUT-Redirector"
+ StdOutThread.daemon = False
StdOutThread.start()
@@ -433,8 +433,8 @@ class BuildTask:
@staticmethod
def StartScheduler(MaxThreadNumber, ExitFlag):
SchedulerThread = Thread(target=BuildTask.Scheduler, args=(MaxThreadNumber, ExitFlag))
- SchedulerThread.setName("Build-Task-Scheduler")
- SchedulerThread.setDaemon(False)
+ SchedulerThread.name = "Build-Task-Scheduler"
+ SchedulerThread.daemon = False
SchedulerThread.start()
# wait for the scheduler to be started, especially useful in Linux
while not BuildTask.IsOnGoing():
@@ -456,7 +456,7 @@ class BuildTask:
# indicated to do so, or there's error in running thread
#
while (len(BuildTask._PendingQueue) > 0 or len(BuildTask._ReadyQueue) > 0 \
- or not ExitFlag.isSet()) and not BuildTask._ErrorFlag.isSet():
+ or not ExitFlag.is_set()) and not BuildTask._ErrorFlag.is_set():
EdkLogger.debug(EdkLogger.DEBUG_8, "Pending Queue (%d), Ready Queue (%d)"
% (len(BuildTask._PendingQueue), len(BuildTask._ReadyQueue)))
@@ -474,7 +474,7 @@ class BuildTask:
BuildTask._PendingQueueLock.release()
# launch build thread until the maximum number of threads is reached
- while not BuildTask._ErrorFlag.isSet():
+ while not BuildTask._ErrorFlag.is_set():
# empty ready queue, do nothing further
if len(BuildTask._ReadyQueue) == 0:
break
@@ -498,12 +498,12 @@ class BuildTask:
time.sleep(0.01)
# wait for all running threads exit
- if BuildTask._ErrorFlag.isSet():
+ if BuildTask._ErrorFlag.is_set():
EdkLogger.quiet("\nWaiting for all build threads exit...")
- # while not BuildTask._ErrorFlag.isSet() and \
+ # while not BuildTask._ErrorFlag.is_set() and \
while len(BuildTask._RunningQueue) > 0:
EdkLogger.verbose("Waiting for thread ending...(%d)" % len(BuildTask._RunningQueue))
- EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.getName() for Th in threading.enumerate()))
+ EdkLogger.debug(EdkLogger.DEBUG_8, "Threads [%s]" % ", ".join(Th.name for Th in threading.enumerate()))
# avoid tense loop
time.sleep(0.1)
except BaseException as X:
@@ -531,7 +531,7 @@ class BuildTask:
#
@staticmethod
def IsOnGoing():
- return not BuildTask._SchedulerStopped.isSet()
+ return not BuildTask._SchedulerStopped.is_set()
## Abort the build
@staticmethod
@@ -547,7 +547,7 @@ class BuildTask:
#
@staticmethod
def HasError():
- return BuildTask._ErrorFlag.isSet()
+ return BuildTask._ErrorFlag.is_set()
## Get error message in running thread
#
@@ -644,7 +644,7 @@ class BuildTask:
# TRICK: hide the output of threads left running, so that the user can
# catch the error message easily
#
- if not BuildTask._ErrorFlag.isSet():
+ if not BuildTask._ErrorFlag.is_set():
GlobalData.gBuildingModule = "%s [%s, %s, %s]" % (str(self.BuildItem.BuildObject),
self.BuildItem.BuildObject.Arch,
self.BuildItem.BuildObject.ToolChain,
@@ -653,7 +653,7 @@ class BuildTask:
EdkLogger.SetLevel(EdkLogger.ERROR)
BuildTask._ErrorFlag.set()
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \
- (threading.currentThread().getName(), Command, WorkingDir)
+ (threading.current_thread().name, Command, WorkingDir)
# indicate there's a thread is available for another build task
BuildTask._RunningQueueLock.acquire()
@@ -667,8 +667,8 @@ class BuildTask:
EdkLogger.quiet("Building ... %s" % repr(self.BuildItem))
Command = self.BuildItem.BuildCommand + [self.BuildItem.Target]
self.BuildTread = Thread(target=self._CommandThread, args=(Command, self.BuildItem.WorkingDir))
- self.BuildTread.setName("build thread")
- self.BuildTread.setDaemon(False)
+ self.BuildTread.name = "build thread"
+ self.BuildTread.daemon = False
self.BuildTread.start()
## The class contains the information related to EFI image
@@ -1177,14 +1177,14 @@ class Build():
EndOfProcedure.clear()
if Process.stdout:
StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
- StdOutThread.setName("STDOUT-Redirector")
- StdOutThread.setDaemon(False)
+ StdOutThread.name = "STDOUT-Redirector"
+ StdOutThread.daemon = False
StdOutThread.start()
if Process.stderr:
StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
- StdErrThread.setName("STDERR-Redirector")
- StdErrThread.setDaemon(False)
+ StdErrThread.name = "STDERR-Redirector"
+ StdErrThread.daemon = False
StdErrThread.start()
# waiting for program exit
Process.wait()
@@ -1217,14 +1217,14 @@ class Build():
EndOfProcedure.clear()
if Process.stdout:
StdOutThread = Thread(target=ReadMessage, args=(Process.stdout, EdkLogger.info, EndOfProcedure))
- StdOutThread.setName("STDOUT-Redirector")
- StdOutThread.setDaemon(False)
+ StdOutThread.name = "STDOUT-Redirector"
+ StdOutThread.daemon = False
StdOutThread.start()
if Process.stderr:
StdErrThread = Thread(target=ReadMessage, args=(Process.stderr, EdkLogger.quiet, EndOfProcedure))
- StdErrThread.setName("STDERR-Redirector")
- StdErrThread.setDaemon(False)
+ StdErrThread.name = "STDERR-Redirector"
+ StdErrThread.daemon = False
StdErrThread.start()
# waiting for program exit
Process.wait()
--
2.31.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/3] python: Replace distutils.utils.split_quotes with shlex.split
2021-07-23 20:02 [PATCH 0/3] BaseTools: fix some python DeprecationWarnings Cole
2021-07-23 20:02 ` [PATCH 1/3] build: Fix python3.10 threading DeprecationWarnings Cole
@ 2021-07-23 20:02 ` Cole
2021-07-23 20:02 ` [PATCH 3/3] BaseTools: Drop check for distutils.utils Cole
` (2 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Cole @ 2021-07-23 20:02 UTC (permalink / raw)
To: devel; +Cc: Cole Robinson
distutils is deprecated and may be removed in python 3.12.
Use shlex.split which has been around since python 2.3.
shlex.split does not split on all the ASCII control characters that
split_quoted will[1], but for edk2 usage I don't think that matters.
[1] https://stackoverflow.com/questions/54999301/what-is-the-difference-between-distutils-util-split-quoted-and-shlex-split
Signed-off-by: Cole Robinson <crobinso@redhat.com>
---
BaseTools/Source/Python/AutoGen/UniClassObject.py | 4 ++--
BaseTools/Source/Python/UPT/Library/UniClassObject.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/UniClassObject.py b/BaseTools/Source/Python/AutoGen/UniClassObject.py
index 883c2356e0..b16330e368 100644
--- a/BaseTools/Source/Python/AutoGen/UniClassObject.py
+++ b/BaseTools/Source/Python/AutoGen/UniClassObject.py
@@ -12,7 +12,7 @@
#
from __future__ import print_function
import Common.LongFilePathOs as os, codecs, re
-import distutils.util
+import shlex
import Common.EdkLogger as EdkLogger
from io import BytesIO
from Common.BuildToolError import *
@@ -233,7 +233,7 @@ class UniFileClassObject(object):
# Get Language definition
#
def GetLangDef(self, File, Line):
- Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
+ Lang = shlex.split(Line.split(u"//")[0])
if len(Lang) != 3:
try:
FileIn = UniFileClassObject.OpenUniFile(LongFilePath(File.Path))
diff --git a/BaseTools/Source/Python/UPT/Library/UniClassObject.py b/BaseTools/Source/Python/UPT/Library/UniClassObject.py
index d25f300146..8c44dc2252 100644
--- a/BaseTools/Source/Python/UPT/Library/UniClassObject.py
+++ b/BaseTools/Source/Python/UPT/Library/UniClassObject.py
@@ -14,7 +14,7 @@ from __future__ import print_function
# Import Modules
#
import os, codecs, re
-import distutils.util
+import shlex
from Logger import ToolError
from Logger import Log as EdkLogger
from Logger import StringTable as ST
@@ -320,7 +320,7 @@ class UniFileClassObject(object):
# Get Language definition
#
def GetLangDef(self, File, Line):
- Lang = distutils.util.split_quoted((Line.split(u"//")[0]))
+ Lang = shlex.split(Line.split(u"//")[0])
if len(Lang) != 3:
try:
FileIn = codecs.open(File.Path, mode='rb', encoding='utf_8').readlines()
--
2.31.1
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [edk2-devel] [PATCH 0/3] BaseTools: fix some python DeprecationWarnings
2021-07-23 20:02 [PATCH 0/3] BaseTools: fix some python DeprecationWarnings Cole
` (3 preceding siblings ...)
2021-07-26 3:52 ` [edk2-devel] [PATCH 0/3] BaseTools: fix some python DeprecationWarnings Bob Feng
@ 2021-08-10 7:48 ` Yuwei Chen
4 siblings, 0 replies; 7+ messages in thread
From: Yuwei Chen @ 2021-08-10 7:48 UTC (permalink / raw)
To: devel@edk2.groups.io, crobinso@redhat.com, Liming Gao
Cc: Feng, Bob C, Liu, Zhiguang
Hi Robinson,
When we doing the internal test, the issue is found that py27 is blocked by this patch.
You can reproduce the error with below steps:
For Linux:
[Error Reproduce steps]
$ export PYTHON3_ENABLE=FALSE
$ export PYTHON_COMMAND=/usr/bin/python2.7
$ build -p OvmfPkg/OvmfPkgIa32X64.dsc -a IA32 -a X64 -t GCC5
Active Platform = /home/jshi19/wksp_efi/edk2-3/OvmfPkg/OvmfPkgIa32X64.dsc
......
- Failed -
Build end time: 13:00:59, Aug.10 2021
Build total time: 00:00:07
[After Reverting the patch]
$ git revert 0b1b0a9674e27c858f05436ed92250f4498245cf
$ build -p OvmfPkg/OvmfPkgIa32X64.dsc -a IA32 -a X64 -t GCC5
Pass
For Windows:
[Error Reproduce steps]
set PYTHON3_ENABLE=FALSE
set PYTHON_COMMAND=C:\Python27\python.exe
build -p OvmfPkg\OvmfPkgIa32X64.dsc -a IA32 -a X64 -t VS2015x86
Active Platform = c:\users\yuweiche\code\edk2\edk2\OvmfPkg\OvmfPkgIa32X64.dsc
......
- Failed -
Build end time: 15:40:35, Aug.10 2021
Build total time: 00:00:07
[After Reverting the patch]
$ git revert 0b1b0a9674e27c858f05436ed92250f4498245cf
$ build -p OvmfPkg\OvmfPkgIa32X64.dsc -a IA32 -a X64 -t VS2015x86
- Done -
Build end time: 15:35:16, Aug.10 2021
Build total time: 00:01:58
Hope you can fix this bug. 😊 @crobinso@redhat.com
Hi Liming @Liming Gao, what do you think about this issue? If we need fix it before this quarter release?
Thanks,
Yuwei (Christine)
> -----Original Message-----
> From: devel@edk2.groups.io <devel@edk2.groups.io> On Behalf Of Cole
> Sent: Saturday, July 24, 2021 4:02 AM
> To: devel@edk2.groups.io
> Cc: Cole Robinson <crobinso@redhat.com>
> Subject: [edk2-devel] [PATCH 0/3] BaseTools: fix some python
> DeprecationWarnings
>
> This addresses some python DeprecationWarnings that are popping up with
> python 3.10
>
> Cole Robinson (3):
> build: Fix python3.10 threading DeprecationWarnings
> python: Replace distutils.utils.split_quotes with shlex.split
> BaseTools: Drop check for distutils.utils
>
> .../Source/Python/AutoGen/UniClassObject.py | 4 +-
> .../Python/UPT/Library/UniClassObject.py | 4 +-
> BaseTools/Source/Python/build/build.py | 48 +++++++++----------
> BaseTools/Tests/RunTests.py | 7 ---
> 4 files changed, 28 insertions(+), 35 deletions(-)
>
> --
> 2.31.1
>
>
>
>
>
^ permalink raw reply [flat|nested] 7+ messages in thread