[Buildroot] [PATCH 5/5] support/testing: switch to Python 3 only
Romain Naour
romain.naour at smile.fr
Sat Oct 26 10:09:53 UTC 2019
Hi Ricardo,
Le 09/08/2019 à 01:10, Ricardo Martincoski a écrit :
> Python 2.7 will not be maintained past 2020.
>
> Many scripts on the tree are used during the build and should keep
> Python 2 compatibility for a while.
> This is not the case for the runtime test infra. It's meant to be run in
> modern distros only, so it can safely switch to support Python 3 only.
>
> An advantage of this approach is to have less scenarios to test in.
> Otherwise every change to the test infra or runtime tests would need to
> be tested against both versions of the interpreter, increasing the
> effort of the developers, to ensure the compatibility to Python 2 was
> not broken.
>
> In order to accomplish the change to Python 3:
> - change the shebang for run-tests;
> - use Python 3 urllib as a drop-in replacement for Python 2 urllib2;
> - when writing the downloaded binary files, explicitly open the output
> file as binary;
> - when subprocess is used to retrieve the text output from commands,
> explicitly ask for text output. For this, use 'universal_newlines'
> because 'text' was added only on Python 3.7;
> - when pexpect is used to retrieve the text output from qemu or git,
> explicitly ask for text output using 'encoding';
> - the code using csv currently follows the example in the documentation
> for the Python 2 module, change it to follow the example in the
> documentation for the Python 3 module;
> - fix the relative import for test_git.py to be Python 3 compliant.
I'm using Fedora 30 and there no python2-nose2 package anymore.
So, I'm using this patch to avoid installing python2-nose2 from pypi.
Tested-by: Romain Naour <romain.naour at smile.fr>
Best regards,
Romain
>
> Signed-off-by: Ricardo Martincoski <ricardo.martincoski at gmail.com>
> Cc: Arnout Vandecappelle <arnout at mind.be>
> Cc: Thomas Petazzoni <thomas.petazzoni at bootlin.com>
> Cc: Yann E. MORIN <yann.morin.1998 at free.fr>
> ---
> NOTE: obviously this patch cannot be applied before:
> - patch 4 is applied;
> - the new docker image is generated and uploaded to dockerhub;
> - a patch changing the name of the docker image to use is created and applied;
> ---
> support/testing/infra/__init__.py | 8 +++++---
> support/testing/infra/emulator.py | 1 +
> support/testing/run-tests | 2 +-
> support/testing/tests/core/test_post_scripts.py | 2 +-
> support/testing/tests/download/gitremote.py | 3 ++-
> support/testing/tests/download/test_git.py | 2 +-
> support/testing/tests/utils/test_check_package.py | 3 ++-
> 7 files changed, 13 insertions(+), 8 deletions(-)
>
> diff --git a/support/testing/infra/__init__.py b/support/testing/infra/__init__.py
> index 43045d0173..6392aa679b 100644
> --- a/support/testing/infra/__init__.py
> +++ b/support/testing/infra/__init__.py
> @@ -3,7 +3,8 @@ import re
> import sys
> import tempfile
> import subprocess
> -from urllib2 import urlopen, HTTPError, URLError
> +from urllib.request import urlopen
> +from urllib.error import HTTPError, URLError
>
> ARTIFACTS_URL = "http://autobuild.buildroot.net/artefacts/"
> BASE_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../.."))
> @@ -44,7 +45,7 @@ def download(dldir, filename):
>
> try:
> url_fh = urlopen(os.path.join(ARTIFACTS_URL, filename))
> - with open(tmpfile, "w+") as tmpfile_fh:
> + with open(tmpfile, "w+b") as tmpfile_fh:
> tmpfile_fh.write(url_fh.read())
> except (HTTPError, URLError) as err:
> os.unlink(tmpfile)
> @@ -60,7 +61,8 @@ def run_cmd_on_host(builddir, cmd):
> out = subprocess.check_output(cmd,
> stderr=open(os.devnull, "w"),
> cwd=builddir,
> - env={"LANG": "C"})
> + env={"LANG": "C"},
> + universal_newlines=True)
> return out
>
>
> diff --git a/support/testing/infra/emulator.py b/support/testing/infra/emulator.py
> index 093a643a8b..5611ec96e8 100644
> --- a/support/testing/infra/emulator.py
> +++ b/support/testing/infra/emulator.py
> @@ -76,6 +76,7 @@ class Emulator(object):
> self.logfile.write("> starting qemu with '%s'\n" % " ".join(qemu_cmd))
> self.qemu = pexpect.spawn(qemu_cmd[0], qemu_cmd[1:],
> timeout=5 * self.timeout_multiplier,
> + encoding='utf-8',
> env={"QEMU_AUDIO_DRV": "none"})
> # We want only stdout into the log to avoid double echo
> self.qemu.logfile_read = self.logfile
> diff --git a/support/testing/run-tests b/support/testing/run-tests
> index 813b927045..74741aee1b 100755
> --- a/support/testing/run-tests
> +++ b/support/testing/run-tests
> @@ -1,4 +1,4 @@
> -#!/usr/bin/env python2
> +#!/usr/bin/env python3
> import argparse
> import sys
> import os
> diff --git a/support/testing/tests/core/test_post_scripts.py b/support/testing/tests/core/test_post_scripts.py
> index 40a36b7904..bc61f4af58 100644
> --- a/support/testing/tests/core/test_post_scripts.py
> +++ b/support/testing/tests/core/test_post_scripts.py
> @@ -20,7 +20,7 @@ class TestPostScripts(infra.basetest.BRTest):
>
> def check_post_log_file(self, f, what, target_dir):
> lines = {}
> - with open(os.path.join(self.builddir, "build", f), 'rb') as csvfile:
> + with open(os.path.join(self.builddir, "build", f), newline='') as csvfile:
> r = csv.reader(csvfile, delimiter=',')
> for row in r:
> lines[row[0]] = row[1]
> diff --git a/support/testing/tests/download/gitremote.py b/support/testing/tests/download/gitremote.py
> index 3b35456dd1..7df252d031 100644
> --- a/support/testing/tests/download/gitremote.py
> +++ b/support/testing/tests/download/gitremote.py
> @@ -32,7 +32,8 @@ class GitRemote(object):
> for port in range(GIT_REMOTE_PORT_INITIAL, GIT_REMOTE_PORT_LAST + 1):
> cmd = daemon_cmd + ["--port={port}".format(port=port)]
> self.logfile.write("> starting git remote with '{}'\n".format(" ".join(cmd)))
> - self.daemon = pexpect.spawn(cmd[0], cmd[1:], logfile=self.logfile)
> + self.daemon = pexpect.spawn(cmd[0], cmd[1:], logfile=self.logfile,
> + encoding='utf-8')
> ret = self.daemon.expect(["Ready to rumble",
> "Address already in use"])
> if ret == 0:
> diff --git a/support/testing/tests/download/test_git.py b/support/testing/tests/download/test_git.py
> index 2455557298..ec5b8f3fdd 100644
> --- a/support/testing/tests/download/test_git.py
> +++ b/support/testing/tests/download/test_git.py
> @@ -1,7 +1,7 @@
> import os
> import shutil
>
> -from gitremote import GitRemote
> +from tests.download.gitremote import GitRemote
>
> import infra
>
> diff --git a/support/testing/tests/utils/test_check_package.py b/support/testing/tests/utils/test_check_package.py
> index 17c2fcf3bc..c70ba02324 100644
> --- a/support/testing/tests/utils/test_check_package.py
> +++ b/support/testing/tests/utils/test_check_package.py
> @@ -16,7 +16,8 @@ import infra
> def call_script(args, env, cwd):
> """Call a script and return stdout and stderr as lists."""
> out, err = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE,
> - stderr=subprocess.PIPE, env=env).communicate()
> + stderr=subprocess.PIPE, env=env,
> + universal_newlines=True).communicate()
> return out.splitlines(), err.splitlines()
>
>
>
More information about the buildroot
mailing list