From: Waldemar Kozaczuk <[email protected]> Committer: Waldemar Kozaczuk <[email protected]> Branch: master
scripts: upgrade to python 3 - part 1.1 This patch incorporates changes from original patch submitted by Matt Bass as well as extra changes made by Waldemar Kozaczuk to upgrade python scripts from version 2 to version 3. Some trivial changes are the result of automated upgrade tools like "Future" module or 2to3. Others involving proper handling of binary and string data especially when using subprocess module have been manually made. Same goes for any code that tried to compare non-uniform data which python 3 is more strict. Most of the scripts have been tested but there maybe be scenarios/flows which have been missed. Ref #1056 Signed-off-by: Matt Bass <[email protected]> Signed-off-by: Waldemar Kozaczuk <[email protected]> --- diff --git a/docker/Dockerfile.Fedora b/docker/Dockerfile.Fedora --- a/docker/Dockerfile.Fedora +++ b/docker/Dockerfile.Fedora @@ -8,10 +8,9 @@ # This Docker file defines a container intended to build, test and publish # OSv kernel as well as many applications ... # -#FROM fedora:27 #On Jenkins build machine FROM fedora:29 -RUN yum install -y git python2 file which +RUN yum install -y git python2 python3 file which # # PREPARE ENVIRONMENT diff --git a/docker/Dockerfile.Ubuntu b/docker/Dockerfile.Ubuntu --- a/docker/Dockerfile.Ubuntu +++ b/docker/Dockerfile.Ubuntu @@ -16,7 +16,7 @@ ENV TERM=linux COPY ./etc/keyboard /etc/default/keyboard COPY ./etc/console-setup /etc/default/console-setup -RUN apt-get update -y && apt-get install -y git python2 +RUN apt-get update -y && apt-get install -y git python2 python3 # # PREPARE ENVIRONMENT diff --git a/modules/openjdk8-from-host/module.py b/modules/openjdk8-from-host/module.py --- a/modules/openjdk8-from-host/module.py +++ b/modules/openjdk8-from-host/module.py @@ -11,12 +11,12 @@ print('Could not find any jdk on the host. Please install openjdk8!') os.exit(-1) -java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) +java_version = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT).decode('utf-8') if not 'openjdk version "1.8.0' in java_version: print('Could not find openjdk version 8 on the host. Please install openjdk8!') os.exit(-1) -javac_path = subprocess.check_output(['which', 'javac']).split('\n')[0] +javac_path = subprocess.check_output(['which', 'javac']).decode('utf-8').split('\n')[0] javac_real_path = os.path.realpath(javac_path) jdk_path = os.path.dirname(os.path.dirname(javac_real_path)) diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py --- a/scripts/export_manifest.py +++ b/scripts/export_manifest.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import optparse, os, shutil from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file @@ -8,7 +8,7 @@ # support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. def export_package(manifest, dest): abs_dest = os.path.abspath(dest) - print "[INFO] exporting into directory %s" % abs_dest + print("[INFO] exporting into directory %s" % abs_dest) # Remove and create the base directory where we are going to put all package files. if os.path.exists(abs_dest): @@ -39,7 +39,7 @@ def export_package(manifest, dest): os.makedirs(target_dir) os.symlink(link_source, name) - print "[INFO] added link %s -> %s" % (name, link_source) + print("[INFO] added link %s -> %s" % (name, link_source)) else: # If it is a symlink, then resolve it add to the list of host symlinks to be created later @@ -58,23 +58,23 @@ def export_package(manifest, dest): hostname = strip_file(hostname) shutil.copy(hostname, name) - print "[INFO] exported %s" % name + print("[INFO] exported %s" % name) elif os.path.isdir(hostname): # If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy. if not os.path.exists(name): os.makedirs(name) - print "[INFO] created dir %s" % name + print("[INFO] created dir %s" % name) else: # Inform the user that the rule cannot be applied. For example, this happens for links in OSv. - print "[ERR] unable to export %s" % hostname + print("[ERR] unable to export %s" % hostname) for link_source, name in host_symlinks: target_dir = os.path.dirname(name) if not os.path.exists(target_dir): os.makedirs(target_dir) os.symlink(link_source, name) - print "[INFO] added link %s -> %s" % (name, link_source) + print("[INFO] added link %s -> %s" % (name, link_source)) def main(): make_option = optparse.make_option diff --git a/scripts/firecracker.py b/scripts/firecracker.py --- a/scripts/firecracker.py +++ b/scripts/firecracker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # pip install requests-unixsocket import sys @@ -152,14 +152,14 @@ def print_time(msg): def setup_tap_interface(mode, tap_interface_name, tap_ip=None, physical_nic=None, bridge_name=None): # Setup tun tap interface if does not exist # sudo ip link delete fc_tap0 - this deletes the tap device - tuntap_interfaces = subprocess.check_output(['ip', 'tuntap']) + tuntap_interfaces = subprocess.check_output(['ip', 'tuntap']).decode('utf-8') if tuntap_interfaces.find(tap_interface_name) < 0: print("The tap interface %s not found -> needs to set it up!" % tap_interface_name) dirname = os.path.dirname(os.path.abspath(__file__)) setup_networking_script = os.path.join(dirname, 'setup_fc_networking.sh') # Check if the bridge exists if user specified it if mode == 'bridged' and bridge_name: - bridges = subprocess.check_output(['brctl', 'show']) + bridges = subprocess.check_output(['brctl', 'show']).decode('utf-8') if bridges.find(bridge_name) < 0: print("The bridge %s does not exist per brctl. Please create one!" % bridge_name) exit(-1) @@ -182,7 +182,7 @@ def find_firecracker(dirname): if not os.path.exists(firecracker_path): url_base = 'https://github.com/firecracker-microvm/firecracker/releases/download' download_url = '%s/%s/firecracker-%s' % (url_base, firecracker_version, firecracker_version) - answer = raw_input("Firecracker executable has not been found under %s. " + answer = input("Firecracker executable has not been found under %s. " "Would you like to download it from %s and place it under %s? [y|n]" % (firecracker_path, download_url, firecracker_path)) if answer.capitalize() != 'Y': @@ -233,7 +233,7 @@ def start_firecracker(firecracker_path, socket_path): def start_firecracker_with_no_api(firecracker_path, firecracker_config_json): # Start firecracker process and pass configuration JSON as a file api_file = tempfile.NamedTemporaryFile(delete=False) - api_file.write(firecracker_config_json) + api_file.write(bytes(firecracker_config_json, 'utf-8')) stty_save() return subprocess.Popen([firecracker_path, "--no-api", "--config-file", api_file.name], stdout=sys.stdout, stderr=subprocess.STDOUT), api_file.name diff --git a/scripts/gen-rofs-img.py b/scripts/gen-rofs-img.py --- a/scripts/gen-rofs-img.py +++ b/scripts/gen-rofs-img.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright (c) 2015 Carnegie Mellon University. @@ -93,9 +93,7 @@ def write(self,fp): pos = fp.tell() fp.write(c_ulonglong(self.inode_no)) fp.write(c_ushort(len(self.filename))) - for c in self.filename: - fp.write(c_char(c)) - + fp.write(bytes(self.filename,'utf-8')) return fp.tell() - pos class SymbolicLink(object): @@ -105,9 +103,7 @@ def __init__(self,path): def write(self,fp): pos = fp.tell() fp.write(c_ushort(len(self.path))) - for c in self.path: - fp.write(c_char(c)) - + fp.write(bytes(self.path,'utf-8')) return fp.tell() - pos directory_entries = [] @@ -151,7 +147,7 @@ def next_inode(): return inode def pad(fp, size): - fp.write('\0' * size) + fp.write(b'\0' * size) return size def write_initial_superblock(fp): @@ -218,13 +214,13 @@ def write_dir(fp, manifest, dirpath, parent_dir): inode.data_offset = symlinks_count inode.count = 1 next_symlink(val[2:],manifest) - print 'Link %s to %s' % (dirpath + '/' + entry, val[2:]) + print('Link %s to %s' % (dirpath + '/' + entry, val[2:])) else: #file inode.mode = REG_MODE global block inode.data_offset = block inode.count = write_file(fp, val) - print 'Adding %s' % (dirpath + '/' + entry) + print('Adding %s' % (dirpath + '/' + entry)) # This needs to be added so that later we can walk the tree # when fining symlinks @@ -264,15 +260,15 @@ def write_fs(fp, manifest): return (block_no, bytes_written) def gen_image(out, manifest): - print 'Writing image' + print('Writing image') fp = open(out, 'wb') # write the initial superblock write_initial_superblock(fp) system_structure_block, bytes_written = write_fs(fp, manifest) structure_info_last_block_bytes = bytes_written % OSV_BLOCK_SIZE - structure_info_blocks_count = bytes_written / OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0) + structure_info_blocks_count = bytes_written // OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0) pad(fp,OSV_BLOCK_SIZE - structure_info_last_block_bytes) @@ -290,10 +286,10 @@ def gen_image(out, manifest): sb.symlinks_count = len(symlinks) sb.inodes_count = len(inodes) - print 'First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count) - print 'Directory entries count %d' % sb.directory_entries_count - print 'Symlinks count %d' % sb.symlinks_count - print 'Inodes count %d' % sb.inodes_count + print('First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count)) + print('Directory entries count %d' % sb.directory_entries_count) + print('Symlinks count %d' % sb.symlinks_count) + print('Inodes count %d' % sb.inodes_count) fp.seek(0) fp.write(sb) diff --git a/scripts/imgedit.py b/scripts/imgedit.py --- a/scripts/imgedit.py +++ b/scripts/imgedit.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import sys, struct import subprocess @@ -31,14 +31,15 @@ def chs(x): return c, h, s def read_chars_up_to_null(file): - while True: + keep_reading = True + while keep_reading: try: - c = file.read(1) + c = file.read(1).decode() if c == '\0': - raise StopIteration + keep_reading = False yield c except ValueError: - raise StopIteration + keep_reading = False def read_cstr(file): return ''.join(read_chars_up_to_null(file)) diff --git a/scripts/libosv.py b/scripts/libosv.py --- a/scripts/libosv.py +++ b/scripts/libosv.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import sys import re @@ -30,7 +30,7 @@ try: value, tp, bnd, sym = elf.match(line).groups() out.write("%s = 0x%s;\n"%(sym, value)) - print (".global %s\n.type %s,@%s"%(sym, sym, asm_type[tp])) + print(".global %s\n.type %s,@%s"%(sym, sym, asm_type[tp])) except AttributeError: pass diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py --- a/scripts/manifest_common.py +++ b/scripts/manifest_common.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import os, io, re, subprocess diff --git a/scripts/metadata.py b/scripts/metadata.py --- a/scripts/metadata.py +++ b/scripts/metadata.py @@ -1,7 +1,7 @@ -import SimpleHTTPServer as http -import SocketServer import subprocess import os +import http.server as http +import socketserver METADATA_IP = '169.254.169.254' port = 80 @@ -32,7 +32,7 @@ def start_server(path): try: os.chdir(path) handler = http.SimpleHTTPRequestHandler - server = SocketServer.TCPServer(("", port), handler, False) + server = socketserver.TCPServer(("", port), handler, False) server.allow_reuse_address = True server.server_bind() server.server_activate() diff --git a/scripts/mkbootfs.py b/scripts/mkbootfs.py --- a/scripts/mkbootfs.py +++ b/scripts/mkbootfs.py @@ -1,10 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/python3 import os, struct, optparse, io -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file def main(): @@ -80,7 +77,7 @@ def main(): if hostname.startswith("->"): link = hostname[2:] out.write(link.encode()) - out.write('\0') + out.write(b'\0') else: out.write(open(hostname, 'rb').read()) diff --git a/scripts/module.py b/scripts/module.py --- a/scripts/module.py +++ b/scripts/module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import re import os diff --git a/scripts/nbd_client.py b/scripts/nbd_client.py --- a/scripts/nbd_client.py +++ b/scripts/nbd_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (C) 2013 Nodalink, SARL. diff --git a/scripts/osv/modules/filemap.py b/scripts/osv/modules/filemap.py --- a/scripts/osv/modules/filemap.py +++ b/scripts/osv/modules/filemap.py @@ -19,15 +19,26 @@ def _pattern_to_regex(path): % bad_token.group(1)) path = '^' + re.escape(path) + '$' + # Python 3.7 and above does NOT escape '/' so we need to detect it and handle accordingly + slash_escaped = '\\/' in path # Normalize path - path = re.sub(r'(\\/)+', '\/', path) + if slash_escaped: + path = re.sub(r'(\\/)+', '\/', path) + else: + path = re.sub(r'(/)+', '/', path) # Merge consecutive **/ components - path = _reduce_path(path, r'\\\*\\\*\\/\\\*\\\*', '\*\*') + if slash_escaped: + path = _reduce_path(path, r'\\\*\\\*\\/\\\*\\\*', '\*\*') + else: + path = _reduce_path(path, r'\\\*\\\*/\\\*\\\*', '\*\*') # Transform ** component - path = re.sub(r'(\^|\\/)\\\*\\\*(\\/|\$)', '((^|\/).*($|\/|^))', path) + if slash_escaped: + path = re.sub(r'(\^|\\/)\\\*\\\*(\\/|\$)', '((^|\/).*($|\/|^))', path) + else: + path = re.sub(r'(\^|/)\\\*\\\*(/|\$)', '((^|/).*($|\|^))', path) path = path.replace('\\*', '[^/]*') path = path.replace('\\?', '.') diff --git a/scripts/run.py b/scripts/run.py --- a/scripts/run.py +++ b/scripts/run.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 + import subprocess import sys import argparse diff --git a/scripts/test.py b/scripts/test.py --- a/scripts/test.py +++ b/scripts/test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import atexit import subprocess import argparse @@ -81,7 +81,7 @@ def is_not_skipped(test): return test.name not in blacklist def run_tests_in_single_instance(): - run(filter(lambda test: not isinstance(test, TestRunnerTest), tests)) + run([test for test in tests if not isinstance(test, TestRunnerTest)]) blacklist_tests = ' '.join(blacklist) args = run_py_args + ["-s", "-e", "/testrunner.so -b %s" % (blacklist_tests)] @@ -103,7 +103,7 @@ def pluralize(word, count): def make_export_and_conf(): export_dir = tempfile.mkdtemp(prefix='share') - os.chmod(export_dir, 0777) + os.chmod(export_dir, 0o777) (conf_fd, conf_path) = tempfile.mkstemp(prefix='export') conf = os.fdopen(conf_fd, "w") conf.write("%s 127.0.0.1(insecure,rw)\n" % export_dir) @@ -155,12 +155,12 @@ def run_tests(): "/tst-nfs.so --server 192.168.122.1 --share %s" % export_dir) ] - line = proc.stdout.readline() + line = proc.stdout.readline().decode() while line: print(line) if "/tmp" in line: break - line = proc.stdout.readline() + line = proc.stdout.readline().decode() run(tests_to_run) diff --git a/scripts/tests/test_app.py b/scripts/tests/test_app.py --- a/scripts/tests/test_app.py +++ b/scripts/tests/test_app.py @@ -1,7 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 from testing import * import argparse -import subprocess from time import sleep def run(command, hypervisor_name, image_path=None, line=None, guest_port=None, host_port=None, input_lines=[], kill_app=False): diff --git a/scripts/tests/test_app_with_test_script.py b/scripts/tests/test_app_with_test_script.py --- a/scripts/tests/test_app_with_test_script.py +++ b/scripts/tests/test_app_with_test_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 from testing import * import argparse import runpy diff --git a/scripts/tests/test_http_app_with_curl_and_ab.py b/scripts/tests/test_http_app_with_curl_and_ab.py --- a/scripts/tests/test_http_app_with_curl_and_ab.py +++ b/scripts/tests/test_http_app_with_curl_and_ab.py @@ -1,11 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python3 from testing import * import argparse import subprocess from time import sleep def check_with_curl(url, expected_http_line): - output = subprocess.check_output(["curl", "-s", url]) + output = subprocess.check_output(["curl", "-s", url]).decode('utf-8') print(output) if expected_http_line not in output: print("FAILED curl: wrong output") @@ -39,9 +39,9 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt check_with_curl(app_url, expected_http_line) if no_keep_alive: - output = subprocess.check_output(["ab", "-l", "-c", str(concurrency), "-n", str(count), app_url]).split('\n') + output = subprocess.check_output(["ab", "-l", "-c", str(concurrency), "-n", str(count), app_url]).decode('utf-8').split('\n') else: - output = subprocess.check_output(["ab", "-l", "-k", "-c", str(concurrency), "-n", str(count), app_url]).split('\n') + output = subprocess.check_output(["ab", "-l", "-k", "-c", str(concurrency), "-n", str(count), app_url]).decode('utf-8').split('\n') failed_requests = 1 complete_requests = 0 @@ -74,11 +74,11 @@ def run(command, hypervisor_name, host_port, guest_port, http_path, expected_htt success = False if failed_requests > 0: - print("FAILED ab - encountered failed requests: %d" % failed_requests) + print("FAILED ab - encountered failed requests: %d" % failed_requests) success = False if complete_requests < count: - print("FAILED ab - too few complete requests : %d ? %d" % (complete_requests, count)) + print("FAILED ab - too few complete requests : %d ? %d" % (complete_requests, count)) success = False if success: diff --git a/scripts/tests/testing.py b/scripts/tests/testing.py --- a/scripts/tests/testing.py +++ b/scripts/tests/testing.py @@ -132,7 +132,7 @@ def append_line(line): self.cv.release() line = '' - ch_bytes = '' + ch_bytes = bytes() while True: ch_bytes = ch_bytes + self.process.stdout.read(1) try: @@ -143,7 +143,7 @@ def append_line(line): if ch == '\n': append_line(line) line = '' - ch_bytes = '' + ch_bytes = bytes() except UnicodeError: continue diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py --- a/scripts/upload_manifest.py +++ b/scripts/upload_manifest.py @@ -1,17 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python3 import optparse, os, subprocess, socket, threading, stat, sys from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file from contextlib import closing -try: - import StringIO - # This works on Python 2 - StringIO = StringIO.StringIO -except ImportError: - import io - # This works on Python 3 - StringIO = io.StringIO +import io +StringIO = io.StringIO def find_free_port(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: @@ -53,8 +47,7 @@ def cpio_send(data): def cpio_field(number, length): return ("%.*x" % (length, number)).encode() def cpio_header(filename, mode, filesize): - if sys.version_info >= (3, 0, 0): - filename = filename.encode("utf-8") + filename = filename.encode("utf-8") return (b"070701" # magic + cpio_field(0, 8) # inode + cpio_field(mode, 8) # mode -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/0000000000000a095d059eecffe4%40google.com.
