Christian Heimes | cb3558d | 2013-12-16 14:35:39 +0100 | [diff] [blame] | 1 | #./python |
| 2 | """Run Python tests with multiple installations of OpenSSL |
| 3 | |
| 4 | The script |
| 5 | |
| 6 | (1) downloads OpenSSL tar bundle |
| 7 | (2) extracts it to ../openssl/src/openssl-VERSION/ |
| 8 | (3) compiles OpenSSL |
| 9 | (4) installs OpenSSL into ../openssl/VERSION/ |
| 10 | (5) forces a recompilation of Python modules using the |
| 11 | header and library files from ../openssl/VERSION/ |
| 12 | (6) runs Python's test suite |
| 13 | |
| 14 | The script must be run with Python's build directory as current working |
| 15 | directory: |
| 16 | |
| 17 | ./python Tools/ssl/test_multiple_versions.py |
| 18 | |
| 19 | The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend |
| 20 | search paths for header files and shared libraries. It's known to work on |
| 21 | Linux with GCC 4.x. |
| 22 | |
| 23 | (c) 2013 Christian Heimes <christian@python.org> |
| 24 | """ |
| 25 | import logging |
| 26 | import os |
| 27 | import tarfile |
| 28 | import shutil |
| 29 | import subprocess |
| 30 | import sys |
| 31 | from urllib.request import urlopen |
| 32 | |
| 33 | log = logging.getLogger("multissl") |
| 34 | |
| 35 | OPENSSL_VERSIONS = [ |
| 36 | "0.9.7m", "0.9.8i", "0.9.8l", "0.9.8m", "0.9.8y", "1.0.0k", "1.0.1e" |
| 37 | ] |
| 38 | FULL_TESTS = [ |
| 39 | "test_asyncio", "test_ftplib", "test_hashlib", "test_httplib", |
| 40 | "test_imaplib", "test_nntplib", "test_poplib", "test_smtplib", |
| 41 | "test_smtpnet", "test_urllib2_localnet", "test_venv" |
| 42 | ] |
| 43 | MINIMAL_TESTS = ["test_ssl", "test_hashlib"] |
| 44 | CADEFAULT = True |
| 45 | HERE = os.path.abspath(os.getcwd()) |
| 46 | DEST_DIR = os.path.abspath(os.path.join(HERE, os.pardir, "openssl")) |
| 47 | |
| 48 | |
| 49 | class BuildSSL: |
| 50 | url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" |
| 51 | |
| 52 | module_files = ["Modules/_ssl.c", |
| 53 | "Modules/socketmodule.c", |
| 54 | "Modules/_hashopenssl.c"] |
| 55 | |
| 56 | def __init__(self, version, openssl_compile_args=(), destdir=DEST_DIR): |
| 57 | self._check_python_builddir() |
| 58 | self.version = version |
| 59 | self.openssl_compile_args = openssl_compile_args |
| 60 | # installation directory |
| 61 | self.install_dir = os.path.join(destdir, version) |
| 62 | # source file |
| 63 | self.src_file = os.path.join(destdir, "src", |
| 64 | "openssl-{}.tar.gz".format(version)) |
| 65 | # build directory (removed after install) |
| 66 | self.build_dir = os.path.join(destdir, "src", |
| 67 | "openssl-{}".format(version)) |
| 68 | |
| 69 | @property |
| 70 | def openssl_cli(self): |
| 71 | """openssl CLI binary""" |
| 72 | return os.path.join(self.install_dir, "bin", "openssl") |
| 73 | |
| 74 | @property |
| 75 | def openssl_version(self): |
| 76 | """output of 'bin/openssl version'""" |
| 77 | env = os.environ.copy() |
| 78 | env["LD_LIBRARY_PATH"] = self.lib_dir |
| 79 | cmd = [self.openssl_cli, "version"] |
| 80 | return self._subprocess_output(cmd, env=env) |
| 81 | |
| 82 | @property |
| 83 | def pyssl_version(self): |
| 84 | """Value of ssl.OPENSSL_VERSION""" |
| 85 | env = os.environ.copy() |
| 86 | env["LD_LIBRARY_PATH"] = self.lib_dir |
| 87 | cmd = ["./python", "-c", "import ssl; print(ssl.OPENSSL_VERSION)"] |
| 88 | return self._subprocess_output(cmd, env=env) |
| 89 | |
| 90 | @property |
| 91 | def include_dir(self): |
| 92 | return os.path.join(self.install_dir, "include") |
| 93 | |
| 94 | @property |
| 95 | def lib_dir(self): |
| 96 | return os.path.join(self.install_dir, "lib") |
| 97 | |
| 98 | @property |
| 99 | def has_openssl(self): |
| 100 | return os.path.isfile(self.openssl_cli) |
| 101 | |
| 102 | @property |
| 103 | def has_src(self): |
| 104 | return os.path.isfile(self.src_file) |
| 105 | |
| 106 | def _subprocess_call(self, cmd, stdout=subprocess.DEVNULL, env=None, |
| 107 | **kwargs): |
| 108 | log.debug("Call '{}'".format(" ".join(cmd))) |
| 109 | return subprocess.check_call(cmd, stdout=stdout, env=env, **kwargs) |
| 110 | |
| 111 | def _subprocess_output(self, cmd, env=None, **kwargs): |
| 112 | log.debug("Call '{}'".format(" ".join(cmd))) |
| 113 | out = subprocess.check_output(cmd, env=env) |
| 114 | return out.strip().decode("utf-8") |
| 115 | |
| 116 | def _check_python_builddir(self): |
| 117 | if not os.path.isfile("python") or not os.path.isfile("setup.py"): |
| 118 | raise ValueError("Script must be run in Python build directory") |
| 119 | |
| 120 | def _download_openssl(self): |
| 121 | """Download OpenSSL source dist""" |
| 122 | src_dir = os.path.dirname(self.src_file) |
| 123 | if not os.path.isdir(src_dir): |
| 124 | os.makedirs(src_dir) |
| 125 | url = self.url_template.format(self.version) |
| 126 | log.info("Downloading OpenSSL from {}".format(url)) |
| 127 | req = urlopen(url, cadefault=CADEFAULT) |
| 128 | # KISS, read all, write all |
| 129 | data = req.read() |
| 130 | log.info("Storing {}".format(self.src_file)) |
| 131 | with open(self.src_file, "wb") as f: |
| 132 | f.write(data) |
| 133 | |
| 134 | def _unpack_openssl(self): |
| 135 | """Unpack tar.gz bundle""" |
| 136 | # cleanup |
| 137 | if os.path.isdir(self.build_dir): |
| 138 | shutil.rmtree(self.build_dir) |
| 139 | os.makedirs(self.build_dir) |
| 140 | |
| 141 | tf = tarfile.open(self.src_file) |
| 142 | base = "openssl-{}/".format(self.version) |
| 143 | # force extraction into build dir |
| 144 | members = tf.getmembers() |
| 145 | for member in members: |
| 146 | if not member.name.startswith(base): |
| 147 | raise ValueError(member.name) |
| 148 | member.name = member.name[len(base):] |
| 149 | log.info("Unpacking files to {}".format(self.build_dir)) |
| 150 | tf.extractall(self.build_dir, members) |
| 151 | |
| 152 | def _build_openssl(self): |
| 153 | """Now build openssl""" |
| 154 | log.info("Running build in {}".format(self.install_dir)) |
| 155 | cwd = self.build_dir |
| 156 | cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] |
| 157 | cmd.extend(self.openssl_compile_args) |
| 158 | self._subprocess_call(cmd, cwd=cwd) |
| 159 | self._subprocess_call(["make"], cwd=cwd) |
| 160 | |
| 161 | def _install_openssl(self, remove=True): |
| 162 | self._subprocess_call(["make", "install"], cwd=self.build_dir) |
| 163 | if remove: |
| 164 | shutil.rmtree(self.build_dir) |
| 165 | |
| 166 | def install_openssl(self): |
| 167 | if not self.has_openssl: |
| 168 | if not self.has_src: |
| 169 | self._download_openssl() |
| 170 | else: |
| 171 | log.debug("Already has src {}".format(self.src_file)) |
| 172 | self._unpack_openssl() |
| 173 | self._build_openssl() |
| 174 | self._install_openssl() |
| 175 | else: |
| 176 | log.info("Already has installation {}".format(self.install_dir)) |
| 177 | # validate installation |
| 178 | version = self.openssl_version |
| 179 | if self.version not in version: |
| 180 | raise ValueError(version) |
| 181 | |
| 182 | def touch_pymods(self): |
| 183 | # force a rebuild of all modules that use OpenSSL APIs |
| 184 | for fname in self.module_files: |
| 185 | os.utime(fname) |
| 186 | |
| 187 | def recompile_pymods(self): |
| 188 | log.info("Using OpenSSL build from {}".format(self.build_dir)) |
| 189 | # overwrite header and library search paths |
| 190 | env = os.environ.copy() |
| 191 | env["CPPFLAGS"] = "-I{}".format(self.include_dir) |
| 192 | env["LDFLAGS"] = "-L{}".format(self.lib_dir) |
| 193 | # set rpath |
| 194 | env["LD_RUN_PATH"] = self.lib_dir |
| 195 | |
| 196 | log.info("Rebuilding Python modules") |
| 197 | self.touch_pymods() |
| 198 | cmd = ["./python", "setup.py", "build"] |
| 199 | self._subprocess_call(cmd, env=env) |
| 200 | |
| 201 | def check_pyssl(self): |
| 202 | version = self.pyssl_version |
| 203 | if self.version not in version: |
| 204 | raise ValueError(version) |
| 205 | |
| 206 | def run_pytests(self, *args): |
| 207 | cmd = ["./python", "-m", "test"] |
| 208 | cmd.extend(args) |
| 209 | self._subprocess_call(cmd, stdout=None) |
| 210 | |
| 211 | def run_python_tests(self, *args): |
| 212 | self.recompile_pymods() |
| 213 | self.check_pyssl() |
| 214 | self.run_pytests(*args) |
| 215 | |
| 216 | |
| 217 | def main(*args): |
| 218 | builders = [] |
| 219 | for version in OPENSSL_VERSIONS: |
| 220 | if version in ("0.9.8i", "0.9.8l"): |
| 221 | openssl_compile_args = ("no-asm",) |
| 222 | else: |
| 223 | openssl_compile_args = () |
| 224 | builder = BuildSSL(version, openssl_compile_args) |
| 225 | builder.install_openssl() |
| 226 | builders.append(builder) |
| 227 | |
| 228 | for builder in builders: |
| 229 | builder.run_python_tests(*args) |
| 230 | # final touch |
| 231 | builder.touch_pymods() |
| 232 | |
| 233 | |
| 234 | if __name__ == "__main__": |
| 235 | logging.basicConfig(level=logging.INFO, |
| 236 | format="*** %(levelname)s %(message)s") |
| 237 | args = sys.argv[1:] |
| 238 | if not args: |
| 239 | args = ["-unetwork", "-v"] |
| 240 | args.extend(FULL_TESTS) |
| 241 | main(*args) |