Add Python3.5 artifact targets
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index 1c480f7..0ba9f3b 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -31,6 +31,8 @@
"""Definition of targets to build artifacts."""
import os.path
+import random
+import string
import sys
import jobset
@@ -79,27 +81,16 @@
'x64': '-m64'
}
-python_windows_version_arch_map = {
- ('x86', '2.7'): 'Python27_32bits',
- ('x64', '2.7'): 'Python27',
- ('x86', '3.4'): 'Python34_32bits',
- ('x64', '3.4'): 'Python34',
-}
class PythonArtifact:
"""Builds Python artifacts."""
- def __init__(self, platform, arch, python_version, manylinux_build=None):
- if manylinux_build:
- self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
- else:
- self.name = 'python%s_%s_%s' % (python_version, platform, arch)
+ def __init__(self, platform, arch, py_version):
+ self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
self.platform = platform
self.arch = arch
- self.labels = ['artifact', 'python', python_version, platform, arch]
- self.python_version = python_version
- self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
- self.manylinux_build = manylinux_build
+ self.labels = ['artifact', 'python', platform, arch, py_version]
+ self.py_version = py_version
def pre_build_jobspecs(self):
return []
@@ -111,8 +102,8 @@
environ['SETARCH_CMD'] = 'linux32'
# Inside the manylinux container, the python installations are located in
# special places...
- environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
- environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
+ environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version)
+ environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
# TODO(atash) get better platform-detection support in core so we don't
@@ -126,14 +117,24 @@
environ=environ,
timeout_seconds=60*60)
elif self.platform == 'windows':
+ if 'Python27' in self.py_version or 'Python34' in self.py_version:
+ environ['EXT_COMPILER'] = 'mingw32'
+ else:
+ environ['EXT_COMPILER'] = 'msvc'
+ # For some reason, the batch script %random% always runs with the same
+ # seed. We create a random temp-dir here
+ dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
- self.python_windows_prefix,
- '32' if self.arch == 'x86' else '64'
+ self.py_version,
+ '32' if self.arch == 'x86' else '64',
+ dir
],
+ environ=environ,
shell=True)
else:
- environ['PYTHON'] = 'python{}'.format(self.python_version)
+ environ['PYTHON'] = self.py_version
+ environ['SKIP_PIP_INSTALL'] = 'TRUE'
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@@ -330,18 +331,23 @@
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
- [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
- PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
- PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
- PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
- PythonArtifact('macos', 'x64', '2.7'),
- PythonArtifact('windows', 'x86', '2.7'),
- PythonArtifact('windows', 'x64', '2.7'),
- PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
- PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
- PythonArtifact('macos', 'x64', '3.4'),
- PythonArtifact('windows', 'x86', '3.4'),
- PythonArtifact('windows', 'x64', '3.4'),
+ [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x86', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x86', 'cp35-cp35m'),
+ PythonArtifact('linux', 'x64', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x64', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x64', 'cp35-cp35m'),
+ PythonArtifact('macos', 'x64', 'python2.7'),
+ PythonArtifact('macos', 'x64', 'python3.4'),
+ PythonArtifact('macos', 'x64', 'python3.5'),
+ PythonArtifact('windows', 'x86', 'Python27_32bits'),
+ PythonArtifact('windows', 'x86', 'Python34_32bits'),
+ PythonArtifact('windows', 'x86', 'Python35_32bits'),
+ PythonArtifact('windows', 'x64', 'Python27'),
+ PythonArtifact('windows', 'x64', 'Python34'),
+ PythonArtifact('windows', 'x64', 'Python35'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index d0b147b..246713a 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -34,56 +34,45 @@
pip install --upgrade setuptools
pip install -rrequirements.txt
-@rem Because this is windows and *everything seems to hate Windows* we have to
-@rem set all of these flags ourselves because Python won't help us (see the
-@rem setup.py of the grpcio_tools project).
-set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11
-
-@rem See https://sourceforge.net/p/mingw-w64/bugs/363/
-if %2 == 32 (
- set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s
-) else (
- set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64
-)
-
-@rem Further confusing things, MSYS2's mingw64 tries to dynamically link
-@rem libgcc, libstdc++, and winpthreads. We have to override this or our
-@rem extensions end up linking to MSYS2 DLLs, which the normal Python on
-@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default
-@rem behavior of linking msvcrt.dll as the C runtime library, which we need to
-@rem override so that Python's distutils doesn't link us against multiple C
-@rem runtimes.
-python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt
-set /p PYTHON_MSVCR=<temp.txt
-set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread
-
set GRPC_PYTHON_BUILD_WITH_CYTHON=1
+@rem Multiple builds are running simultaneously, so to avoid distutils
+@rem file collisions, we build everything in a tmp directory
+if not exist "artifacts" mkdir "artifacts"
+set ARTIFACT_DIR=%cd%\artifacts
+set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\
+mkdir %BUILD_DIR%
+xcopy /s/e/q %cd%\* %BUILD_DIR%
+pushd %BUILD_DIR%
+
@rem Set up gRPC Python tools
python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
pushd tools\distrib\python\grpcio_tools
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
popd
-
@rem Build gRPC Python distributions
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
pushd tools\distrib\python\grpcio_tools
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
popd
-mkdir artifacts
-xcopy /Y /I /S dist\* artifacts\ || goto :error
-xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* artifacts\ || goto :error
+xcopy /Y /I /S dist\* %ARTIFACT_DIR% || goto :error
+xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* %ARTIFACT_DIR% || goto :error
+
+popd
+rmdir /s /q %BUILD_DIR%
goto :EOF
:error
+popd
+rmdir /s /q %BUILD_DIR%
exit /b 1
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 8f8330e..9fed7c5 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -38,38 +38,42 @@
export PIP=${PIP:-pip}
export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
+# Because multiple builds run in parallel, some distutils file
+# operations may collide. To avoid this, each build is run in
+# a temp directory
+mkdir -p artifacts
+ARTIFACT_DIR="$PWD/artifacts"
+BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"`
+trap "rm -rf $BUILD_DIR" EXIT
+cp -r * "$BUILD_DIR"
+cd "$BUILD_DIR"
# Build the source distribution first because MANIFEST.in cannot override
# exclusion of built shared objects among package resources (for some
# inexplicable reason).
-${SETARCH_CMD} ${PYTHON} setup.py \
- sdist
+${SETARCH_CMD} ${PYTHON} setup.py sdist
# Wheel has a bug where directories don't get excluded.
# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
-${SETARCH_CMD} ${PYTHON} setup.py \
- bdist_wheel
+${SETARCH_CMD} ${PYTHON} setup.py bdist_wheel
# Build gRPC tools package distribution
${PYTHON} tools/distrib/python/make_grpcio_tools.py
# Build gRPC tools package source distribution
-${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py \
- sdist
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist
# Build gRPC tools package binary distribution
-CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
- ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
-mkdir -p artifacts
if [ "$BUILD_MANYLINUX_WHEEL" != "" ]
then
for wheel in dist/*.whl; do
- ${AUDITWHEEL} repair $wheel -w artifacts/
+ ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
rm $wheel
done
for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
- ${AUDITWHEEL} repair $wheel -w artifacts/
+ ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
rm $wheel
done
fi
@@ -81,14 +85,14 @@
if [ "$BUILD_HEALTH_CHECKING" != "" ]
then
${PIP} install -rrequirements.txt
- ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/"
- ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/"
+ ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
+ ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
# Build gRPC health check source distribution
${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
preprocess build_package_protos sdist
- cp -r src/python/grpcio_health_checking/dist/* artifacts
+ cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR"
fi
-cp -r dist/* artifacts
-cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
+cp -r dist/* "$ARTIFACT_DIR"
+cp -r tools/distrib/python/grpcio_tools/dist/* "$ARTIFACT_DIR"