[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SVN (GH-1783) (GH-3306)
The Windows build now depends on Python 3.6 to fetch externals, but it
will be downloaded via NuGet (which is downloaded via PowerShell) if it
is not available via `py -3.6`. This means the only thing that must be
installed on a modern Windows box to do a full build of CPython with all
extensions is Visual Studio.
Cherry-picked from 51599e2bdd10ab77212a7cbb41a13ea70ee13da8, parts of 40a23e88994aca92c83c8e84ab8b8cdc11d7ec54, parts of 68d663cf85d1ac5eaf83482eed39c0a6f8093601, d5cd21d75a27a377f2f9c8370fd8e8c7efaeefb1, and possibly others that I've missed.
Also:
* Rename db -> bsddb for disambiguity
* Update sqlite3 to 3.14.2.0 since it's the version we use on 3.x, and it's simpler to just use it than to also upload the old version to cpython-source-deps
* Add PCbuild/*.ilk to .gitignore
diff --git a/.github/appveyor.yml b/.github/appveyor.yml
index 34c5e8b..b809ad6 100644
--- a/.github/appveyor.yml
+++ b/.github/appveyor.yml
@@ -10,6 +10,8 @@
- cmd: PCbuild\python.exe -m test.pythoninfo
test_script:
- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest -j2
+environment:
+ HOST_PYTHON: C:\Python36\python.exe
# Only trigger AppVeyor if actual code or its configuration changes
only_commits:
diff --git a/.gitignore b/.gitignore
index a8c73d4..a44b733 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,6 +44,7 @@
PC/*/Win32-temp-*
PC/*/x64-temp-*
PC/*/amd64
+PCbuild/*.ilk
PCbuild/*.user
PCbuild/*.suo
PCbuild/*.*sdf
diff --git a/Doc/make.bat b/Doc/make.bat
index 5319561..ae6224a 100644
--- a/Doc/make.bat
+++ b/Doc/make.bat
@@ -5,8 +5,19 @@
set this=%~n0
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
-if "%PYTHON%" EQU "" set PYTHON=py
+call ..\PCBuild\find_python.bat %PYTHON%
+if not defined SPHINXBUILD if defined PYTHON (
+ %PYTHON% -c "import sphinx" > nul 2> nul
+ if errorlevel 1 (
+ echo Installing sphinx with %PYTHON%
+ %PYTHON% -m pip install sphinx
+ if errorlevel 1 exit /B
+ )
+ set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
+)
+
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
@@ -73,7 +84,7 @@
if NOT "%PAPER%" == "" (
set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
)
-cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
+cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"
if "%1" EQU "htmlhelp" (
if not exist "%HTMLHELP%" (
diff --git a/Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst b/Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst
new file mode 100644
index 0000000..e7e06a1
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst
@@ -0,0 +1 @@
+Pull build dependencies from GitHub rather than svn.python.org.
diff --git a/PC/VS9.0/pyproject.vsprops b/PC/VS9.0/pyproject.vsprops
index 44b4f53..d4ced7b 100644
--- a/PC/VS9.0/pyproject.vsprops
+++ b/PC/VS9.0/pyproject.vsprops
@@ -50,7 +50,7 @@
/>
<UserMacro
Name="bsddb47Dir"
- Value="$(externalsDir)\db-4.7.25.0\build_windows"
+ Value="$(externalsDir)\bsddb-4.7.25.0\build_windows"
/>
<UserMacro
Name="bsddb47DepLibs"
@@ -74,7 +74,7 @@
/>
<UserMacro
Name="sqlite3Dir"
- Value="$(externalsDir)\sqlite-3.8.11.0"
+ Value="$(externalsDir)\sqlite-3.14.2.0"
/>
<UserMacro
Name="bz2Dir"
diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index a6ef551..e31056b 100644
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -5,12 +5,15 @@
echo.
echo.Build CPython from the command line. Requires the appropriate
echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
-echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
-echo.given.
echo.
echo.After the flags recognized by this script, up to 9 arguments to be passed
echo.directly to MSBuild may be passed. If the argument contains an '=', the
-echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
+echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
+echo.Alternatively you can put extra flags for MSBuild in a file named
+echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
+echo.will be picked automatically by MSBuild. Flags put in this file does not
+echo.need to be quoted. You can still use environment variables inside the
+echo.response file.
echo.
echo.Available flags:
echo. -h Display this help message
@@ -47,7 +50,6 @@
:Run
setlocal
set platf=Win32
-set vs_platf=x86
set conf=Release
set target=Build
set dir=%~dp0
@@ -56,10 +58,6 @@
set kill=
set do_pgo=
set pgo_job=-m test.regrtest --pgo
-set on_64_bit=true
-
-rem This may not be 100% accurate, but close enough.
-if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
:CheckOpts
if "%~1"=="-h" goto Usage
@@ -89,18 +87,12 @@
if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
-if "%platf%"=="x64" (
- if "%on_64_bit%"=="true" (
- rem This ought to always be correct these days...
- set vs_platf=amd64
- ) else (
- if "%do_pgo%"=="true" (
- echo.ERROR: Cannot cross-compile with PGO
- echo. 32bit operating system detected, if this is incorrect,
- echo. make sure the ProgramFiles(x86^) environment variable is set
- exit /b 1
- )
- set vs_platf=x86_amd64
+if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
+ if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
+ echo.ERROR: Cannot cross-compile with PGO
+ echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
+ echo. and PROCESSOR_ARCHITEW6432 environment variables are correct.
+ exit /b 1
)
)
@@ -109,13 +101,14 @@
if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
rem Setup the environment
-call "%dir%env.bat" %vs_platf% >nul
+call "%dir%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
if "%kill%"=="true" call :Kill
if "%do_pgo%"=="true" (
set conf=PGInstrument
- call :Build
+ call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
del /s "%dir%\*.pgc"
del /s "%dir%\..\Lib\*.pyc"
echo on
@@ -123,12 +116,13 @@
@echo off
call :Kill
set conf=PGUpdate
+ set target=Build
)
goto Build
:Kill
echo on
-msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
/p:Configuration=%conf% /p:Platform=%platf%^
/p:KillPython=true
@@ -140,7 +134,7 @@
rem Passing %1-9 is not the preferred option, but argument parsing in
rem batch is, shall we say, "lackluster"
echo on
-msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
+%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
/p:Configuration=%conf% /p:Platform=%platf%^
/p:IncludeExternals=%IncludeExternals%^
/p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^
diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat
new file mode 100644
index 0000000..2b7413f
--- /dev/null
+++ b/PCbuild/find_msbuild.bat
@@ -0,0 +1,52 @@
+@rem
+@rem Searches for MSBuild.exe. This is the only tool we need to initiate
+@rem a build, so we no longer search for the full VC toolset.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+@set MSBUILD=
+
+@rem If msbuild.exe is on the PATH, assume that the user wants that one.
+@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+ @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+ @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+
+@exit /b 1
+
+:found
+@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+@set _Py_MSBuild_Source=
diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
new file mode 100644
index 0000000..ba3a0f5
--- /dev/null
+++ b/PCbuild/find_python.bat
@@ -0,0 +1,75 @@
+@rem
+@rem Searches for python.exe and may download a private copy from nuget.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
+
+:begin_search
+@set PYTHON=
+
+@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
+@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
+
+@rem If we have Python in externals, use that one
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
+
+@rem If HOST_PYTHON is recent enough, use that
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+
+@rem If py.exe finds a recent enough version, use that one
+@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
+
+@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
+@set _Py_NUGET=%NUGET%
+@set _Py_NUGET_URL=%NUGET_URL%
+@set _Py_HOST_PYTHON=%HOST_PYTHON%
+@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
+@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
+@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
+@if NOT exist "%_Py_NUGET%" (
+ @echo Downloading nuget...
+ @rem NB: Must use single quotes around NUGET here, NOT double!
+ @rem Otherwise, a space in the path would break things
+ @rem If it fails, retry with any available copy of Python
+ @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
+ @if errorlevel 1 (
+ @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+ )
+)
+@echo Installing Python via nuget...
+@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
+@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
+
+
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=
+@exit /b 1
+
+:found
+@echo Using %PYTHON% (%_Py_Python_Source%)
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=
diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py
new file mode 100644
index 0000000..a682d38
--- /dev/null
+++ b/PCbuild/get_external.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import zipfile
+from urllib.request import urlretrieve
+
+
+def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
+ repo = f'cpython-{"bin" if binary else "source"}-deps'
+ url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
+ reporthook = None
+ if verbose:
+ reporthook = print
+ zip_dir.mkdir(parents=True, exist_ok=True)
+ filename, headers = urlretrieve(
+ url,
+ zip_dir / f'{commit_hash}.zip',
+ reporthook=reporthook,
+ )
+ return filename
+
+
+def extract_zip(externals_dir, zip_path):
+ with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+ zf.extractall(os.fspath(externals_dir))
+ return externals_dir / zf.namelist()[0].split('/')[0]
+
+
+def parse_args():
+ p = argparse.ArgumentParser()
+ p.add_argument('-v', '--verbose', action='store_true')
+ p.add_argument('-b', '--binary', action='store_true',
+ help='Is the dependency in the binary repo?')
+ p.add_argument('-O', '--organization',
+ help='Organization owning the deps repos', default='python')
+ p.add_argument('-e', '--externals-dir', type=pathlib.Path,
+ help='Directory in which to store dependencies',
+ default=pathlib.Path(__file__).parent.parent / 'externals')
+ p.add_argument('tag',
+ help='tag of the dependency')
+ return p.parse_args()
+
+
+def main():
+ args = parse_args()
+ zip_path = fetch_zip(
+ args.tag,
+ args.externals_dir / 'zips',
+ org=args.organization,
+ binary=args.binary,
+ verbose=args.verbose,
+ )
+ final_name = args.externals_dir / args.tag
+ extract_zip(args.externals_dir, zip_path).replace(final_name)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index ad42eb8..e83e1d4 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -2,51 +2,38 @@
setlocal
rem Simple script to fetch source for external libraries
-if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
-pushd "%~dp0..\externals"
+if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
+if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
-if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+set DO_FETCH=true
+set DO_CLEAN=false
-rem Optionally clean up first. Be warned that this can be very destructive!
-if not "%1"=="" (
- for %%c in (-c --clean --clean-only) do (
- if "%1"=="%%c" goto clean
- )
- goto usage
-)
-goto fetch
+:CheckOpts
+if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
+if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
+if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
+if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
+if "x%~1" NEQ "x" goto usage
+if "%DO_CLEAN%"=="false" goto fetch
:clean
echo.Cleaning up external libraries.
-for /D %%d in (
- bzip2-*
- db-*
- nasm-*
- openssl-*
- tcl-*
- tcltk*
- tk-*
- tix-*
- sqlite-*
- xz-*
- ) do (
- echo.Removing %%d
- rmdir /s /q %%d
-)
-if "%1"=="--clean-only" (
- goto end
+if exist "%EXTERNALS_DIR%" (
+ rem Sometimes this fails the first time; try it twice
+ rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
)
+if "%DO_FETCH%"=="false" goto end
:fetch
-rem Fetch current versions
-svn --version > nul 2>&1
-if ERRORLEVEL 9009 (
- echo.svn.exe must be on your PATH.
- echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
- echo.command line tools option.
- popd
- exit /b 1
+if "%ORG%"=="" (set ORG=python)
+call "%PCBUILD%find_python.bat" "%PYTHON%"
+
+if "%PYTHON%"=="" (
+ where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1
)
echo.Fetching external libraries...
@@ -56,52 +43,62 @@
set libraries=
set libraries=%libraries% bzip2-1.0.6
-if NOT "%IncludeBsddb%"=="false" set libraries=%libraries% db-4.7.25.0
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06
+if NOT "%IncludeBsddb%"=="false" set libraries=%libraries% bsddb-4.7.25.0
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k
-set libraries=%libraries% sqlite-3.8.11.0
+set libraries=%libraries% sqlite-3.14.2.0
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-8.5.15.0
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.5.15.0
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.5
for %%e in (%libraries%) do (
- if exist %%e (
+ if exist "%EXTERNALS_DIR%\%%e" (
echo.%%e already exists, skipping.
+ ) else if "%PYTHON%"=="" (
+ echo.Fetching %%e with git...
+ git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e"
) else (
echo.Fetching %%e...
- svn export -q %SVNROOT%%%e
+ %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e
)
)
+echo.Fetching external binaries...
+
+set binaries=
+set binaries=%binaries%
+if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06
+
+for %%b in (%binaries%) do (
+ if exist "%EXTERNALS_DIR%\%%b" (
+ echo.%%b already exists, skipping.
+ ) else if "%PYTHON%"=="" (
+ echo.Fetching %%b with git...
+ git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b"
+ ) else (
+ echo.Fetching %%b...
+ %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b
+ )
+)
+
+echo Finished.
goto end
:usage
-echo.invalid argument: %1
-echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.Valid options: -c, --clean, --clean-only, --organization, --python,
+echo.--no-tkinter, --no-openssl
echo.
-echo.Pull all sources necessary for compiling optional extension modules
-echo.that rely on external libraries. Requires svn.exe to be on your PATH
-echo.and pulls sources from %SVNROOT%.
+echo.Pull all sources and binaries necessary for compiling optional extension
+echo.modules that rely on external libraries.
echo.
-echo.Use the -c or --clean option to clean up all external library sources
-echo.before pulling in the current versions.
+echo.The --organization option determines which github organization to download
+echo.from, the --python option determines which Python 3.6+ interpreter to use
+echo.with PCbuild\get_external.py.
+echo.
+echo.Use the -c or --clean option to remove the entire externals directory.
echo.
echo.Use the --clean-only option to do the same cleaning, without pulling in
echo.anything new.
echo.
-echo.Only the first argument is checked, all others are ignored.
-echo.
-echo.**WARNING**: the cleaning options unconditionally remove any directory
-echo.that is a child of
-echo. %CD%
-echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
-echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
-echo.to be very destructive if you are not aware of what it is doing. Use with
-echo.caution!
-popd
exit /b -1
-
:end
-echo Finished.
-popd
diff --git a/PCbuild/python.props b/PCbuild/python.props
index 486a7c9..e0b0d82 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -32,9 +32,9 @@
<!-- Directories of external projects. tcltk is handled in tcltk.props -->
<ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
- <sqlite3Dir>$(ExternalsDir)sqlite-3.8.11.0\</sqlite3Dir>
+ <sqlite3Dir>$(ExternalsDir)sqlite-3.14.2.0\</sqlite3Dir>
<bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
- <bsddbDir>$(ExternalsDir)db-4.7.25.0</bsddbDir>
+ <bsddbDir>$(ExternalsDir)bsddb-4.7.25.0</bsddbDir>
<opensslDir>$(ExternalsDir)openssl-1.0.2k\</opensslDir>
<opensslIncludeDir>$(opensslDir)include32</opensslIncludeDir>
<opensslIncludeDir Condition="'$(ArchName)' == 'amd64'">$(opensslDir)include64</opensslIncludeDir>
diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
index 4380fed..c1ab6e9 100644
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -93,6 +93,8 @@
</ImportGroup>
<Target Name="GeneratePythonBat" AfterTargets="AfterBuild">
<PropertyGroup>
+ <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
+ <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
<_Content>@rem This script invokes the most recently built Python with all arguments
@rem passed through to the interpreter. This file is generated by the
@rem build process and any changes *will* be thrown away by the next
@@ -100,6 +102,9 @@
@rem This is only meant as a convenience for developing CPython
@rem and using it outside of that context is ill-advised.
@echo Running $(Configuration)^|$(Platform) interpreter...
+@setlocal
+@set PYTHONHOME=$(PySourcePath)
+$(_PGOPath)
@"$(OutDir)python$(PyDebugExt).exe" %*
</_Content>
<_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat'))</_ExistingContent>
diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt
index d05a503..a1777a6 100644
--- a/PCbuild/readme.txt
+++ b/PCbuild/readme.txt
@@ -4,9 +4,11 @@
1. Install Microsoft Visual Studio 2008, any edition.
2. Install Microsoft Visual Studio 2010, any edition, or Windows SDK 7.1
and any version of Microsoft Visual Studio newer than 2010.
-3. Install Subversion, and make sure 'svn.exe' is on your PATH.
-4. Run "build.bat -e" to build Python in 32-bit Release configuration.
-5. (Optional, but recommended) Run the test suite with "rt.bat -q".
+2a. Optionally install Python 3.6 or later. If not installed,
+ get_externals.bat (build.bat -e) will download and use Python via
+ NuGet.
+3. Run "build.bat -e" to build Python in 32-bit Release configuration.
+4. (Optional, but recommended) Run the test suite with "rt.bat -q".
Building Python using MSVC 9.0 via MSBuild
@@ -250,9 +252,16 @@
can be built. However, a simple script is provided to make this as
painless as possible, called "get_externals.bat" and located in this
directory. This script extracts all the external sub-projects from
- http://svn.python.org/projects/external
-via Subversion (so you'll need svn.exe on your PATH) and places them
-in ..\externals (relative to this directory).
+ https://github.com/python/cpython-source-deps
+and
+ https://github.com/python/cpython-bin-deps
+via a Python script called "get_external.py", located in this directory.
+If Python 3.6 or later is not available via the "py.exe" launcher, the
+path or command to use for Python can be provided in the PYTHON_FOR_BUILD
+environment variable, or get_externals.bat will download the latest
+version of NuGet and use it to download the latest "pythonx86" package
+for use with get_external.py. Everything downloaded by these scripts is
+stored in ..\externals (relative to this directory).
It is also possible to download sources from each project's homepage,
though you may have to change folder names or pass the names to MSBuild
diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat
index 7dcaa25..f4e15c7 100644
--- a/Tools/nuget/build.bat
+++ b/Tools/nuget/build.bat
@@ -21,6 +21,8 @@
if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1)
if not defined NUGET where nuget -q || echo Cannot find nuget.exe on PATH and NUGET is not set. && exit /B 1
+call "%PCBUILD%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
if not defined PYTHON set PYTHON=py -3
@%PYTHON% -c "" >nul 2>nul
@@ -29,7 +31,6 @@
set PYTHON="%D%obj\python\tools\python.exe"
)
-call "%PCBUILD%env.bat" x86
if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%"
@@ -38,7 +39,7 @@
) else if not exist "%PCBUILD%python.exe" call "%PCBUILD%build.bat" -e
if errorlevel 1 goto :eof
- msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
+ %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES%
if errorlevel 1 goto :eof
)
@@ -47,7 +48,7 @@
) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e
if errorlevel 1 goto :eof
- msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
+ %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES%
if errorlevel 1 goto :eof
)