bpo-30450: Pull Windows dependencies from GitHub rather than svn (GH-1783)
The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`. This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio.
Also fixes an outdated note about _lzma in PCbuild/readme.txt
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 5767ab2..6e466a3 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -2,58 +2,59 @@
setlocal
rem Simple script to fetch source for external libraries
-if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
-pushd "%~dp0..\externals"
+if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
+if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
+if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
+if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
-if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
+set DO_FETCH=true
+set DO_CLEAN=false
-rem Optionally clean up first. Be warned that this can be very destructive!
-if not "%1"=="" (
- for %%c in (-c --clean --clean-only) do (
- if "%1"=="%%c" goto clean
- )
- goto usage
-)
-goto fetch
+:CheckOpts
+if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
+if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
+if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
+if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
+if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
+if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
+if "x%~1" NEQ "x" goto usage
+if "%DO_CLEAN%"=="false" goto fetch
:clean
echo.Cleaning up external libraries.
-for /D %%d in (
- bzip2-*
- db-*
- nasm-*
- openssl-*
- tcl-*
- tcltk*
- tk-*
- tix-*
- sqlite-*
- xz-*
- ) do (
- echo.Removing %%d
- rmdir /s /q %%d
-)
-if "%1"=="--clean-only" (
- goto end
+if exist "%EXTERNALS_DIR%" (
+ rem Sometimes this fails the first time; try it twice
+ rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
)
+if "%DO_FETCH%"=="false" goto end
:fetch
-rem Fetch current versions
-svn --version > nul 2>&1
-if ERRORLEVEL 9009 (
- echo.svn.exe must be on your PATH.
- echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
- echo.command line tools option.
- popd
- exit /b 1
+if "%ORG%"=="" (set ORG=python)
+
+if "%PYTHON_FOR_BUILD%"=="" (
+ echo Checking for installed python...
+ py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
+)
+if "%PYTHON_FOR_BUILD%"=="" (
+ if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
+ if NOT exist "%NUGET%" (
+ echo Downloading nuget...
+ rem NB: Must use single quotes around NUGET here, NOT double!
+ rem Otherwise, a space in the path would break things
+ powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
+ )
+ echo Installing Python via nuget...
+ "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
+ rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+ set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
)
echo.Fetching external libraries...
set libraries=
set libraries=%libraries% bzip2-1.0.6
-if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k
set libraries=%libraries% sqlite-3.14.2.0
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.6.0
@@ -62,43 +63,48 @@
set libraries=%libraries% xz-5.2.2
for %%e in (%libraries%) do (
- if exist %%e (
+ if exist "%EXTERNALS_DIR%\%%e" (
echo.%%e already exists, skipping.
) else (
echo.Fetching %%e...
- svn export -q %SVNROOT%%%e
+ %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
)
)
+echo.Fetching external binaries...
+
+set binaries=
+set binaries=%binaries%
+if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06
+
+for %%b in (%binaries%) do (
+ if exist "%EXTERNALS_DIR%\%%b" (
+ echo.%%b already exists, skipping.
+ ) else (
+ echo.Fetching %%b...
+ %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
+ )
+)
+
+echo Finished.
goto end
:usage
-echo.invalid argument: %1
-echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
+echo.Valid options: -c, --clean, --clean-only, --organization, --python,
+echo.--no-tkinter, --no-openssl
echo.
-echo.Pull all sources necessary for compiling optional extension modules
-echo.that rely on external libraries. Requires svn.exe to be on your PATH
-echo.and pulls sources from %SVNROOT%.
+echo.Pull all sources and binaries necessary for compiling optional extension
+echo.modules that rely on external libraries.
echo.
-echo.Use the -c or --clean option to clean up all external library sources
-echo.before pulling in the current versions.
+echo.The --organization option determines which github organization to download
+echo.from, the --python option determines which Python 3.6+ interpreter to use
+echo.with PCbuild\get_external.py.
+echo.
+echo.Use the -c or --clean option to remove the entire externals directory.
echo.
echo.Use the --clean-only option to do the same cleaning, without pulling in
echo.anything new.
echo.
-echo.Only the first argument is checked, all others are ignored.
-echo.
-echo.**WARNING**: the cleaning options unconditionally remove any directory
-echo.that is a child of
-echo. %CD%
-echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
-echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
-echo.to be very destructive if you are not aware of what it is doing. Use with
-echo.caution!
-popd
exit /b -1
-
:end
-echo Finished.
-popd