pw_env_setup: Change how files are passed in
Change how files are passed into pw_env_setup. In most cases they're
passed in with --use-pigweed-defaults, but downstream projects now have
the option to not include Pigweed's defaults.
Change-Id: I82383705e156be14276a8498648ca376e3340efb
Bug: 274
Requires: pigweed-internal:7120
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/19380
Commit-Queue: Rob Mohr <mohrr@google.com>
Reviewed-by: Joe Ethier <jethier@google.com>
Reviewed-by: Michael Spang <spang@google.com>
diff --git a/bootstrap.bat b/bootstrap.bat
index 9532775..c4e5265 100644
--- a/bootstrap.bat
+++ b/bootstrap.bat
@@ -69,16 +69,6 @@
)
set "shell_file=%_PW_ACTUAL_ENVIRONMENT_ROOT%\activate.bat"
-set _PW_OLD_CIPD_PACKAGE_FILES=%PW_CIPD_PACKAGE_FILES%
-set _PW_OLD_VIRTUALENV_REQUIREMENTS=%PW_VIRTUALENV_REQUIREMENTS%
-set _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS=%PW_VIRTUALENV_SETUP_PY_ROOTS%
-set _PW_OLD_CARGO_PACKAGE_FILES=%PW_CARGO_PACKAGE_FILES%
-
-set PW_CIPD_PACKAGE_FILES=%PW_ROOT%\pw_env_setup\py\pw_env_setup\cipd_setup\pigweed.json;%PW_ROOT%\pw_env_setup\py\pw_env_setup\cipd_setup\luci.json;%PW_CIPD_PACKAGE_FILES%
-set PW_VIRTUALENV_REQUIREMENTS=%PW_ROOT%\pw_env_setup\py\pw_env_setup\virtualenv_setup\requirements.txt;%PW_VIRTUALENV_REQUIREMENTS%
-set PW_VIRTUALENV_SETUP_PY_ROOTS=%PW_ROOT%;%PW_VIRTUALENV_SETUP_PY_ROOTS%
-set PW_CARGO_PACKAGE_FILES=%PW_ROOT%\pw_env_setup\py\pw_env_setup\cargo_setup\packages.txt;%PW_CARGO_PACKAGE_FILES%
-
set "_pw_start_script=%PW_ROOT%\pw_env_setup\py\pw_env_setup\windows_env_start.py"
:: If PW_SKIP_BOOTSTRAP is set, only run the activation stage instead of the
@@ -89,7 +79,8 @@
call "%python%" "%PW_ROOT%\pw_env_setup\py\pw_env_setup\env_setup.py" ^
--pw-root "%PW_ROOT%/" ^
--shell-file "%shell_file%" ^
- --install-dir "%_PW_ACTUAL_ENVIRONMENT_ROOT%"
+ --install-dir "%_PW_ACTUAL_ENVIRONMENT_ROOT%" ^
+ --use-pigweed-defaults
) else (
if exist "%shell_file%" (
call "%python%" "%_pw_start_script%"
@@ -99,11 +90,6 @@
)
)
-set PW_CIPD_PACKAGE_FILES=%_PW_OLD_CIPD_PACKAGE_FILES%
-set PW_VIRTUALENV_REQUIREMENTS=%_PW_OLD_VIRTUALENV_REQUIREMENTS%
-set PW_VIRTUALENV_SETUP_PY_ROOTS=%_PW_OLD_VIRTUALENV_SETUP_PY_ROOTS%
-set PW_CARGO_PACKAGE_FILES=%_PW_OLD_CARGO_PACKAGE_FILES%
-
call "%shell_file%"
:finish
diff --git a/bootstrap.sh b/bootstrap.sh
index 0eac2b1..6089f47 100644
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -14,88 +14,10 @@
# This script must be tested on bash, zsh, and dash.
-_pw_abspath () {
+_bootstrap_abspath () {
python -c "import os.path; print(os.path.abspath('$@'))"
}
-
-# Note: Colors are unfortunately duplicated in several places; and removing the
-# duplication is not easy. Their locations are:
-#
-# - bootstrap.sh
-# - pw_cli/color.py
-# - pw_env_setup/py/pw_env_setup/colors.py
-#
-# So please keep them matching then modifying them.
-_pw_none() {
- echo -e "$*"
-}
-
-_pw_red() {
- echo -e "\033[0;31m$*\033[0m"
-}
-
-_pw_bold_red() {
- echo -e "\033[1;31m$*\033[0m"
-}
-
-_pw_yellow() {
- echo -e "\033[0;33m$*\033[0m"
-}
-
-_pw_bold_yellow() {
- echo -e "\033[1;33m$*\033[0m"
-}
-
-_pw_green() {
- echo -e "\033[0;32m$*\033[0m"
-}
-
-_pw_bold_green() {
- echo -e "\033[1;32m$*\033[0m"
-}
-
-_pw_blue() {
- echo -e "\033[1;34m$*\033[0m"
-}
-
-_pw_cyan() {
- echo -e "\033[1;36m$*\033[0m"
-}
-
-_pw_magenta() {
- echo -e "\033[0;35m$*\033[0m"
-}
-
-_pw_bold_white() {
- echo -e "\033[1;37m$*\033[0m"
-}
-
-# Note: This banner is duplicated in three places; which is a lesser evil than
-# the contortions that would be needed to share this snippet acros shell,
-# batch, and Python. Locations:
-#
-# - bootstrap.sh
-# - pw_cli/branding.py
-# - pw_env_setup/py/pw_env_setup/windows_env_start.py
-#
-_PW_BANNER=$(cat <<EOF
- ▒█████▄ █▓ ▄███▒ ▒█ ▒█ ░▓████▒ ░▓████▒ ▒▓████▄
- ▒█░ █░ ░█▒ ██▒ ▀█▒ ▒█░ █ ▒█ ▒█ ▀ ▒█ ▀ ▒█ ▀█▌
- ▒█▄▄▄█░ ░█▒ █▓░ ▄▄░ ▒█░ █ ▒█ ▒███ ▒███ ░█ █▌
- ▒█▀ ░█░ ▓█ █▓ ░█░ █ ▒█ ▒█ ▄ ▒█ ▄ ░█ ▄█▌
- ▒█ ░█░ ░▓███▀ ▒█▓▀▓█░ ░▓████▒ ░▓████▒ ▒▓████▀
-EOF
-)
-
-# Support customizing the branding with a different banner and color.
-if test -f "$PW_BRANDING_BANNER"; then
- _PW_BANNER=$(cat $PW_BRANDING_BANNER)
-fi
-if test -z "$PW_BRANDING_BANNER_COLOR"; then
- PW_BRANDING_BANNER_COLOR=magenta
-fi
-
# Users are not expected to set PW_CHECKOUT_ROOT, it's only used because it
# seems to be impossible to reliably determine the path to a sourced file in
# dash when sourced from a dash script instead of a dash interactive prompt.
@@ -104,21 +26,23 @@
# variable set.
# TODO(mohrr) find out a way to do this without PW_CHECKOUT_ROOT.
if test -n "$PW_CHECKOUT_ROOT"; then
- PW_SETUP_SCRIPT_PATH="$(_pw_abspath "$PW_CHECKOUT_ROOT/bootstrap.sh")"
+ _BOOTSTRAP_PATH="$(_bootstrap_abspath "$PW_CHECKOUT_ROOT/bootstrap.sh")"
+ # Downstream projects need to set PW_CHECKOUT_ROOT to point to Pigweed if
+ # they're using Pigweed's CI/CQ system.
unset PW_CHECKOUT_ROOT
# Shell: bash.
elif test -n "$BASH"; then
- PW_SETUP_SCRIPT_PATH="$(_pw_abspath "$BASH_SOURCE")"
+ _BOOTSTRAP_PATH="$(_bootstrap_abspath "$BASH_SOURCE")"
# Shell: zsh.
elif test -n "$ZSH_NAME"; then
- PW_SETUP_SCRIPT_PATH="$(_pw_abspath "${(%):-%N}")"
+ _BOOTSTRAP_PATH="$(_bootstrap_abspath "${(%):-%N}")"
# Shell: dash.
elif test ${0##*/} = dash; then
- PW_SETUP_SCRIPT_PATH="$(_pw_abspath \
+ _BOOTSTRAP_PATH="$(_bootstrap_abspath \
"$(lsof -p $$ -Fn0 | tail -1 | sed 's#^[^/]*##;')")"
# If everything else fails, try $0. It could work.
else
- PW_SETUP_SCRIPT_PATH="$(_pw_abspath "$0")"
+ _BOOTSTRAP_PATH="$(_bootstrap_abspath "$0")"
fi
# Check if this file is being executed or sourced.
@@ -139,148 +63,36 @@
case ${0##*/} in sh|dash) _pw_sourced=1;; esac
fi
-if [ "$_pw_sourced" -eq 0 ]; then
- _PW_NAME=$(basename "$PW_SETUP_SCRIPT_PATH" .sh)
- _pw_bold_red "Error: Attempting to $_PW_NAME in a subshell"
- _pw_red " Since $_PW_NAME.sh modifies your shell's environment variables, it"
- _pw_red " must be sourced rather than executed. In particular, "
- _pw_red " 'bash $_PW_NAME.sh' will not work since the modified environment "
- _pw_red " will get destroyed at the end of the script. Instead, source the "
- _pw_red " script's contents in your shell:"
- _pw_red ""
- _pw_red " \$ source $_PW_NAME.sh"
- exit 1
-fi
-
-PW_ROOT="$(dirname "$PW_SETUP_SCRIPT_PATH")"
-
-if [[ "$PW_ROOT" = *" "* ]]; then
- _pw_bold_red "Error: The Pigweed path contains spaces\n"
- _pw_red " The path '$PW_ROOT' contains spaces. "
- _pw_red " Pigweed's Python environment currently requires Pigweed to be "
- _pw_red " at a path without spaces. Please checkout Pigweed in a directory "
- _pw_red " without spaces and retry running bootstrap."
- return
-fi
-
+# Downstream projects need to set something other than PW_ROOT here, like
+# YOUR_PROJECT_ROOT. Please also set PW_ROOT before invoking pw_bootstrap or
+# pw_activate.
+PW_ROOT="$(dirname "$_BOOTSTRAP_PATH")"
export PW_ROOT
-# PW_ENVIRONMENT_ROOT allows developers to specify where the environment should
-# be installed. _PW_ACTUAL_ENVIRONMENT_ROOT is where Pigweed keeps that
-# information. This separation allows Pigweed to assume PW_ENVIRONMENT_ROOT
-# came from the developer and not from a previous bootstrap possibly from
-# another workspace.
-if [ -z "$PW_ENVIRONMENT_ROOT" ]; then
- _PW_ACTUAL_ENVIRONMENT_ROOT="$PW_ROOT/.environment"
- export _PW_ACTUAL_ENVIRONMENT_ROOT
-else
- _PW_ACTUAL_ENVIRONMENT_ROOT="$PW_ENVIRONMENT_ROOT"
- export _PW_ACTUAL_ENVIRONMENT_ROOT
-fi
+. "$PW_ROOT/pw_env_setup/util.sh"
+
+pw_eval_sourced "$_pw_sourced"
+pw_check_root "$PW_ROOT"
+_PW_ACTUAL_ENVIRONMENT_ROOT="$(pw_get_env_root)"
SETUP_SH="$_PW_ACTUAL_ENVIRONMENT_ROOT/activate.sh"
-if [ -z "$PW_ENVSETUP_QUIET" ] && [ -z "$PW_ENVSETUP_NO_BANNER" ]; then
- _pw_green "\n WELCOME TO...\n"
- "_pw_$PW_BRANDING_BANNER_COLOR" "$_PW_BANNER\n"
-fi
+# Downstream projects may wish to set PW_BANNER_FUNC to a function that prints
+# an ASCII art banner here.
# Run full bootstrap when invoked as bootstrap, or env file is missing/empty.
-if [ "$(basename "$PW_SETUP_SCRIPT_PATH")" = "bootstrap.sh" ] || \
+if [ "$(basename "$_BOOTSTRAP_PATH")" = "bootstrap.sh" ] || \
[ ! -f "$SETUP_SH" ] || \
[ ! -s "$SETUP_SH" ]; then
- _PW_IS_BOOTSTRAP=0
+ pw_bootstrap --shell-file "$SETUP_SH" --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" --use-pigweed-defaults
+ pw_finalize bootstrap "$SETUP_SH"
else
- _PW_IS_BOOTSTRAP=1
+ pw_activate
+ pw_finalize activate "$SETUP_SH"
fi
-if [ "$_PW_IS_BOOTSTRAP" -eq 0 ]; then
- _PW_NAME="bootstrap"
-
- if [ -z "$PW_ENVSETUP_QUIET" ]; then
- _pw_green " BOOTSTRAP! Bootstrap may take a few minutes; please be patient.\n"
- fi
-
- # Allow forcing a specific version of Python for testing pursposes.
- if [ -n "$PW_BOOTSTRAP_PYTHON" ]; then
- PYTHON="$PW_BOOTSTRAP_PYTHON"
- elif which python &> /dev/null; then
- PYTHON=python
- else
- _pw_bold_red "Error: No system Python present\n"
- _pw_red " Pigweed's bootstrap process requires a local system Python."
- _pw_red " Please install Python on your system, add it to your PATH"
- _pw_red " and re-try running bootstrap."
- return
- fi
-
- _PW_OLD_CIPD_PACKAGE_FILES="$PW_CIPD_PACKAGE_FILES"
- PW_CIPD_PACKAGE_FILES="$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json:$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json:$PW_CIPD_PACKAGE_FILES"
- export PW_CIPD_PACKAGE_FILES
-
- if [ -z "$PW_VIRTUALENV_REQUIREMENTS" -o -n "$PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT" ]; then
- _PW_OLD_VIRTUALENV_REQUIREMENTS="$PW_VIRTUALENV_REQUIREMENTS"
- PW_VIRTUALENV_REQUIREMENTS="$PW_ROOT/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt:$PW_VIRTUALENV_REQUIREMENTS"
- export PW_VIRTUALENV_REQUIREMENTS
- fi
-
- _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS="$PW_VIRTUALENV_SETUP_PY_ROOTS"
- PW_VIRTUALENV_SETUP_PY_ROOTS="$PW_ROOT/*:$PW_VIRTUALENV_SETUP_PY_ROOTS"
- export PW_VIRTUALENV_SETUP_PY_ROOTS
-
- _PW_OLD_CARGO_PACKAGE_FILES="$PW_CARGO_PACKAGE_FILES"
- PW_CARGO_PACKAGE_FILES="$PW_ROOT/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt:$PW_CARGO_PACKAGE_FILES"
- export PW_CARGO_PACKAGE_FILES
-
- if [ -n "$PW_USE_GCS_ENVSETUP" ]; then
- _PW_ENV_SETUP="$("$PW_ROOT/pw_env_setup/get_pw_env_setup.sh")"
- fi
-
- if [ -n "$_PW_ENV_SETUP" ]; then
- "$_PW_ENV_SETUP" --shell-file "$SETUP_SH" --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT"
- else
- "$PYTHON" "$PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py" --shell-file "$SETUP_SH" --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT"
- fi
-
- PW_CIPD_PACKAGE_FILES="$_PW_OLD_CIPD_PACKAGE_FILES"
- PW_VIRTUALENV_REQUIREMENTS="$_PW_OLD_VIRTUALENV_REQUIREMENTS"
- PW_VIRTUALENV_SETUP_PY_ROOTS="$_PW_OLD_VIRTUALENV_SETUP_PY_ROOTS"
- PW_CARGO_PACKAGE_FILES="$_PW_OLD_CARGO_PACKAGE_FILES"
-else
- _PW_NAME="activate"
-
- if [ -z "$PW_ENVSETUP_QUIET" ]; then
- _pw_green " ACTIVATOR! This sets your shell environment variables.\n"
- fi
-fi
-
-if [ -f "$SETUP_SH" ]; then
- . "$SETUP_SH"
-
- if [ "$?" -eq 0 ]; then
- if [ "$_PW_IS_BOOTSTRAP" -eq 0 ] && [ -z "$PW_ENVSETUP_QUIET" ]; then
- echo "To activate this environment in the future, run this in your "
- echo "terminal:"
- echo
- _pw_green " source ./activate.sh\n"
- fi
- else
- _pw_red "Error during $_PW_NAME--see messages above."
- fi
-else
- _pw_red "Error during $_PW_NAME--see messages above."
-fi
-
-unset _PW_ENV_SETUP
-unset _PW_IS_BOOTSTRAP
-unset _PW_NAME
-unset _PW_BANNER
-unset _PW_OLD_CIPD_PACKAGE_FILES
-unset _PW_OLD_VIRTUALENV_REQUIREMENTS
-unset _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS
-unset _PW_OLD_CARGO_PACKAGE_FILES
-unset _pw_abspath
-unset _pw_red
-unset _pw_bold_red
-unset _pw_green
-unset _pw_magenta
unset _pw_sourced
+unset _BOOTSTRAP_PATH
+unset SETUP_SH
+unset _bootstrap_abspath
+
+pw_cleanup
diff --git a/pw_cli/py/pw_cli/env.py b/pw_cli/py/pw_cli/env.py
index a6f9639..4455fc6 100644
--- a/pw_cli/py/pw_cli/env.py
+++ b/pw_cli/py/pw_cli/env.py
@@ -23,7 +23,6 @@
parser = envparse.EnvironmentParser(prefix='PW_')
parser.add_var('PW_BOOTSTRAP_PYTHON')
- parser.add_var('PW_CARGO_SETUP', type=envparse.strict_bool, default=False)
parser.add_var('PW_ENABLE_PRESUBMIT_HOOK_WARNING', default=False)
parser.add_var('PW_EMOJI', type=envparse.strict_bool, default=False)
parser.add_var('PW_ENVSETUP')
@@ -45,12 +44,18 @@
parser.add_var('PW_DOCTOR_SKIP_CIPD_CHECKS')
+ # TODO(pwbug/274) Remove after some transition time. These are no longer
+ # used but may be set by users or downstream projects, or just in currently
+ # active shells.
parser.add_var('PW_CIPD_PACKAGE_FILES')
parser.add_var('PW_VIRTUALENV_REQUIREMENTS')
parser.add_var('PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT')
parser.add_var('PW_VIRTUALENV_SETUP_PY_ROOTS')
parser.add_var('PW_CARGO_PACKAGE_FILES')
+ parser.add_var('PW_CARGO_SETUP', type=envparse.strict_bool, default=False)
+ parser.add_var('PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT')
+ parser.add_var('PW_BANNER_FUNC')
parser.add_var('PW_BRANDING_BANNER')
parser.add_var('PW_BRANDING_BANNER_COLOR', default='magenta')
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index 9859144..da1ec9c 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -58,29 +58,13 @@
Using pw_env_setup in your project
==================================
-Projects using Pigweed can leverage ``pw_env_setup`` to install their own
-dependencies. The following environment variables are now used to pass options
-into pw_env_setup.
+Downstream Projects Using Pigweed's Packages
+********************************************
- * ``PW_CIPD_PACKAGE_FILES``
- * ``PW_VIRTUALENV_REQUIREMENTS``
- * ``PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT``
- * ``PW_VIRTUALENV_SETUP_PY_ROOTS``
- * ``PW_CARGO_PACKAGE_FILES``
-
-Each of these variables can contain multiple entries separated by ``:``
-(or ``;`` on Windows) like the ``PATH`` environment variable. However, they
-will also be interpreted as globs, so
-``PW_VIRTUALENV_REQUIREMENTS="/foo/bar/*/requirements.txt"`` is perfectly
-valid. They should be full paths.
-
-Projects depending on Pigweed should set these variables and then invoke
-Pigweed's ``bootstrap.sh`` (or ``bootstrap.bat``), which will add to each of
-these variables before invoking ``pw_env_setup``. Users wanting additional
-setup can set these variables in their shell init files. Pigweed will add to
-these variables and will not remove any existing values. At the end of
-Pigweed's bootstrap process, it will reset these variables to their initial
-values.
+Projects using Pigweed can leverage ``pw_env_setup`` to install Pigweed's
+dependencies or their own dependencies. Projects that only want to use Pigweed's
+dependencies without modifying them can just source Pigweed's ``bootstrap.sh``
+and ``activate.sh`` scripts.
An example of what your project's `bootstrap.sh` could look like is below. This
assumes `bootstrap.sh` is at the top level of your repository.
@@ -99,22 +83,6 @@
# instead of sourcing it. See below for an example of how to handle that
# situation.
- # Add the project-specific CIPD manifest.
- PW_CIPD_PACKAGE_FILES="$PROJ_ROOT/tools/cipd.json:$PW_CIPD_PACKAGE_FILES"
- export PW_CIPD_PACKAGE_FILES
-
- # Add the tools folder of this repository to the search path for setup.py
- # files.
- PW_VIRTUALENV_SETUP_PY_ROOTS="$PROJ_ROOT/tools:$PW_VIRTUALENV_SETUP_PY_ROOTS"
- export PW_VIRTUALENV_SETUP_PY_ROOTS
-
- # Process the project-specific requirements.txt file.
- PW_VIRTUALENV_REQUIREMENTS="$PROJ_ROOT/tools/requirements.txt:$PW_VIRTUALENV_REQUIREMENTS"
- export PW_VIRTUALENV_REQUIREMENTS
-
- # Add default requirements (if requirements don't conflict)
- PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT=1
-
# Source Pigweed's bootstrap script.
# Using '.' instead of 'source' for dash compatibility. Since users don't use
# dash directly, using 'source' in documentation so users don't get confused
@@ -122,7 +90,7 @@
. "$PROJ_ROOT/third_party/pigweed/$(basename "$PROJ_SETUP_SCRIPT_PATH")"
User-Friendliness
-*****************
+-----------------
You may wish to allow sourcing `bootstrap.sh` from a different directory. In
that case you'll need the following at the top of `bootstrap.sh`.
@@ -195,6 +163,64 @@
exit 1
fi
+Downstream Projects Using Different Packages
+********************************************
+
+Projects depending on Pigweed but using additional or different packages should
+copy Pigweed's ``bootstrap.sh`` and update the call to ``env_setup.py``. Search
+for "downstream" for other places that may require changes, like setting the
+``PW_ROOT`` environment variable. Relevant arguments to ``env_setup.py`` are
+listed here.
+
+``--use-pigweed-defaults``
+ Use Pigweed default values in addition to the other switches.
+
+``--cipd-package-file path/to/packages.json``
+ CIPD package file. JSON file consisting of a list of dictionaries with "path"
+ and "tags" keys, where "tags" is a list of strings.
+
+``--virtualenv-requierements path/to/requirements.txt``
+ Pip requirements file. Compiled with pip-compile.
+
+``--virtualenv-setup-py-root path/to/directory``
+ Directory in which to recursively search for ``setup.py`` files.
+
+``--cargo-package-file path/to/packages.txt``
+ Rust cargo packages to install. Lines with package name and version separated
+ by a space. Has no effect without ``--enable-cargo``.
+
+``--enable-cargo``
+ Enable cargo package installation.
+
+An example of the changed env_setup.py line is below.
+
+.. code-block:: bash
+
+ "$ROOT/third_party/pigweed/pw_env_setup/py/pw_env_setup/env_setup.py" \
+ --shell-file "$SETUP_SH" \
+ --install-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT" \
+ --use-pigweed-defaults \
+ --cipd-package-file "$ROOT/path/to/cipd.json" \
+ --virtualenv-setup-py-root "$ROOT"
+
+Projects wanting some of the Pigweed environment packages but not all of them
+should not use ``--use-pigweed-defaults`` and must manually add the references
+to Pigweed default packages through the other arguments. The arguments below
+are identical to using ``--use-pigweed-defaults``.
+
+.. code-block:: bash
+
+ --cipd-package-file
+ "$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json"
+ --cipd-package-file
+ "$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/luci.json"
+ --virtualenv-requirements
+ "$PW_ROOT/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt"
+ --virtualenv-setup-py-root
+ "$PW_ROOT"
+ --cargo-package-file
+ "$PW_ROOT/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt"
+
Implementation
**************
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index e6c70b6..a642f29 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -136,8 +136,7 @@
return self._messages
-def _get_env(varname):
- globs = os.environ.get(varname, '').split(os.pathsep)
+def _process_globs(globs):
unique_globs = []
for pat in globs:
if pat and pat not in unique_globs:
@@ -150,12 +149,11 @@
matches = glob.glob(pat)
if not matches:
warnings.append(
- 'warning: pattern "{}" in {} matched 0 files'.format(
- pat, varname))
+ 'warning: pattern "{}" matched 0 files'.format(pat))
files.extend(matches)
if not files:
- warnings.append('warning: variable {} matched 0 files'.format(varname))
+ warnings.append('warning: matched 0 total files')
return files, warnings
@@ -169,10 +167,14 @@
# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
# pylint: disable=useless-object-inheritance
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=too-many-arguments
class EnvSetup(object):
"""Run environment setup for Pigweed."""
def __init__(self, pw_root, cipd_cache_dir, shell_file, quiet, install_dir,
- *args, **kwargs):
+ use_pigweed_defaults, cipd_package_file,
+ virtualenv_requirements, virtualenv_setup_py_root,
+ cargo_package_file, enable_cargo, *args, **kwargs):
super(EnvSetup, self).__init__(*args, **kwargs)
self._env = environment.Environment()
self._pw_root = pw_root
@@ -190,6 +192,38 @@
if isinstance(self._pw_root, bytes) and bytes != str:
self._pw_root = self._pw_root.decode()
+ self._cipd_package_file = []
+ self._virtualenv_requirements = []
+ self._virtualenv_setup_py_root = []
+ self._cargo_package_file = []
+ self._enable_cargo = enable_cargo
+
+ setup_root = os.path.join(pw_root, 'pw_env_setup', 'py',
+ 'pw_env_setup')
+
+ # TODO(pwbug/67, pwbug/68) Investigate pulling these files into an
+ # oxidized env setup executable instead of referring to them in the
+ # source tree. Note that this could be error-prone because users expect
+ # changes to the files in the source tree to affect bootstrap.
+ if use_pigweed_defaults:
+ # If updating this section make sure to update
+ # $PW_ROOT/pw_env_setup/docs.rst as well.
+ self._cipd_package_file.append(
+ os.path.join(setup_root, 'cipd_setup', 'pigweed.json'))
+ self._cipd_package_file.append(
+ os.path.join(setup_root, 'cipd_setup', 'luci.json'))
+ self._virtualenv_requirements.append(
+ os.path.join(setup_root, 'virtualenv_setup',
+ 'requirements.txt'))
+ self._virtualenv_setup_py_root.append(pw_root)
+ self._cargo_package_file.append(
+ os.path.join(setup_root, 'cargo_setup', 'packages.txt'))
+
+ self._cipd_package_file.extend(cipd_package_file)
+ self._virtualenv_requirements.extend(virtualenv_requirements)
+ self._virtualenv_setup_py_root.extend(virtualenv_setup_py_root)
+ self._cargo_package_file.extend(cargo_package_file)
+
# No need to set PW_ROOT or _PW_ACTUAL_ENVIRONMENT_ROOT, that will be
# done by bootstrap.sh and bootstrap.bat for both bootstrap and
# activate.
@@ -217,7 +251,7 @@
]
# TODO(pwbug/63): Add a Windows version of cargo to CIPD.
- if not self._is_windows and os.environ.get('PW_CARGO_SETUP', ''):
+ if not self._is_windows and self._enable_cargo:
steps.append(("Rust cargo", self.cargo))
self._log(
@@ -310,7 +344,7 @@
cipd_client = cipd_wrapper.init(install_dir, silent=True)
- package_files, glob_warnings = _get_env('PW_CIPD_PACKAGE_FILES')
+ package_files, glob_warnings = _process_globs(self._cipd_package_file)
result = result_func(glob_warnings)
if not package_files:
@@ -330,10 +364,10 @@
venv_path = os.path.join(self._install_dir, 'python3-env')
- requirements, req_glob_warnings = _get_env(
- 'PW_VIRTUALENV_REQUIREMENTS')
- setup_py_roots, setup_glob_warnings = _get_env(
- 'PW_VIRTUALENV_SETUP_PY_ROOTS')
+ requirements, req_glob_warnings = _process_globs(
+ self._virtualenv_requirements)
+ setup_py_roots, setup_glob_warnings = _process_globs(
+ self._virtualenv_setup_py_root)
result = result_func(req_glob_warnings + setup_glob_warnings)
orig_python3 = _which('python3')
@@ -373,17 +407,9 @@
return _Result(_Result.Status.DONE)
def cargo(self):
- if not os.environ.get('PW_CARGO_SETUP', ''):
- return _Result(
- _Result.Status.SKIPPED,
- ' Note: Re-run bootstrap with PW_CARGO_SETUP=1 set '
- 'in your environment',
- ' to enable Rust. (Rust is usually not needed.)',
- )
-
install_dir = os.path.join(self._install_dir, 'cargo')
- package_files, glob_warnings = _get_env('PW_CARGO_PACKAGE_FILES')
+ package_files, glob_warnings = _process_globs(self._cargo_package_file)
result = result_func(glob_warnings)
if not package_files:
@@ -441,7 +467,64 @@
required=True,
)
- return parser.parse_args(argv)
+ parser.add_argument(
+ '--use-pigweed-defaults',
+ help='Use Pigweed default values in addition to the given environment '
+ 'variables.',
+ action='store_true',
+ )
+
+ parser.add_argument(
+ '--cipd-package-file',
+ help='CIPD package file. JSON file consisting of a list of dicts with '
+ '"path" and "tags" keys, where "tags" a list of str.',
+ default=[],
+ action='append',
+ )
+
+ parser.add_argument(
+ '--virtualenv-requirements',
+ help='Pip requirements file. Compiled with pip-compile.',
+ default=[],
+ action='append',
+ )
+
+ parser.add_argument(
+ '--virtualenv-setup-py-root',
+ help='Directory in which to recursively search for setup.py files.',
+ default=[],
+ action='append',
+ )
+
+ parser.add_argument(
+ '--cargo-package-file',
+ help='Rust cargo packages to install. Lines with package name and '
+ 'version separated by a space.',
+ default=[],
+ action='append',
+ )
+
+ parser.add_argument(
+ '--enable-cargo',
+ help='Enable cargo installation.',
+ action='store_true',
+ )
+
+ args = parser.parse_args(argv)
+
+ one_required = (
+ 'use_pigweed_defaults',
+ 'cipd_package_file',
+ 'virtualenv_requirements',
+ 'virtualenv_setup_py_root',
+ 'cargo_package_file',
+ )
+
+ if not any(getattr(args, x) for x in one_required):
+ parser.error('At least one of ({}) is required'.format(', '.join(
+ '"--{}"'.format(x.replace('_', '-')) for x in one_required)))
+
+ return args
def main():
diff --git a/pw_env_setup/util.sh b/pw_env_setup/util.sh
new file mode 100644
index 0000000..88cdec3
--- /dev/null
+++ b/pw_env_setup/util.sh
@@ -0,0 +1,254 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+_pw_abspath () {
+ python -c "import os.path; print(os.path.abspath('$@'))"
+}
+
+
+# Note: Colors are unfortunately duplicated in several places; and removing the
+# duplication is not easy. Their locations are:
+#
+# - bootstrap.sh
+# - pw_cli/color.py
+# - pw_env_setup/py/pw_env_setup/colors.py
+#
+# So please keep them matching then modifying them.
+pw_none() {
+ echo -e "$*"
+}
+
+pw_red() {
+ echo -e "\033[0;31m$*\033[0m"
+}
+
+pw_bold_red() {
+ echo -e "\033[1;31m$*\033[0m"
+}
+
+pw_yellow() {
+ echo -e "\033[0;33m$*\033[0m"
+}
+
+pw_bold_yellow() {
+ echo -e "\033[1;33m$*\033[0m"
+}
+
+pw_green() {
+ echo -e "\033[0;32m$*\033[0m"
+}
+
+pw_bold_green() {
+ echo -e "\033[1;32m$*\033[0m"
+}
+
+pw_blue() {
+ echo -e "\033[1;34m$*\033[0m"
+}
+
+pw_cyan() {
+ echo -e "\033[1;36m$*\033[0m"
+}
+
+pw_magenta() {
+ echo -e "\033[0;35m$*\033[0m"
+}
+
+pw_bold_white() {
+ echo -e "\033[1;37m$*\033[0m"
+}
+
+pw_eval_sourced() {
+ if [ "$1" -eq 0 ]; then
+ _PW_NAME=$(basename "$PW_SETUP_SCRIPT_PATH" .sh)
+ pw_bold_red "Error: Attempting to $_PW_NAME in a subshell"
+ pw_red " Since $_PW_NAME.sh modifies your shell's environment variables,"
+ pw_red " it must be sourced rather than executed. In particular, "
+ pw_red " 'bash $_PW_NAME.sh' will not work since the modified "
+ pw_red " environment will get destroyed at the end of the script. "
+ pw_red " Instead, source the script's contents in your shell:"
+ pw_red ""
+ pw_red " \$ source $_PW_NAME.sh"
+ exit 1
+ fi
+}
+
+pw_check_root() {
+ _PW_ROOT="$1"
+ if [[ "$_PW_ROOT" = *" "* ]]; then
+ pw_bold_red "Error: The Pigweed path contains spaces\n"
+ pw_red " The path '$_PW_ROOT' contains spaces. "
+ pw_red " Pigweed's Python environment currently requires Pigweed to be "
+ pw_red " at a path without spaces. Please checkout Pigweed in a "
+ pw_red " directory without spaces and retry running bootstrap."
+ return
+ fi
+}
+
+pw_get_env_root() {
+ # PW_ENVIRONMENT_ROOT allows developers to specify where the environment
+ # should be installed. bootstrap.sh scripts should not use that variable to
+ # store the result of this function. This separation allows scripts to assume
+ # PW_ENVIRONMENT_ROOT came from the developer and not from a previous
+ # bootstrap possibly from another workspace.
+ if [ -z "$PW_ENVIRONMENT_ROOT" ]; then
+ echo "$PW_ROOT/.environment"
+ else
+ echo "$PW_ENVIRONMENT_ROOT"
+ fi
+}
+
+# Note: This banner is duplicated in three places; which is a lesser evil than
+# the contortions that would be needed to share this snippet across shell,
+# batch, and Python. Locations:
+#
+# - pw_env_setup/util.sh
+# - pw_cli/branding.py
+# - pw_env_setup/py/pw_env_setup/windows_env_start.py
+#
+_PW_BANNER=$(cat <<EOF
+ ▒█████▄ █▓ ▄███▒ ▒█ ▒█ ░▓████▒ ░▓████▒ ▒▓████▄
+ ▒█░ █░ ░█▒ ██▒ ▀█▒ ▒█░ █ ▒█ ▒█ ▀ ▒█ ▀ ▒█ ▀█▌
+ ▒█▄▄▄█░ ░█▒ █▓░ ▄▄░ ▒█░ █ ▒█ ▒███ ▒███ ░█ █▌
+ ▒█▀ ░█░ ▓█ █▓ ░█░ █ ▒█ ▒█ ▄ ▒█ ▄ ░█ ▄█▌
+ ▒█ ░█░ ░▓███▀ ▒█▓▀▓█░ ░▓████▒ ░▓████▒ ▒▓████▀
+EOF
+)
+
+_pw_banner() {
+ if [ -z "$PW_ENVSETUP_QUIET" ] && [ -z "$PW_ENVSETUP_NO_BANNER" ]; then
+ pw_magenta "$_PW_BANNER\n"
+ fi
+}
+
+_PW_BANNER_FUNC="_pw_banner"
+
+_pw_hello() {
+ _PW_TEXT="$1"
+ if [ -n "$PW_BANNER_FUNC" ]; then
+ _PW_BANNER_FUNC="$PW_BANNER_FUNC"
+ fi
+ if [ -z "$PW_ENVSETUP_QUIET" ]; then
+ pw_green "\n WELCOME TO...\n"
+ "$_PW_BANNER_FUNC"
+ pw_green "$_PW_TEXT"
+ fi
+}
+
+# The next three functions use the following variables.
+# * PW_BANNER_FUNC: function to print banner
+# * PW_BOOTSTRAP_PYTHON: specific Python interpreter to use for bootstrap
+# * PW_USE_GCS_ENVSETUP: attempt to grab env setup executable from GCS if "true"
+# * PW_ROOT: path to Pigweed root
+# * PW_ENVSETUP_QUIET: limit output if "true"
+#
+# All arguments passed in are passed on to env_setup.py in pw_bootstrap,
+# pw_activate takes no arguments, and pw_finalize takes the name of the script
+# "bootstrap" or "activate" and the path to the setup script written by
+# bootstrap.sh.
+pw_bootstrap() {
+ _pw_hello " BOOTSTRAP! Bootstrap may take a few minutes; please be patient.\n"
+
+ # Allow forcing a specific version of Python for testing pursposes.
+ if [ -n "$PW_BOOTSTRAP_PYTHON" ]; then
+ _PW_PYTHON="$PW_BOOTSTRAP_PYTHON"
+ elif which python &> /dev/null; then
+ _PW_PYTHON=python
+ else
+ pw_bold_red "Error: No system Python present\n"
+ pw_red " Pigweed's bootstrap process requires a local system Python."
+ pw_red " Please install Python on your system, add it to your PATH"
+ pw_red " and re-try running bootstrap."
+ return
+ fi
+
+ if [ -n "$PW_USE_GCS_ENVSETUP" ]; then
+ _PW_ENV_SETUP="$("$PW_ROOT/pw_env_setup/get_pw_env_setup.sh")"
+ fi
+
+ if [ -n "$_PW_ENV_SETUP" ]; then
+ "$_PW_ENV_SETUP" "$@"
+ else
+ "$_PW_PYTHON" "$PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py" "$@"
+ fi
+}
+
+pw_activate() {
+ _pw_hello " ACTIVATOR! This sets your shell environment variables.\n"
+}
+
+pw_finalize() {
+ _PW_NAME="$1"
+ _PW_SETUP_SH="$2"
+ if [ -f "$_PW_SETUP_SH" ]; then
+ . "$_PW_SETUP_SH"
+
+ if [ "$?" -eq 0 ]; then
+ if [ "$_PW_NAME" = "bootstrap" ] && [ -z "$PW_ENVSETUP_QUIET" ]; then
+ echo "To activate this environment in the future, run this in your "
+ echo "terminal:"
+ echo
+ pw_green " source ./activate.sh\n"
+ fi
+ else
+ pw_red "Error during $_PW_NAME--see messages above."
+ fi
+ else
+ pw_red "Error during $_PW_NAME--see messages above."
+ fi
+}
+
+pw_cleanup() {
+ unset _PW_BANNER
+ unset _PW_BANNER_FUNC
+ unset _PW_ENV_SETUP
+ unset _PW_NAME
+ unset _PW_PYTHON
+ unset _PW_SETUP_SH
+
+ unset _pw_abspath
+ unset pw_none
+ unset pw_red
+ unset pw_bold_red
+ unset pw_yellow
+ unset pw_bold_yellow
+ unset pw_green
+ unset pw_bold_green
+ unset pw_blue
+ unset pw_cyan
+ unset pw_magenta
+ unset pw_bold_white
+ unset pw_eval_sourced
+ unset pw_check_root
+ unset pw_get_env_root
+ unset _pw_banner
+ unset pw_bootstrap
+ unset pw_activate
+ unset pw_finalize
+ unset _pw_cleanup
+
+ # TODO(pwbug/274) Remove after some transition time. These are no longer
+ # used but may be set by users or downstream projects, or just in currently
+ # active shells.
+ unset _PW_OLD_CIPD_PACKAGE_FILES
+ unset _PW_OLD_VIRTUALENV_REQUIREMENTS
+ unset _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS
+ unset _PW_OLD_CARGO_PACKAGE_FILES
+ unset PW_CIPD_PACKAGE_FILES
+ unset PW_VIRTUALENV_REQUIREMENTS
+ unset PW_VIRTUALENV_SETUP_PY_ROOTS
+ unset PW_CARGO_PACKAGE_FILES
+ unset PW_CARGO_SETUP
+ unset PW_VIRTUALENV_REQUIREMENTS_APPEND_DEFAULT
+}