pw_env_setup: use env vars for bootstrap opts
The following environment variables are now used to pass options into
pw_env_setup:
* PW_CIPD_PACKAGEFILES
* PW_VIRTUALENV_REQUIREMENTS
* PW_VIRTUALENV_SETUPPYROOTS
* PW_CARGO_PACKAGEFILES
Each of these variables can contain multiple entries separated by ':'
(or ';' on Windows) like PATH. However, they will also be interpreted
as globs, so PW_VIRTUALENV_REQUIREMENTS="/foo/bar/*/requirements.txt"
is perfectly valid. They should be full paths.
Projects depending on Pigweed should prepend to these variables and
then invoke Pigweed's bootstrap.sh (or bootstrap.bat). Users wanting
additional setup should set these variables in their shell init files.
Change-Id: Ibf0f1d5279028856a590ffc50850df2174c38e28
Bug: 138
diff --git a/bootstrap.bat b/bootstrap.bat
index 2cf0cdf..8713718 100644
--- a/bootstrap.bat
+++ b/bootstrap.bat
@@ -48,6 +48,16 @@
)
)
+set _PW_OLD_CIPD_PACKAGE_FILES=%PW_CIPD_PACKAGE_FILES%
+set _PW_OLD_VIRTUALENV_REQUIREMENTS=%PW_VIRTUALENV_REQUIREMENTS%
+set _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS=%PW_VIRTUALENV_SETUP_PY_ROOTS%
+set _PW_OLD_CARGO_PACKAGE_FILES=%PW_CARGO_PACKAGE_FILES%
+
+set PW_CIPD_PACKAGE_FILES=%PW_ROOT%\pw_env_setup\py\pw_env_setup\cipd_setup\*.json;%PW_CIPD_PACKAGE_FILES%
+set PW_VIRTUALENV_REQUIREMENTS=%PW_ROOT%\pw_env_setup\py\pw_env_setup\virtualenv_setup\requirements.txt;%PW_VIRTUALENV_REQUIREMENTS%
+set PW_VIRTUALENV_SETUP_PY_ROOTS=%PW_ROOT%;%PW_VIRTUALENV_SETUP_PY_ROOTS%
+set PW_CARGO_PACKAGE_FILES=%PW_ROOT%\pw_env_setup\py\pw_env_setup\cargo_setup\packages.txt;%PW_CARGO_PACKAGE_FILES%
+
set "_pw_start_script=%PW_ROOT%\pw_env_setup\py\pw_env_setup\windows_env_start.py"
set "shell_file=%PW_ROOT%\pw_env_setup\.env_setup.bat"
@@ -68,6 +78,11 @@
)
)
+set PW_CIPD_PACKAGE_FILES=%_PW_OLD_CIPD_PACKAGE_FILES%
+set PW_VIRTUALENV_REQUIREMENTS=%_PW_OLD_VIRTUALENV_REQUIREMENTS%
+set PW_VIRTUALENV_SETUP_PY_ROOTS=%_PW_OLD_VIRTUALENV_SETUP_PY_ROOTS%
+set PW_CARGO_PACKAGE_FILES=%_PW_OLD_CARGO_PACKAGE_FILES%
+
call "%shell_file%"
:finish
diff --git a/bootstrap.sh b/bootstrap.sh
index 3b5e417..dbb8059 100644
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -145,7 +145,29 @@
return
fi
+ _PW_OLD_CIPD_PACKAGE_FILES="$PW_CIPD_PACKAGE_FILES"
+ PW_CIPD_PACKAGE_FILES="$PW_ROOT/pw_env_setup/py/pw_env_setup/cipd_setup/*.json:$PW_CIPD_PACKAGE_FILES"
+ export PW_CIPD_PACKAGE_FILES
+
+ _PW_OLD_VIRTUALENV_REQUIREMENTS="$PW_VIRTUALENV_REQUIREMENTS"
+ PW_VIRTUALENV_REQUIREMENTS="$PW_ROOT/pw_env_setup/py/pw_env_setup/virtualenv_setup/requirements.txt:$PW_VIRTUALENV_REQUIREMENTS"
+ export PW_VIRTUALENV_REQUIREMENTS
+
+ _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS="$PW_VIRTUALENV_SETUP_PY_ROOTS"
+ PW_VIRTUALENV_SETUP_PY_ROOTS="$PW_ROOT/*:$PW_VIRTUALENV_SETUP_PY_ROOTS"
+ export PW_VIRTUALENV_SETUP_PY_ROOTS
+
+ _PW_OLD_CARGO_PACKAGE_FILES="$PW_CARGO_PACKAGE_FILES"
+ PW_CARGO_PACKAGE_FILES="$PW_ROOT/pw_env_setup/py/pw_env_setup/cargo_setup/packages.txt:$PW_CARGO_PACKAGE_FILES"
+ export PW_CARGO_PACKAGE_FILES
+
"$PYTHON" "$PW_ROOT/pw_env_setup/py/pw_env_setup/env_setup.py" --shell-file "$SETUP_SH"
+
+
+ PW_CIPD_PACKAGE_FILES="$_PW_OLD_CIPD_PACKAGE_FILES"
+ PW_VIRTUALENV_REQUIREMENTS="$_PW_OLD_VIRTUALENV_REQUIREMENTS"
+ PW_VIRTUALENV_SETUP_PY_ROOTS="$_PW_OLD_VIRTUALENV_SETUP_PY_ROOTS"
+ PW_CARGO_PACKAGE_FILES="$_PW_OLD_CARGO_PACKAGE_FILES"
else
_PW_NAME="activate"
@@ -174,6 +196,10 @@
unset _PW_IS_BOOTSTRAP
unset _PW_NAME
unset _PIGWEED_BANNER
+unset _PW_OLD_CIPD_PACKAGE_FILES
+unset _PW_OLD_VIRTUALENV_REQUIREMENTS
+unset _PW_OLD_VIRTUALENV_SETUP_PY_ROOTS
+unset _PW_OLD_CARGO_PACKAGE_FILES
unset _pw_abspath
unset _pw_red
unset _pw_bold_red
diff --git a/pw_cli/py/pw_cli/env.py b/pw_cli/py/pw_cli/env.py
index 1784c60..613fc51 100644
--- a/pw_cli/py/pw_cli/env.py
+++ b/pw_cli/py/pw_cli/env.py
@@ -40,6 +40,11 @@
parser.add_var('PW_LUCI_CIPD_INSTALL_DIR')
parser.add_var('PW_CIPD_INSTALL_DIR')
+ parser.add_var('PW_CIPD_PACKAGE_FILES')
+ parser.add_var('PW_VIRTUALENV_REQUIREMENTS')
+ parser.add_var('PW_VIRTUALENV_SETUP_PY_ROOTS')
+ parser.add_var('PW_CARGO_PACKAGE_FILES')
+
return parser
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index f67edf3..bd0b02f 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -37,3 +37,26 @@
experience.
.. _send us a note: pigweed@googlegroups.com
+
+Projects using Pigweed can leverage ``pw_env_setup`` to install their own
+dependencies. The following environment variables are now used to pass options
+into pw_env_setup.
+
+ * ``PW_CIPD_PACKAGE_FILES``
+ * ``PW_VIRTUALENV_REQUIREMENTS``
+ * ``PW_VIRTUALENV_SETUP_PY_ROOTS``
+ * ``PW_CARGO_PACKAGE_FILES``
+
+Each of these variables can contain multiple entries separated by ``:``
+(or ``;`` on Windows) like the ``PATH`` environment variable. However, they
+will also be interpreted as globs, so
+``PW_VIRTUALENV_REQUIREMENTS="/foo/bar/*/requirements.txt"`` is perfectly
+valid. They should be full paths.
+
+Projects depending on Pigweed should set these variables and then invoke
+Pigweed's ``bootstrap.sh`` (or ``bootstrap.bat``), which will add to each of
+these variables before invoking ``pw_env_setup``. Users wanting additional
+setup can set these variables in their shell init files. Pigweed will add to
+these variables and will not remove any existing values. At the end of
+Pigweed's bootstrap process, it will reset these variables to their initial
+values.
diff --git a/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
index ac50b3e..0238788 100644
--- a/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
+++ b/pw_env_setup/py/pw_env_setup/cargo_setup/__init__.py
@@ -19,7 +19,7 @@
import tempfile
-def install(pw_root, env):
+def install(pw_root, package_files, env):
"""Installs rust tools using cargo."""
prefix = os.path.join(pw_root, '.cargo')
@@ -30,37 +30,35 @@
if 'CARGO_TARGET_DIR' not in os.environ:
env.set('CARGO_TARGET_DIR', os.path.expanduser('~/.cargo-cache'))
- # packages.txt contains packages one per line with two fields: package
- # name and version.
- package_path = os.path.join(pw_root, 'pw_env_setup', 'py', 'pw_env_setup',
- 'cargo_setup', 'packages.txt')
- with env(), open(package_path, 'r') as ins:
- for line in ins:
- line = line.strip()
- if not line or line.startswith('#'):
- continue
+ with env():
+ for package_file in package_files:
+ with open(package_file, 'r') as ins:
+ for line in ins:
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
- package, version = line.split()
- cmd = [
- 'cargo',
- 'install',
- # If downgrading (which could happen when switching branches)
- # '--force' is required.
- '--force',
- '--root', prefix,
- '--version', version,
- package,
- ] # yapf: disable
+ package, version = line.split()
+ cmd = [
+ 'cargo',
+ 'install',
+ # If downgrading (which could happen when switching
+ # branches) '--force' is required.
+ '--force',
+ '--root', prefix,
+ '--version', version,
+ package,
+ ] # yapf: disable
- # TODO(pwbug/135) Use function from common utility module.
- with tempfile.TemporaryFile(mode='w+') as temp:
- try:
- subprocess.check_call(cmd,
- stdout=temp,
- stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError:
- temp.seek(0)
- sys.stderr.write(temp.read())
- raise
+ # TODO(pwbug/135) Use function from common utility module.
+ with tempfile.TemporaryFile(mode='w+') as temp:
+ try:
+ subprocess.check_call(cmd,
+ stdout=temp,
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError:
+ temp.seek(0)
+ sys.stderr.write(temp.read())
+ raise
return True
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
index fb19c3c..73ec2b7 100755
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py
@@ -22,7 +22,6 @@
from __future__ import print_function
import argparse
-import glob
import json
import os
import shutil
@@ -137,13 +136,16 @@
pw_root = os.environ['PW_ROOT']
# Run cipd for each json file.
- default_packages = os.path.join(pw_root, 'pw_env_setup', 'py',
- 'pw_env_setup', 'cipd_setup', '*.json')
- for package_file in package_files or glob.glob(default_packages):
- ensure_file = os.path.join(
- root_install_dir,
- os.path.basename(os.path.splitext(package_file)[0] + '.ensure'))
- write_ensure_file(package_file, ensure_file)
+ for package_file in package_files:
+ if os.path.splitext(package_file)[1] == '.ensure':
+ ensure_file = package_file
+ else:
+ ensure_file = os.path.join(
+ root_install_dir,
+ os.path.basename(
+ os.path.splitext(package_file)[0] + '.ensure'))
+ write_ensure_file(package_file, ensure_file)
+
install_dir = os.path.join(
root_install_dir,
os.path.basename(os.path.splitext(package_file)[0]))
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index 3eaabb2..c7db5c5 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -96,6 +96,37 @@
return self._messages
+def _get_env(varname):
+ globs = os.environ.get(varname, '').split(os.pathsep)
+ unique_globs = []
+ for pat in globs:
+ if pat and pat not in unique_globs:
+ unique_globs.append(pat)
+
+ files = []
+ warnings = []
+ for pat in unique_globs:
+ if pat:
+ matches = glob.glob(pat)
+ if not matches:
+ warnings.append(
+ 'warning: pattern "{}" in {} matched 0 files'.format(
+ pat, varname))
+ files.extend(matches)
+
+ if not files:
+ warnings.append('warning: variable {} matched 0 files'.format(varname))
+
+ return files, warnings
+
+
+def result_func(glob_warnings):
+ def result(status, *args):
+ return _Result(status, *([str(x) for x in glob_warnings] + list(args)))
+
+ return result
+
+
# TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
# pylint: disable=useless-object-inheritance
class EnvSetup(object):
@@ -183,15 +214,16 @@
with spin():
result = step()
+ self._log(result.status_str())
+
self._env.echo(result.status_str())
for message in result.messages():
+ sys.stderr.write('{}\n'.format(message))
self._env.echo(message)
if not result.ok():
return -1
- self._log('done')
-
self._log('')
self._env.echo('')
@@ -220,27 +252,33 @@
cipd_client = cipd_wrapper.init(install_dir, silent=True)
- package_files = glob.glob(
- os.path.join(self._setup_root, 'cipd_setup', '*.json'))
- if not cipd_update.update(
- cipd=cipd_client,
- root_install_dir=install_dir,
- package_files=package_files,
- cache_dir=self._cipd_cache_dir,
- env_vars=self._env,
- ):
- return _Result(_Result.Status.FAILED)
+ package_files, glob_warnings = _get_env('PW_CIPD_PACKAGE_FILES')
+ result = result_func(glob_warnings)
- return _Result(_Result.Status.DONE)
+ if not package_files:
+ return result(_Result.Status.SKIPPED)
+
+ if not cipd_update.update(cipd=cipd_client,
+ root_install_dir=install_dir,
+ package_files=package_files,
+ cache_dir=self._cipd_cache_dir,
+ env_vars=self._env):
+ return result(_Result.Status.FAILED)
+
+ return result(_Result.Status.DONE)
def virtualenv(self):
"""Setup virtualenv."""
venv_path = os.path.join(self._pw_root, '.python3-env')
- requirements = os.path.join(self._setup_root, 'virtualenv_setup',
- 'requirements.txt')
+ requirements, req_glob_warnings = _get_env(
+ 'PW_VIRTUALENV_REQUIREMENTS')
+ setup_py_roots, setup_glob_warnings = _get_env(
+ 'PW_VIRTUALENV_SETUP_PY_ROOTS')
+ result = result_func(req_glob_warnings + setup_glob_warnings)
+ # TODO(pwbug/138) don't hardcode the path to Python.
cipd_bin = os.path.join(
self._pw_root,
'.cipd',
@@ -262,19 +300,22 @@
python = os.path.join(cipd_bin, py_executable)
- if not virtualenv_setup.install(
- venv_path=venv_path,
- requirements=[requirements],
- python=python,
- env=self._env,
- ):
- return _Result(_Result.Status.FAILED)
+ if not requirements and not setup_py_roots:
+ return result(_Result.Status.SKIPPED)
- return _Result(_Result.Status.DONE)
+ if not virtualenv_setup.install(venv_path=venv_path,
+ requirements=requirements,
+ setup_py_roots=setup_py_roots,
+ python=python,
+ env=self._env):
+ return result(_Result.Status.FAILED)
+
+ return result(_Result.Status.DONE)
def host_tools(self):
# The host tools are grabbed from CIPD, at least initially. If the
# user has a current host build, that build will be used instead.
+ # TODO(mohrr) find a way to do stuff like this for all projects.
host_dir = os.path.join(self._pw_root, 'out', 'host')
self._env.prepend('PATH', os.path.join(host_dir, 'host_tools'))
return _Result(_Result.Status.DONE)
@@ -288,10 +329,18 @@
' to enable Rust. (Rust is usually not needed.)',
)
- if not cargo_setup.install(pw_root=self._pw_root, env=self._env):
- return _Result(_Result.Status.FAILED)
+ package_files, glob_warnings = _get_env('PW_CARGO_PACKAGE_FILES')
+ result = result_func(glob_warnings)
- return _Result(_Result.Status.DONE)
+ if not package_files:
+ return result(_Result.Status.SKIPPED)
+
+ if not cargo_setup.install(pw_root=self._pw_root,
+ package_files=package_files,
+ env=self._env):
+ return result(_Result.Status.FAILED)
+
+ return result(_Result.Status.DONE)
def parse(argv=None):
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
index 385252d..0bb2d32 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
@@ -32,6 +32,11 @@
default=[],
action='append',
help='requirements.txt files to install')
+ parser.add_argument('-s',
+ '--setup-py-roots',
+ default=[],
+ action='append',
+ help='places to search for setup.py files')
parser.add_argument('--quick-setup',
dest='full_envsetup',
action='store_false',
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index 768f241..88e2143 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
@@ -27,11 +27,6 @@
return subprocess.check_output(['git'] + list(args), **kwargs).strip()
-def git_list_files(*args, **kwargs):
- """Run git ls-files, passing args as git params and kwargs to subprocess."""
- return git_stdout('ls-files', *args, **kwargs).split()
-
-
def git_repo_root(path='./'):
"""Find git repository root."""
try:
@@ -80,11 +75,21 @@
raise
+def _find_files_by_name(roots, name):
+ matches = []
+ for root in roots:
+ for dirpart, _, files in os.walk(root):
+ if name in files:
+ matches.append(os.path.join(dirpart, name))
+ return matches
+
+
def install(
venv_path,
full_envsetup=True,
requirements=(),
python=sys.executable,
+ setup_py_roots=(),
env=None,
):
"""Creates a venv and installs all packages in this Git repo."""
@@ -120,7 +125,7 @@
if not pw_root:
raise GitRepoNotFound()
- setup_py_files = git_list_files('setup.py', '*/setup.py', cwd=pw_root)
+ setup_py_files = _find_files_by_name(setup_py_roots, 'setup.py')
# Sometimes we get an error saying "Egg-link ... does not match
# installed location". This gets around that. The egg-link files
@@ -137,18 +142,24 @@
pip_install('--upgrade', 'pip')
def package(pkg_path):
- return os.path.join(pw_root, os.path.dirname(pkg_path.decode()))
+ if isinstance(pkg_path, bytes) and bytes != str:
+ pkg_path = pkg_path.decode()
+ return os.path.join(pw_root, os.path.dirname(pkg_path))
- package_args = tuple('--editable={}'.format(package(path))
- for path in setup_py_files)
+ if requirements:
+ requirement_args = tuple('--requirement={}'.format(req)
+ for req in requirements)
+ pip_install('--log', os.path.join(venv_path, 'pip-requirements.log'),
+ *requirement_args)
- requirement_args = tuple('--requirement={}'.format(req)
- for req in requirements)
-
- pip_install('--log', os.path.join(venv_path, 'pip-requirements.log'),
- *requirement_args)
- pip_install('--log', os.path.join(venv_path, 'pip-packages.log'),
- *package_args)
+ if setup_py_files:
+ # Run through sorted so pw_cli (on which other packages depend) comes
+ # early in the list.
+ # TODO(mohrr) come up with a way better than just using sorted().
+ package_args = tuple('--editable={}'.format(package(path))
+ for path in sorted(setup_py_files))
+ pip_install('--log', os.path.join(venv_path, 'pip-packages.log'),
+ *package_args)
if env:
env.set('VIRTUAL_ENV', venv_path)
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index c2ed940..8cb5e18 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -48,11 +48,24 @@
#
def init_cipd(ctx: PresubmitContext):
# TODO(mohrr) invoke by importing rather than by subprocess.
- call(
+
+ # TODO(pwbug/138) find way to support dependent project package files.
+
+ cmd = [
sys.executable,
ctx.repository_root.joinpath('pw_env_setup', 'py', 'pw_env_setup',
'cipd_setup', 'update.py'),
- '--install-dir', ctx.output_directory)
+ '--install-dir', ctx.output_directory,
+ ] # yapf: disable
+
+ package_files = ctx.repository_root.joinpath('pw_env_setup', 'py',
+ 'pw_env_setup',
+ 'cipd_setup').glob('*.json')
+
+ for package_file in package_files:
+ cmd.extend(('--package-file', package_file))
+
+ call(*cmd)
paths = [ctx.output_directory, ctx.output_directory.joinpath('bin')]
for base in ctx.output_directory.glob('*'):
@@ -71,6 +84,8 @@
'pw_env_setup',
'virtualenv_setup')
+ # TODO(pwbug/138) find way to support dependent project requirements.
+
# For speed, don't build the venv if it exists. Use --clean to recreate it.
if not ctx.output_directory.joinpath('pyvenv.cfg').is_file():
call(
@@ -79,6 +94,7 @@
f'--venv_path={ctx.output_directory}',
'--requirements={}'.format(
virtualenv_source.joinpath('requirements.txt')),
+ '--setup-py-roots={}'.format(ctx.repository_root),
)
os.environ['PATH'] = os.pathsep.join((