Files
5ed2b7c6b2e2a5da50c3db9cda9e9b8e4ae4402f
devstack /inc /python

519 lines
17 KiB
Plaintext
Raw Normal View History

#!/bin/bash
#
# **inc/python** - Python-related functions
#
# Support for pip/setuptools interfaces and virtual environments
#
# External functions used:
# - GetOSVersion
# - is_fedora
# - safe_chown
# Save trace setting
INC_PY_TRACE=$(set +o | grep xtrace)
set +o xtrace
# Global Config Variables
# PROJECT_VENV contains the name of the virtual environment for each
# project. A null value installs to the system Python directories.
declare -A -g PROJECT_VENV
# Utility Functions
# =================
# Joins bash array of extras with commas as expected by other functions
function join_extras {
local IFS=","
echo "$*"
}
# Python Functions
# ================
Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de> Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
2018年04月04日 14:02:30 -07:00
# Setup the global devstack virtualenvs and the associated environment
# updates.
function setup_devstack_virtualenv {
# We run devstack out of a global virtualenv.
if [[ ! -d $DEVSTACK_VENV ]] ; then
# Using system site packages to enable nova to use libguestfs.
# This package is currently installed via the distro and not
# available on pypi.
python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
pip_install -U pip setuptools
#NOTE(rpittau): workaround for simplejson removal in osc
# https://review.opendev.org/c/openstack/python-openstackclient/+/920001
pip_install -U simplejson
Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de> Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
2018年04月04日 14:02:30 -07:00
fi
if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
export PATH="$DEVSTACK_VENV/bin:$PATH"
export PYTHON="$DEVSTACK_VENV/bin/python3"
fi
}
# Get the path to the pip command.
# get_pip_command
function get_pip_command {
local version="1ドル"
if [ -z "$version" ]; then
die $LINENO "pip python version is not set."
fi
# NOTE(dhellmann): I don't know if we actually get a pip3.4-python
# under any circumstances.
which pip${version} || which pip${version}-python
if [ $? -ne 0 ]; then
die $LINENO "Unable to find pip${version}; cannot continue"
fi
}
# Get the path to the directory where python executables are installed.
# get_python_exec_prefix
function get_python_exec_prefix {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
$xtrace
Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de> Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
2018年04月04日 14:02:30 -07:00
if [[ "$GLOBAL_VENV" == "True" ]] ; then
echo "$DEVSTACK_VENV/bin"
else
echo "/usr/local/bin"
fi
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr packagename
function pip_install_gr {
local name=1ドル
local clean_name
clean_name=$(get_from_global_requirements $name)
pip_install $clean_name
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification with extras.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr_extras packagename extra1,extra2,...
function pip_install_gr_extras {
local name=1ドル
local extras=2ドル
local version_constraints
version_constraints=$(get_version_constraints_from_global_requirements $name)
pip_install $name[$extras]$version_constraints
}
# enable_python3_package() -- no-op for backwards compatibility
allow config to manage python3 use explicitly Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which packages are installed using Python 3. Move the list of whitelisted packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES, except swift which is not enabled by default for now. Add enable_python3_package and disable_python3_package functions to make editing the variables from local.conf easier. Add python3_enabled_for and python3_disabled_for functions to check the settings against packages being installed by pip. Update pip_install to check if python3 is disabled for a service, then see if it is explicitly enabled, and only then fall back to looking at the classifiers in the packaging metadata. Update pip_install messages to give more detail about why the choice between python 2 and 3 is being made for a given package. Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f Signed-off-by: Doug Hellmann <doug@doughellmann.com>
2017年01月09日 21:24:24 +00:00
#
# enable_python3_package dir [dir ...]
function enable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
echo "It is no longer necessary to call enable_python3_package()."
allow config to manage python3 use explicitly Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which packages are installed using Python 3. Move the list of whitelisted packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES, except swift which is not enabled by default for now. Add enable_python3_package and disable_python3_package functions to make editing the variables from local.conf easier. Add python3_enabled_for and python3_disabled_for functions to check the settings against packages being installed by pip. Update pip_install to check if python3 is disabled for a service, then see if it is explicitly enabled, and only then fall back to looking at the classifiers in the packaging metadata. Update pip_install messages to give more detail about why the choice between python 2 and 3 is being made for a given package. Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f Signed-off-by: Doug Hellmann <doug@doughellmann.com>
2017年01月09日 21:24:24 +00:00
$xtrace
}
# disable_python3_package() -- no-op for backwards compatibility
allow config to manage python3 use explicitly Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which packages are installed using Python 3. Move the list of whitelisted packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES, except swift which is not enabled by default for now. Add enable_python3_package and disable_python3_package functions to make editing the variables from local.conf easier. Add python3_enabled_for and python3_disabled_for functions to check the settings against packages being installed by pip. Update pip_install to check if python3 is disabled for a service, then see if it is explicitly enabled, and only then fall back to looking at the classifiers in the packaging metadata. Update pip_install messages to give more detail about why the choice between python 2 and 3 is being made for a given package. Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f Signed-off-by: Doug Hellmann <doug@doughellmann.com>
2017年01月09日 21:24:24 +00:00
#
# disable_python3_package dir [dir ...]
function disable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
echo "It is no longer possible to call disable_python3_package()."
allow config to manage python3 use explicitly Add variables ENABLED_PYTHON3_PACKAGES and DISABLED_PYTHON3_PACKAGES to work like ENABLED_SERVICES and DISABLED_SERVICES and to manage which packages are installed using Python 3. Move the list of whitelisted packages in pip_install to the default for ENABLED_PYTHON3_PACKAGES, except swift which is not enabled by default for now. Add enable_python3_package and disable_python3_package functions to make editing the variables from local.conf easier. Add python3_enabled_for and python3_disabled_for functions to check the settings against packages being installed by pip. Update pip_install to check if python3 is disabled for a service, then see if it is explicitly enabled, and only then fall back to looking at the classifiers in the packaging metadata. Update pip_install messages to give more detail about why the choice between python 2 and 3 is being made for a given package. Change-Id: I69857d4e11f4767928614a3b637c894bcd03491f Signed-off-by: Doug Hellmann <doug@doughellmann.com>
2017年01月09日 21:24:24 +00:00
$xtrace
}
# Wrapper for ``pip install`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
# ``PIP_UPGRADE``, ``*_proxy``,
# Usage:
# pip_install pip_arguments
function pip_install {
local xtrace result
xtrace=$(set +o | grep xtrace)
set +o xtrace
local upgrade=""
local offline=${OFFLINE:-False}
if [[ "$offline" == "True" || -z "$@" ]]; then
$xtrace
return
fi
time_start "pip_install"
PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
if [[ "$PIP_UPGRADE" = "True" ]] ; then
upgrade="--upgrade"
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
# Try to extract the path of the package we are installing into
# package_dir. We need this to check for test-requirements.txt,
# at least.
#
# ${!#} expands to the last positional argument to this function.
# With "extras" syntax included, our arguments might be something
# like:
# -e /path/to/fooproject[extra]
# Thus this magic line grabs just the path without extras
#
# Note that this makes no sense if this is a pypi (rather than
# local path) install; ergo you must check this path exists before
# use. Also, if we had multiple or mixed installs, we would also
# likely break. But for historical reasons, it's basically only
# the other wrapper functions in here calling this to install
# local packages, and they do so with single call per install. So
# this works (for now...)
local package_dir=${!#%\[*\]}
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de> Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
2018年04月04日 14:02:30 -07:00
elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
# We have to check that the DEVSTACK_VENV exists because early
# devstack boostrapping needs to operate in a system context
# too bootstrap pip. Once pip is bootstrapped we create the
# global venv and can start to use it.
local cmd_pip=$DEVSTACK_VENV/bin/pip
local sudo_pip="env"
echo "Using python $PYTHON3_VERSION to install $package_dir"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
# See
# https://github.com/pypa/setuptools/issues/2232
# http://lists.openstack.org/pipermail/openstack-discuss/2020-August/016905.html
# this makes setuptools >=50 use the platform distutils.
# We only want to do this on global pip installs, not if
# installing in a virtualenv
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8 SETUPTOOLS_USE_DISTUTILS=stdlib "
echo "Using python $PYTHON3_VERSION to install $package_dir"
fi
cmd_pip="$cmd_pip install"
# Always apply constraints
cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
$xtrace
$sudo_pip \
http_proxy="${http_proxy:-}" \
https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
PIP_FIND_LINKS=$PIP_FIND_LINKS \
$cmd_pip $upgrade \
$@
result=$?
time_stop "pip_install"
return $result
}
function pip_uninstall {
# Skip uninstall if offline
[[ "${OFFLINE}" = "True" ]] && return
local name=1ドル
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
local sudo_pip="sudo -H LC_ALL=en_US.UTF-8"
fi
# don't error if we can't uninstall, it might not be there
$sudo_pip $cmd_pip uninstall -y $name || /bin/true
}
# get version of a package from global requirements file
# get_from_global_requirements <package>
function get_from_global_requirements {
local package=1ドル
local required_pkg
required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
if [[ $required_pkg == "" ]]; then
die $LINENO "Can't find package $package in requirements"
fi
echo $required_pkg
}
# get only version constraints of a package from global requirements file
# get_version_constraints_from_global_requirements <package>
function get_version_constraints_from_global_requirements {
local package=1ドル
local required_pkg_version_constraint
# drop the package name from output (\K)
required_pkg_version_constraint=$(grep -i -h -o -P "^${package}\K.*" $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
if [[ $required_pkg_version_constraint == "" ]]; then
die $LINENO "Can't find package $package in requirements"
fi
echo $required_pkg_version_constraint
}
# should we use this library from their git repo, or should we let it
# get pulled in via pip dependencies.
function use_library_from_git {
local name=1ドル
local enabled=1
[[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
return $enabled
}
# determine if a package was installed from git
function lib_installed_from_git {
local name=1ドル
local safe_name
safe_name=$(python -c "from pkg_resources import safe_name; \
print(safe_name('${name}'))")
# Note "pip freeze" doesn't always work here, because it tries to
# be smart about finding the remote of the git repo the package
# was installed from. This doesn't work with zuul which clones
# repos with no remote.
#
# The best option seems to be to use "pip list" which will tell
# you the path an editable install was installed from; for example
# in response to something like
# pip install -e 'git+https://opendev.org/openstack/bashate#egg=bashate'
# pip list --format columns shows
# bashate 0.5.2.dev19 /tmp/env/src/bashate
# Thus we check the third column to see if we're installed from
# some local place.
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \3ドル}") ]]
}
# setup a library by name. If we are trying to use the library from
# git, we'll do a git based install, otherwise we'll punt and the
# library should be installed by a requirements pull from another
# project.
function setup_lib {
local name=1ドル
local dir=${GITDIR[$name]}
setup_install $dir
}
# setup a library by name in editable mode. If we are trying to use
# the library from git, we'll do a git based install, otherwise we'll
# punt and the library should be installed by a requirements pull from
# another project.
#
# use this for non namespaced libraries
#
# setup_dev_lib [-bindep] <name> [<extras>]
function setup_dev_lib {
local bindep
if [[ 1ドル == -bindep* ]]; then
bindep="${1}"
shift
fi
local name=1ドル
local dir=${GITDIR[$name]}
local extras=2ドル
setup_develop $bindep $dir $extras
}
# this should be used if you want to install globally, all libraries should
# use this, especially *oslo* ones
#
# setup_install project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# bindep: Set "-bindep" as first argument to install bindep.txt packages
# The command is like "pip install <project_dir>[<extras>]"
function setup_install {
local bindep
if [[ 1ドル == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=1ドル
local extras=2ドル
_setup_package_with_constraints_edit $bindep $project_dir "" $extras
}
# this should be used for projects which run services, like all services
#
# setup_develop project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install -e <project_dir>[<extras>]"
function setup_develop {
local bindep
if [[ 1ドル == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=1ドル
local extras=2ドル
_setup_package_with_constraints_edit $bindep $project_dir -e $extras
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`
#
# Updates the constraints from REQUIREMENTS_DIR to reflect the
# future installed state of this package. This ensures when we
# install this package we get the from source version.
#
# Uses globals ``REQUIREMENTS_DIR``
# _setup_package_with_constraints_edit project_dir flags [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# flags: pip CLI options/flags
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install <flags> <project_dir>[<extras>]"
function _setup_package_with_constraints_edit {
local bindep
if [[ 1ドル == -bindep* ]]; then
bindep="${1}"
shift
fi
local project_dir=1ドル
local flags=2ドル
local extras=3ドル
# Normalize the directory name to avoid
# "installation from path or url cannot be constrained to a version"
# error.
# REVISIT(yamamoto): Remove this when fixed in pip.
# https://github.com/pypa/pip/pull/3582
project_dir=$(cd $project_dir && pwd)
if [ -n "$REQUIREMENTS_DIR" ]; then
Revert "Workaround for new pip 20.3 behavior" This reverts commit 7a3a7ce876a37376fe0dca7278e41a4f46867daa and bcd0acf6c0b5d6501e91133c3a937b3fc40f7122 and part of f1ed7c77c50ac28cb58c9f7ed885c6a3e0a75403 which all cap our pip installs. Given the pip ecosystem can often incorporate major changes, tracking upstream at least generally gives us one problem at a time to solve rather than trying to handle version jumps when LTS distros update. The new dependency resolver included some changes that disallow setting URL's like "file:///path/to/project#egg=project" in constraints. Apparently the fact it used to work was an accident of the requires/constraints mechanism; it does make some sense as the URL doesn't really have a version-number that the resolver can put in an ordering graph. The _setup_package_with_constraints_edit function comment highlights what this is trying to do # Updates the constraints from REQUIREMENTS_DIR to reflect the # future installed state of this package. This ensures when we # install this package we get the from source version. In other words; if constraints has "foo==1.2.3" and Zuul has checked out "foo" for testing, we have to make sure pip doesn't choose version 1.2.3 from pypi. It seems like removing the entry from upper-requirements.txt is the important part; adding the URL path to the on-disk version was just something that seemed to work at the time, but isn't really necessary. We will install the package in question which will be the latest version (from Zuul checkout) and without the package in upper-requirements.txt nothing will try and downgrade it. Therefore the solution proposed here is to remove the adding of the URL parts. This allows us to uncap pip and restore testing with the new dependency resolver. Closes-Bug: #1906322 Change-Id: Ib9ba52147199a9d6d0293182d5db50c4a567d677
2021年07月28日 11:19:57 +10:00
# Remove this package from constraints before we install it.
# That way, later installs won't "downgrade" the install from
# source we are about to do.
local name
name=$(awk '/^name.*=/ {print 3ドル}' $project_dir/setup.cfg)
if [ -z $name ]; then
name=$(awk '/^name =/ {gsub(/"/, "", 3ドル); print 3ドル}' $project_dir/pyproject.toml)
fi
$REQUIREMENTS_DIR/.venv/bin/edit-constraints \
Revert "Workaround for new pip 20.3 behavior" This reverts commit 7a3a7ce876a37376fe0dca7278e41a4f46867daa and bcd0acf6c0b5d6501e91133c3a937b3fc40f7122 and part of f1ed7c77c50ac28cb58c9f7ed885c6a3e0a75403 which all cap our pip installs. Given the pip ecosystem can often incorporate major changes, tracking upstream at least generally gives us one problem at a time to solve rather than trying to handle version jumps when LTS distros update. The new dependency resolver included some changes that disallow setting URL's like "file:///path/to/project#egg=project" in constraints. Apparently the fact it used to work was an accident of the requires/constraints mechanism; it does make some sense as the URL doesn't really have a version-number that the resolver can put in an ordering graph. The _setup_package_with_constraints_edit function comment highlights what this is trying to do # Updates the constraints from REQUIREMENTS_DIR to reflect the # future installed state of this package. This ensures when we # install this package we get the from source version. In other words; if constraints has "foo==1.2.3" and Zuul has checked out "foo" for testing, we have to make sure pip doesn't choose version 1.2.3 from pypi. It seems like removing the entry from upper-requirements.txt is the important part; adding the URL path to the on-disk version was just something that seemed to work at the time, but isn't really necessary. We will install the package in question which will be the latest version (from Zuul checkout) and without the package in upper-requirements.txt nothing will try and downgrade it. Therefore the solution proposed here is to remove the adding of the URL parts. This allows us to uncap pip and restore testing with the new dependency resolver. Closes-Bug: #1906322 Change-Id: Ib9ba52147199a9d6d0293182d5db50c4a567d677
2021年07月28日 11:19:57 +10:00
$REQUIREMENTS_DIR/upper-constraints.txt -- $name
fi
setup_package $bindep $project_dir "$flags" $extras
Automatically set LIBS_FROM_GIT based on required projects If a project shows up in zuul's required-projects list, add it to LIBS_FROM_GIT automatically. This way, when a user specifies that a job requires a zuul-project, it gets used in testing, but otherwise, it doesn't (pypi is used instead). Also add information about what happens behind the scenes for both LIBS_FROM_GIT and plugin dependencies. This moves the check performed in check_libs_from_git to a helper function which is installed for most kinds of installations. This means that if someone sets LIBS_FROM_GIT to "foobar", devstack won't error anymore, as nothing is going to try to install foobar, therefore the check won't run on that. However, as we move to automated generation of the local config, that error is not likely to happen. This check was originally added due to an error in the upper-constraints file (where a constraint name did not match a package name). This location of the check would still catch that type of error. Change-Id: Ifcf3ad008cf42d3d4762cfb3b6c31c93cfeb40db
2018年03月02日 15:05:14 +00:00
# If this project is in LIBS_FROM_GIT, verify it was actually installed
# correctly. This helps catch errors caused by constraints mismatches.
if use_library_from_git "$project_dir"; then
if ! lib_installed_from_git "$project_dir"; then
die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
fi
fi
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`. The command is like
# "pip install <flags> <project_dir>[<extras>]"
#
# Uses globals ``STACK_USER``
#
# Usage:
# setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
#
# -bindep : Use bindep to install dependencies; select extra profiles
# as comma separated arguments after "="
# project_dir : directory of project repo (e.g., /opt/stack/keystone)
# flags : pip CLI options/flags
# extras : comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
function setup_package {
local bindep=0
local bindep_flag=""
local bindep_profiles=""
if [[ 1ドル == -bindep* ]]; then
bindep=1
IFS="=" read bindep_flag bindep_profiles <<< ${1}
shift
fi
local project_dir=1ドル
local flags=2ドル
local extras=3ドル
# if the flags variable exists, and it doesn't look like a flag,
# assume it's actually the extras list.
if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
extras=$flags
flags=""
fi
if [[ ! -z "$extras" ]]; then
extras="[$extras]"
fi
# install any bindep packages
if [[ $bindep == 1 ]]; then
install_bindep $project_dir/bindep.txt $bindep_profiles
fi
pip_install $flags "$project_dir$extras"
# ensure that further actions can do things like setup.py sdist
Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott <frickler@offenerstapel.de> Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
2018年04月04日 14:02:30 -07:00
if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
safe_chown -R $STACK_USER 1ドル/*.egg-info
fi
}
# Report whether python 3 should be used
# TODO(frickler): drop this once all legacy uses are removed
function python3_enabled {
return 0
}
# Provide requested python version and sets PYTHON variable
function install_python {
install_python3
export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null)
}
# Install python3 packages
function install_python3 {
if is_ubuntu; then
apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
elif is_fedora; then
if [ "$os_VENDOR" = "Fedora" ]; then
install_package python${PYTHON3_VERSION//.}
else
install_package python${PYTHON3_VERSION//.} python${PYTHON3_VERSION//.}-devel
fi
fi
}
function install_devstack_tools {
# intentionally old to ensure devstack-gate has control
local dstools_version=${DSTOOLS_VERSION:-0.1.2}
install_python3
sudo pip3 install -U devstack-tools==${dstools_version}
}
# Restore xtrace
$INC_PY_TRACE
# Local variables:
# mode: shell-script
# End: