python-env-setup 8.19 KB
Newer Older
1 2 3 4 5 6
#!/bin/bash
##
# Create a virtualenv environment containing the environment that we need to run
# build and test code in.
#
# Common usage:
7
#   ci/python-env-setup [-e <environment>] [-3] [-v] requirements.txt ...
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
#
# The requirements files may use the following format for their lines:
#   any-normal-requirement
#       - Requirement used for both Python 2 and Python 3
#   {2:requirement-for-python2}
#       - Requirement only to be used in Python 2
#   {3:requirement-for-python3}
#       - Requirement only to be used in Python 3
#
# Alternatively the requirements passed may be an explicit requirement,
# prefixed by a '+', rather than a filename.
#

set -eo pipefail

# The icon to use beside the environment name.
# This is a Snake symbol, to indicate this is Python.
environment_icon="🐍 "

# VirtualEnv requirements, so that we don't remove them
requirements_virtualenv="
pip
setuptools
wheel
appdirs
packaging
pyparsing
six
"


##
# Convert a given path into one that starts from the root.
#
# @param $1     Path to make absolute
function abspath() {
    local path="$1"
    if [[ "$path" != '' && "${path:0:1}" != '/' ]] ; then
        path="$(pwd)/$path"
    fi

    while [[ "$path" =~ /\.\.?/ || "$path" =~ /\.\.$ ]] ; do
        path="$(sed -E 's!//*!/!g;
                        s!/\.\/!/!g;
                        s!/[^/][^/]*/\.\./!/!;
                        s!/[^/][^/]*/\.\.$!/!;
                        s!^/\.\./!/!;
                        s!^/\.\.$!/!;' <<< "$path")"
    done

    local parent
    local leaf
    parent="$(dirname "$path")"
    leaf="$(basename "$path")"
    if [[ "$leaf" == '.' ]] ; then
        leaf=''
    fi

    if [[ "$parent" != '/' && "$parent" != '.' && ! -d "$parent" ]] ; then
        path="$(abspath "$parent")${leaf:+/$leaf}"
    else
        path="$( cd "$parent" && pwd -P)${leaf:+/$leaf}"
    fi

    echo -n "${path//\/\//\/}"
}


function usage() {
    script_name=$(basename "${BASH_SOURCE[0]}")
    cat <<EOM
Usage: $script_name [-3] [-v] [-e <environment>] {<requirements>}*

Script for configuring and updating a virtualenv based Python environment.

Optional arguments:
    -v                    verbose; be more noisy about what is being done
    -e <env-dir>          path to the directory in which to create the
                          environment (default 'perllib')
    -3                    build the environment for Python 3
    -h                    show this help message and exit

Positional arguments:
    requirements          path to the pip formatted (requirements.txt) Python requirements
                          OR '+<requirement line>'
EOM
}

function error() {
    local extra_path=""
    local os
    os="$(uname -s)"
    if [[ "$os" == 'Linux' ]] ; then
        extra_path="~/.local/bin"
    elif [[ "$os" == 'Darwin' ]] ; then
        extra_path="~/Library/Python/2.7/bin"
    else
        extra_path="<user python path for $os>"
    fi

    cat <<EOM >&2
You may need to install pip and virtualenv:

    easy_install --user pip
    easy_install --user virtualenv

You may also need to add the easy_install directory to your path:

    export PATH=$extra_path:\$PATH
EOM
}

120 121 122 123 124 125 126
##
# Filter out the warnings that are produced by Python 2.7's old SSL library.
function filter_warnings() {
    grep -Ev '^(.*ssl_.py:[0-9]*: |  )(SNIMissingWarning|InsecurePlatformWarning)'
}


127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
scriptdir=$( abspath "$(dirname "${BASH_SOURCE[0]}")" )

python_tool=python2.7
pip_tool=pip
sed_rules='s/\{[^2]:(.*)\}//g; s/\{2:(.*)\}/\1/g;'

environment="${scriptdir}/../venv"

# Options.
quiet=-q
while getopts ":he:3v" opt ; do
    case $opt in
        v)
            quiet=
            ;;
        e)
            environment="$OPTARG"
            ;;
        3)
            python_tool=python3
            pip_tool=pip3
            sed_rules='s/\{[^3]:(.*)\}//g; s/\{3:(.*)\}/\1/g;'
            ;;
        h)
            usage
            exit
            ;;
    esac
done


# Our temporary directory
tmpdir="$(mktemp -d -t python-env.XXXXXXXX)"
if [ "$?" != '0' -o "$tmpdir" == '' ] ; then
    echo "Cannot create temporary directory. That would be bad." >&2
    exit 1
fi
function cleanup() {
    if [[ "$(uname -s)" == 'Darwin' ]] ; then
        # On OSX, '--one-file-system' does not exist.
        rm -rf "${tmpdir}"
    else
        rm -rf --one-file-system "${tmpdir}"
    fi
}
trap cleanup EXIT


tmprequirements="${tmpdir}/requirements.txt"
# Apply the transformation rules to all the files in the parameters.
for file in "${@:$OPTIND}" ; do
    if [[ "${file:0:1}" == '+' ]] ; then
        echo "${file:1}"
    elif [ -f "$file" ] ; then
        cat "$file"
    fi
done | sed -E "${sed_rules}; s/^#.*//" \
    | sort -u > "${tmprequirements}"


mkdir -p "${environment}"
environment="$( abspath "${environment}" )"

environment_name="$(basename "$environment")"
envpath="$environment"
while [[ "$environment_name" == 'venv' ||
         "$environment_name" == 'virtualenv' ||
         "$environment_name" == 'python' ||
         "${environment_name:0:1}" == '.' ]] ; do
    # That's not a helpful name, so use the name of the parent directory
    envpath="$(dirname "$envpath")"
    environment_name="$(basename "$envpath")"
done



# Activate the virtualenv.
if [[ ! -e "${environment}/bin/activate" ]]; then
    if ! virtualenv -p "${python_tool}" \
                    --prompt "(<icon>${environment_name}) " \
                    --no-site-packages \
                    ${quiet:-} \
209 210
                    "${environment}" \
            2> >(filter_warnings >&2); then
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
        error
        exit 1
    fi

    # We cannot place the actual icon into the prompt name in the virtualenv.
    # If we do, the tool complains that it cannot process the string in 'ascii'.
    # So instead, we post-process the activation scripts.
    sed_inplace_option='-i'
    if [[ "$(uname -s)" == 'Darwin' ]] ; then
        sed_inplace_option='-i ""'
    fi
    for file in "${environment}/bin/activate"* ; do
        sed $sed_inplace_option "s/<icon>/${environment_icon}/g" "${file}"
    done
fi

source "${environment}/bin/activate" || exit 1

if [ "$quiet" == '' ] ; then
    echo Installing packages into virtualenv with PIP ...
fi
if ! "${pip_tool}" install \
                   --disable-pip-version-check \
                   ${quiet:-} \
235 236
                   -r "${tmprequirements}" \
        2> >(filter_warnings >&2) ; then
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
    error
    exit 1
fi

if [ "$quiet" == '' ] ; then
    echo Checking for any packages that are not in the requirements...
fi

# The echos in the second argument ensure that PIP and Easy_Install are not removed
# from the virtualenv.
# The stderr redirection hides a warning about the format of 'list' changing -
# which we do not care about because the bit we need is the list of names which
# will be unchanged.
set +e
to_remove=($(diff <("${pip_tool}" list --disable-pip-version-check \
                        2> >(grep -v 'DEPRECATION: The default format' >&2) \
                        | cut -d' ' -f 1 \
                        | sort) \
                  <((echo "$requirements_virtualenv" ; \
                     cut -d'=' -f 1 "${tmprequirements}") | sort) \
             | grep '^<' \
             | cut -d' ' -f2))
set -e
if [[ "${#to_remove[@]}" != 0 ]] ; then
    if [ "$quiet" == '' ] ; then
        echo "  - removing ${#to_remove[@]} packages"
    fi
    if ! "${pip_tool}" uninstall --disable-pip-version-check \
                                 ${quiet:-} -y \
                                 "${to_remove[@]}" ; then
        echo "Could not remove the unneeded packages. Maybe deleting '${environment}' will help?" >&2
        exit 1
    fi
else
    if [[ -z "${quiet:-}" ]] ; then
        echo "  - none to remove"
    fi
fi

if [[ -t 0 && -t 1 ]] ; then
    # Tiny bit of simplification of the path
    dir="${scriptdir}"
    pwd="$(pwd -P)/"
    home="$HOME/"
    if [[ "${dir:0:${#pwd}}" == "$pwd" ]] ; then
        dir="${dir:${#pwd}}"
    elif [[ "${dir:0:${#home}}" == "$home" ]] ; then
        dir="~/${dir:${#home}}"
    fi

    cat <<EOM
To use the Python virtual environment, use:

  ${dir}/python-env <commands>
      - to execute commands within the environment and return
  ${dir}/python-env -s
      - to drop to a shell within the environment (bash and fish supported)
  source ${dir}/python-env
      - to enter the environment directly (bash only)
EOM
fi