common.python 8.22 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
#!/bin/false /bin/bash
##
# Common functions used by the Python code.
#

set -eo pipefail

# Defaults for the tools we shall use
python_tool_test='nose'
python_tool_coverage='coverage'
python_tool_lint='pylint'
python_tool_docs='pydoctor'

# Default versions of the tools
python_toolversion_nose='1.3.7'
python_toolversion_coverage='4.4'
python_toolversion_pylint='1.4.4'
python_toolversion_pydoctor='15.1.0'

# And the definitions of what we need in requirements
python_tool_nose_1_3_7="nose==1.3.7"
python_tool_coverage_4_4="coverage==4.4"
python_tool_pylint_1_4_4="
pylint==1.4.4
astroid==1.3.6
logilab-common==1.0.2
six==1.10.0
"
python_tool_pydoctor_15_1_0="
{2:pydoctor==15.1.0}
doc2dash==2.1.0
lxml==3.4.4
zope.interface==4.1.3
alabaster==0.7.8
Babel==2.3.4
beautifulsoup4==4.4.1
attrs==15.2.0
click==6.3
colorama==0.3.7
docutils==0.12
epydoc==3.0.1
Jinja2==2.8
MarkupSafe==0.23
Pygments==2.1.3
pytz==2016.6.1
six==1.10.0
snowballstemmer==1.2.1
Sphinx==1.3.5
sphinx-rtd-theme==0.1.9
Twisted==16.3.0
"

PYTHON_TOOL=python
PYTHON_SWITCH=
VENV_DIR=venv

function use_python3() {
    PYTHON_TOOL=python3
    PYTHON_SWITCH=-3
    VENV_DIR=venv3
}


##
# Is python active for this action?
#
# @param $1     the action we are attempting to perform
function python_active() {
    local action="$1"

    local files=''
    if [[ "$action" == 'test' || "$action" == 'coverage' ]] ; then
        files="$(config -e test_python_files '*_test.py')"
    elif [[ "$action" == 'lint' ]] ; then
75
        files="$(config -e lint_python_files '{python}')"
76
    elif [[ "$action" == 'docs' ]] ; then
77
        files="$(config -e docs_python_files '{python}')"
78
    elif [[ "$action" == 'clean' ]] ; then
79 80
        files="$(config -e docs_python_files '{python}')"
        files="$files $(config -e lint_python_files '{python}') "
81
        files="$files $(config -e test_python_files '*_test.py')"
82 83 84 85 86
    else
        # Unrecognised type
        return 1
    fi

87 88
    files="$(trim "$files")"

89 90 91 92 93 94 95 96
    if [[ -n "$files" ]] ; then
        return 0
    fi

    return 1
}


97 98 99 100 101 102 103 104 105 106
##
# Matching functions for filename expansion.
function python_match_magic() {
    echo -n 'Python script'
}
function python_match_extensions() {
    echo -n 'py'
}


107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
##
# Help messages
function python_help_any() {
    echo "-3    Use Python 3"
}


##
# Use Python 3 rather than Python 2
function python_switch_any_3() {
    use_python3
}


##
# Set up the environment for Python
#
# @param $@ Requirements files to install
function python_environment() {
    # Set up and enter the Virtual env.
127
    "${scripts}/python-env-setup" ${PYTHON_SWITCH} -e "${environment_dir}/${VENV_DIR}" \
128
                                  "$@" < /dev/null
129
    source "${scripts}/python-env" -e "${environment_dir}/${VENV_DIR}"
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
}

function python_tool_requirements() {
    local action="$1"
    local reqs
    reqs="$(tool_config_expansion python "$action")"
    local req
    for req in $reqs ; do
        echo -n " +$req"
    done
}


function python_environment_test() {
    python_environment "${root}/requirements.txt" \
                       "${root}/requirements-test.txt" \
                       $(python_tool_requirements test)
}

function python_environment_coverage() {
    python_environment "${root}/requirements.txt" \
                       "${root}/requirements-test.txt" \
                       "${root}/requirements-coverage.txt" \
                       $(python_tool_requirements test) \
                       $(python_tool_requirements coverage)
}

function python_environment_lint() {
    python_environment "${root}/requirements.txt" \
                       "${root}/requirements-test.txt" \
                       "${root}/requirements-lint.txt" \
                       $(python_tool_requirements test) \
                       $(python_tool_requirements lint)
}

function python_environment_docs() {
    python_environment "${root}/requirements.txt" \
                       "${root}/requirements-docs.txt" \
                       $(python_tool_requirements docs)
}


function python_setup_test() {
    : # Nothing to do
}

function python_setup_coverage() {
    python_setup_test "$@"
}

function python_setup_lint() {
    : # Nothing to do
}

function python_setup_docs() {
    : # Nothing to do
}



function python_run_test() {
    local switches="$*"

    for test in $(config -e test_python_files '*_test.py') ; do
194 195 196
        if "$PYTHON_TOOL" "$test" \
                          --with-xunit --xunit-file "${artifact_dir}/test-results-${tests_total}.xml" \
                          -v $switches 2>&1 \
197
                | output_filter "Test '$test'..." ; then
198 199 200 201 202 203 204 205 206 207
            tests_passed=$(( tests_passed + 1 ))
        else
            echo "  Test '$test' failed"
            tests_failed=$(( tests_failed + 1 ))
        fi
        tests_total=$(( tests_total + 1 ))
    done
}

function python_run_coverage() {
208 209 210 211 212
    # We need to set the 'COVERAGE_FILE' variable so that the output from the
    # coverage is written to the log directory, rather than the root of the
    # project. This is also needed when processing the results.
    COVERAGE_FILE="${log_dir}/python-coverage" \
        python_run_test --with-coverage --cover-branches
213 214 215
}

function python_run_lint() {
216
    if ! pylint --reports no \
217
                --rcfile "${root}/pylintrc" \
218
                $(config -e lint_python_files '{python}') 2>&1 \
219
            | output_filter "Linting Python files" ; then
220 221 222 223 224 225 226 227 228 229
        lint_failed=$(( lint_failed+1 ))
    else
        lint_passed=$(( lint_passed+1 ))
    fi
    lint_total=$(( lint_total+1 ))
}

function python_run_docs() {
    if ! "${scripts}/python-build-docs" --initial-tag "$(config version_git_tag none)" \
                                        --major-version "$(config version 1.0)" \
230
                                        --paths "$(config docs_python_files '{python}')" \
231
                                        --output-dir "${artifact_dir}/python-docs" \
232
            | output_filter ; then
233 234 235 236 237 238 239
        docs_failed=$(( docs_failed+1 ))
    else
        docs_passed=$(( docs_passed+1 ))
    fi
    docs_total=$(( docs_total+1 ))
}

240 241 242 243 244 245
function python_run_clean() {
    local files=''
    local lastdir=''
    local thisdir

    files="$(config -e docs_python_files '*.py')"
246
    files="$files $(config -e lint_python_files '{python}') "
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
    files="$files $(config -e test_python_files '*_test.py')"

    lastdir=''
    for file in $files ; do
        file="./$file"
        if [[ "${file: -3}" == '.py' ]] ; then
            if [[ -f "${file//.py/.pyc}" ]] ; then
                rm "${file//.py/.pyc}"
            fi
            if [[ -f "${file//.py/.pyo}" ]] ; then
                rm "${file//.py/.pyo}"
            fi

            # Only check each directory once.
            if [[ "${file:0:${#lastdir} + 1}" != "${lastdir}/" ]] || \
               [[ "${file:${#lastdir} + 1}" =~ / ]] ; then
                thisdir="$(dirname "$file")"
                if [[ -d "${thisdir}/__pycache__" ]] ; then
                    rmtree "${thisdir:-THISDIR_NOT_SET}/__pycache__"
                fi
                lastdir="$thisdir"
            fi
        fi
    done
}

273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289

function python_process_test() {
    : # Nothing to do
}

function python_process_coverage() {
    python_process_test "$@"

    local ignore
    local ignore_options=''
    local pattern
    ignore="$(config coverage_python_ignore '*_test.py')"

    for pattern in ${ignore} ; do
        ignore_options="${ignore_options},${pattern}"
    done

290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
    # Generate the text report
    COVERAGE_FILE="${log_dir}/python-coverage" \
        coverage report --omit "${ignore_options}" \
            > "${artifact_dir}/python-coverage.txt"

    # And the HTML report
    COVERAGE_FILE="${log_dir}/python-coverage" \
        coverage html --dir "${artifact_dir}/python-coverage" \
                      --omit "${ignore_options}"

    # And the XML report
    COVERAGE_FILE="${log_dir}/python-coverage" \
        coverage xml -o "${artifact_dir}/coverage.xml" \
                     --omit "${ignore_options}"

    coverage_percentage=$(tail "${artifact_dir}/python-coverage.txt" \
306 307 308 309 310 311 312 313 314 315 316 317
                            | sed -n 's/^.*[^0-9]\([0-9][0-9\.]*\)%.*$/\1/ p' \
                            | awk 'END { print int($1) }')
    echo "Python coverage: ${coverage_percentage}%"
}

function python_process_lint() {
    : # Nothing to do
}

function python_process_docs() {
    : # Nothing to do
}