#!/bin/false /bin/bash ## # Common functions used by the Python code. # set -eo pipefail # Defaults for the tools we shall use python_tool_test='nose' python_tool_coverage='coverage' python_tool_lint='pylint' python_tool_docs='pydoctor' # Default versions of the tools python_toolversion_nose='1.3.7' python_toolversion_coverage='4.4' python_toolversion_pylint='1.4.4' python_toolversion_pydoctor='15.1.0' # And the definitions of what we need in requirements python_tool_nose_1_3_7="nose==1.3.7" python_tool_coverage_4_4="coverage==4.4" python_tool_pylint_1_4_4=" pylint==1.4.4 astroid==1.3.6 logilab-common==1.0.2 six==1.10.0 " python_tool_pydoctor_15_1_0=" {2:pydoctor==15.1.0} doc2dash==2.1.0 lxml==3.4.4 zope.interface==4.1.3 alabaster==0.7.8 Babel==2.3.4 beautifulsoup4==4.4.1 attrs==15.2.0 click==6.3 colorama==0.3.7 docutils==0.12 epydoc==3.0.1 Jinja2==2.8 MarkupSafe==0.23 Pygments==2.1.3 pytz==2016.6.1 six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.3.5 sphinx-rtd-theme==0.1.9 Twisted==16.3.0 " PYTHON_TOOL=python PYTHON_SWITCH= VENV_DIR=venv function use_python3() { PYTHON_TOOL=python3 PYTHON_SWITCH=-3 VENV_DIR=venv3 } ## # Is python active for this action? # # @param $1 the action we are attempting to perform function python_active() { local action="$1" local files='' if [[ "$action" == 'test' || "$action" == 'coverage' ]] ; then files="$(config -e test_python_files '*_test.py')" elif [[ "$action" == 'lint' ]] ; then files="$(config -e lint_python_files '{python}')" elif [[ "$action" == 'docs' ]] ; then files="$(config -e docs_python_files '{python}')" elif [[ "$action" == 'clean' ]] ; then files="$(config -e docs_python_files '{python}')" files="$files $(config -e lint_python_files '{python}') " files="$files $(config -e test_python_files '*_test.py')" else # Unrecognised type return 1 fi files="$(trim "$files")" if [[ -n "$files" ]] ; then return 0 fi return 1 } ## # Matching functions for filename expansion. function python_match_magic() { echo -n 'Python script' } function python_match_extensions() { echo -n 'py' } ## # Help messages function python_help_any() { echo "-3 Use Python 3" } ## # Use Python 3 rather than Python 2 function python_switch_any_3() { use_python3 } ## # Set up the environment for Python # # @param $@ Requirements files to install function python_environment() { # Set up and enter the Virtual env. "${scripts}/python-env-setup" ${PYTHON_SWITCH} -e "${environment_dir}/${VENV_DIR}" \ "$@" < /dev/null source "${scripts}/python-env" -e "${environment_dir}/${VENV_DIR}" } function python_tool_requirements() { local action="$1" local reqs reqs="$(tool_config_expansion python "$action")" local req for req in $reqs ; do echo -n " +$req" done } function python_environment_test() { python_environment "${root}/requirements.txt" \ "${root}/requirements-test.txt" \ $(python_tool_requirements test) } function python_environment_coverage() { python_environment "${root}/requirements.txt" \ "${root}/requirements-test.txt" \ "${root}/requirements-coverage.txt" \ $(python_tool_requirements test) \ $(python_tool_requirements coverage) } function python_environment_lint() { python_environment "${root}/requirements.txt" \ "${root}/requirements-test.txt" \ "${root}/requirements-lint.txt" \ $(python_tool_requirements test) \ $(python_tool_requirements lint) } function python_environment_docs() { python_environment "${root}/requirements.txt" \ "${root}/requirements-docs.txt" \ $(python_tool_requirements docs) } function python_setup_test() { : # Nothing to do } function python_setup_coverage() { python_setup_test "$@" } function python_setup_lint() { : # Nothing to do } function python_setup_docs() { : # Nothing to do } function python_run_test() { local switches="$*" for test in $(config -e test_python_files '*_test.py') ; do if "$PYTHON_TOOL" "$test" \ --with-xunit --xunit-file "${artifact_dir}/test-results-${tests_total}.xml" \ -v $switches 2>&1 \ | output_filter "Test '$test'..." ; then tests_passed=$(( tests_passed + 1 )) else echo " Test '$test' failed" tests_failed=$(( tests_failed + 1 )) fi tests_total=$(( tests_total + 1 )) done } function python_run_coverage() { # We need to set the 'COVERAGE_FILE' variable so that the output from the # coverage is written to the log directory, rather than the root of the # project. This is also needed when processing the results. COVERAGE_FILE="${log_dir}/python-coverage" \ python_run_test --with-coverage --cover-branches } function python_run_lint() { if ! pylint --reports no \ --rcfile "${root}/pylintrc" \ $(config -e lint_python_files '{python}') 2>&1 \ | output_filter "Linting Python files" ; then lint_failed=$(( lint_failed+1 )) else lint_passed=$(( lint_passed+1 )) fi lint_total=$(( lint_total+1 )) } function python_run_docs() { if ! "${scripts}/python-build-docs" --initial-tag "$(config version_git_tag none)" \ --major-version "$(config version 1.0)" \ --paths "$(config docs_python_files '{python}')" \ --output-dir "${artifact_dir}/python-docs" \ | output_filter ; then docs_failed=$(( docs_failed+1 )) else docs_passed=$(( docs_passed+1 )) fi docs_total=$(( docs_total+1 )) } function python_run_clean() { local files='' local lastdir='' local thisdir files="$(config -e docs_python_files '*.py')" files="$files $(config -e lint_python_files '{python}') " files="$files $(config -e test_python_files '*_test.py')" lastdir='' for file in $files ; do file="./$file" if [[ "${file: -3}" == '.py' ]] ; then if [[ -f "${file//.py/.pyc}" ]] ; then rm "${file//.py/.pyc}" fi if [[ -f "${file//.py/.pyo}" ]] ; then rm "${file//.py/.pyo}" fi # Only check each directory once. if [[ "${file:0:${#lastdir} + 1}" != "${lastdir}/" ]] || \ [[ "${file:${#lastdir} + 1}" =~ / ]] ; then thisdir="$(dirname "$file")" if [[ -d "${thisdir}/__pycache__" ]] ; then rmtree "${thisdir:-THISDIR_NOT_SET}/__pycache__" fi lastdir="$thisdir" fi fi done } function python_process_test() { : # Nothing to do } function python_process_coverage() { python_process_test "$@" local ignore local ignore_options='' local pattern ignore="$(config coverage_python_ignore '*_test.py')" for pattern in ${ignore} ; do ignore_options="${ignore_options},${pattern}" done # Generate the text report COVERAGE_FILE="${log_dir}/python-coverage" \ coverage report --omit "${ignore_options}" \ > "${artifact_dir}/python-coverage.txt" # And the HTML report COVERAGE_FILE="${log_dir}/python-coverage" \ coverage html --dir "${artifact_dir}/python-coverage" \ --omit "${ignore_options}" # And the XML report COVERAGE_FILE="${log_dir}/python-coverage" \ coverage xml -o "${artifact_dir}/coverage.xml" \ --omit "${ignore_options}" coverage_percentage=$(tail "${artifact_dir}/python-coverage.txt" \ | sed -n 's/^.*[^0-9]\([0-9][0-9\.]*\)%.*$/\1/ p' \ | awk 'END { print int($1) }') echo "Python coverage: ${coverage_percentage}%" } function python_process_lint() { : # Nothing to do } function python_process_docs() { : # Nothing to do }