Skip to content

GitLab

Commit b694baaf authored by Charles Ferguson's avatar Charles Ferguson
Browse files

Creating a new submodule for the CI scripts. 

In order that we keep the CI scripts in line, it is better to have them
in a submodule that can be synchronised separately. This avoids the
problem of having to update the scripts directory in every component.
parents
Hide whitespace changes
Inline Side-by-side
Showing with 2196 additions and 0 deletions
common 0 → 100644
View file @ b694baaf
#!/bin/false /bin/bash
##
# Common functions used by the test code.
#
set -eo pipefail
scripts="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd -P)"
root="$(cd "$(dirname "${scripts}")" > /dev/null && pwd -P)"
##
# Convert a string to a boolean return code
function bool() {
local value
value="$(tr '[:upper:]' '[:lower:]' <<<"$1")"
if [[ '-yes-true-on-1-enable-enabled-' =~ -${value}- ]] ; then
return 0
elif [[ '-no-false-off-0-disable-disabled-' =~ -${value}- ]] ; then
return 1
fi
# Put a warning out on stderr so that we know.
echo "Boolean parameter '$1' not recognised" >&2
return 1
}
##
# Read a configuration parameter from the project's config.
#
# Option '-e' can be used to expand wildcards.
# Option '-b' can be used to return a boolean state for the string.
#
# @param $1 The key to read from the configuration file
# @param $2 The default value to return if the config is not set
function config() {
if [ "$1" = '-e' ] ; then
shift
expand_filenames "$(config "$@")"
return
elif [ "$1" = '-b' ] ; then
shift
bool "$(config "$@")"
return
fi
local key="$1"
local default="${2:-}"
if [[ ! -f "${root}/project.config" ]] ; then
# No configuration file
echo "${default}"
elif grep -q "^${key}: \?" "${root}/project.config" ; then
# Key found, so we can return it
grep "^${key}: \?" "${root}/project.config" | sed "s/^${key}: *//" || true
else
# Key not found in configuration file
echo "${default}"
fi
}
##
# Expand a set of filenames from wildcards.
#
# @param $@ filenames with wildcards
function expand_filenames() {
(
# If nothing matches a wildcard string, return empty
shopt -s nullglob
# Note: Cannot use 'shopt -s globstar' to allow '**' to be subdirectory match.
# It isn't supported on OSX.
eval echo "$*"
)
}
##
# Find the tool configuration for a given action.
#
# Most actions require a particular tool to be used.
# These can be configured in the project configuration file and will
# fall back to the language configuration.#
#
# project.config:
# <action>_<language>_tool
# common.<language> variable:
# <language>_tool_<action>
#
# The ordering of names it in line with each file's definitions,
# based on how the files are to be used.
#
# Commonly, the tool configuration will contain either:
# <module-name>
# OR
# <module-name> <version>
#
# The expansion of these names can be performed with the function
# 'tool_config_expansion'.
function tool_expansion() {
local lang="$1"
local action="$2"
local toolvar="${lang}_tool_${action}"
local default="${!toolvar}"
config "${action}_${lang}_tool" "$default"
}
##
# Return the configuration for the tool in use for a given action.
#
# The actions may use different tools (eg 'nose' or 'unittest' for
# the 'test' action, in python), with different versions. Each version
# requires a different configuration.
#
# This function will expand the configuration given by the user (or the
# default) and return configuration entries.
#
# Given the configuration value for a <language>/<action> combination
# (see the 'tool_expansion' function), this function expands the variables
# to give the specific configuration.
#
# If the configuration contains a string previxed by equal ('='), the
# following content is considered the literal configuration to return.
#
# If only a single module name was given in the configuration, then the
# default tool version is used.
# This default tool version is looked up in the environment variable
# <language>_toolversion_<module>
#
# If the version number was given in the configuration, it is used
# verbatim.
#
# The configuration itself resides in:
# <language>_tool_<module>_<version-with-underscores>
# The version has periods ('.') replaced with underscores ('_').
# The content of this variable is returned.
function tool_config_expansion() {
local lang="$1"
local action="$2"
local toolpair="$(tool_expansion "$lang" "$action")"
if [[ "$toolpair" == '' ]] ; then
echo "Tool configuration for language '${lang}', action '${action}' is not known" >&2
exit 1
fi
if [[ "${toolpair:0:1}" == '=' ]] ; then
# They specified a literal configuration
echo "${toolpair:1}"
return 0
fi
local tool
local version
if [[ "$toolpair" =~ \ ]] ; then
tool="${toolpair%% *}"
version="${toolpair#* }"
else
tool="$toolpair"
local var="${lang}_toolversion_${tool}"
version="${!var}"
if [[ "${version}" == '' ]] ; then
echo "Default version for language '${lang}' tool '${tool}' is not known" >&2
exit 1
fi
fi
local confvar="${lang}_tool_${tool}_${version//./_}"
echo "${!confvar}"
return 0
}
##
# Check if a named function exists
#
# @param $1 function name to check
function is_function() {
local funcname="$1"
if [ "$(type -t "$funcname")" == 'function' ] ; then
return 0
fi
return 1
}
##
# Call the named function if it exists.
#
# @param $1 function name to call
function call_if_function() {
local funcname="$1"
shift
if is_function "$funcname" ; then
"$funcname" "$@"
fi
}
function all_setup_test() {
local running="${1:-tests}"
echo ">>> Running ${running}"
tests_passed=0
tests_failed=0
tests_total=0
}
function all_setup_coverage() {
all_setup_test coverage
coverage_percentage='<unknown>'
coverage_limit=$(config coverage_limit 0)
}
function all_setup_lint() {
echo '>>> Lint code'
lint_passed=0
lint_failed=0
lint_total=0
}
function all_setup_docs() {
echo '>>> Building documentation'
docs_passed=0
docs_failed=0
docs_total=0
}
function all_end_test() {
if [[ "$tests_failed" != "0" ]] ; then
echo "<<< Tests FAILED ($tests_failed test files failed, $tests_passed test files passed)"
exit 1
fi
echo "<<< Tests passed ($tests_passed test files)"
exit 0
}
function all_end_coverage() {
echo "Overall Coverage: ${coverage_percentage}% (limit is ${coverage_limit}%)"
if [[ "${coverage_percentage}" != '<unknown>' && \
"${coverage_percentage%.*}" -lt "${coverage_limit}" ]] ; then
echo "<<< Coverage failed (${coverage_percentage%.*}% < ${coverage_limit}%)"
exit 1
fi
echo "<<< Coverage passed"
exit 0
}
function all_end_lint() {
if [[ "$lint_failed" != "0" ]] ; then
echo "<<< Lint FAILED ($lint_failed checks failed, $lint_passed checks passed)"
exit 1
fi
echo "<<< Lint passed ($lint_passed checks)"
exit 0
}
function all_end_docs() {
if [[ "$docs_failed" != "0" ]] ; then
echo '<<< Documentation failed'
exit 1
fi
echo '<<< Documentation built'
exit 0
}
##
# Provide help messages.
#
# @param $1 Name of the action, or '' if none
function help_message() {
local action="$1"
local global_message
local action_message
if [[ "$action" == '' ]] ; then
echo "Syntax: $0 <global-options> {${actions// /|}} <action-options>"
else
echo "Syntax: $0 <global-options> $action <action-options>"
fi
global_message=$'Global options:\n'
for lang in $languages ; do
if ! active "${lang}" "${action}" ; then
continue
fi
if is_function "${lang}_help_any" ; then
echo -n "$global_message"
global_message=''
"${lang}_help_any" "$action" | sed 's/^/ /'
fi
done
if [[ "$action" != '' ]] ; then
action_message=$'\nOptions for action '"'$action'"$':\n'
for lang in $languages ; do
if ! active "${lang}" "${action}" ; then
continue
fi
if is_function "${lang}_help_${action}" ; then
echo -n "$action_message"
action_message=''
"${lang}_help_${action}" "$action" | sed 's/^/ /'
fi
done
fi
exit 0
}
##
# Check if a language/action combination is active or not
#
# @param $1 Language to check
# @param $2 Action to check, or '' to check if any active
function active() {
local lang="$1"
local action="$2"
if ! config -b "${lang}_enabled" 'true' ; then
return 1
fi
if [[ "$action" != '' ]] ; then
"${lang}_active" "$action"
return
fi
for action in $actions ; do
if "${lang}_active" "$action" ; then
return 0
fi
done
return 1
}
##
# Execute one of the actions based on the languages and activations
# we have.
#
# @param $1 Name of the action ('test', 'coverage', 'docs', 'lint')
function run_action() {
local action
local opt
local value
local optfunc
local handled
# Process any additional arguments that we were passed.
local next_param=false
for param in "$@" ; do
if "$next_param" "$param" ; then
next_param=false
continue
fi
if [[ "${param:0:1}" == '-' ]] ; then
if [[ "${param:0:2}" == '--' ]] ; then
param="${param:2}"
else
param="${param:1}"
fi
if [[ "${param}" == 'h' || "${param}" == 'help' ]] ; then
# Help is special, as we'll call each language's functions
help_message "$action"
exit 0
else
handled=false
if [[ "$param" =~ = ]] ; then
value=${param#*=}
opt=${param%%=*}
else
value=''
opt=${param}
fi
optfunc=${opt//-/_}
for lang in $languages ; do
if ! active "$lang" "$action" ; then
continue
fi
if [[ -z "$value" ]] ; then
if is_function "${lang}_switch_any_${optfunc}" ; then
"${lang}_switch_any_${optfunc}" "$action"
handled=true
elif is_function "${lang}_switch_${action}_${optfunc}" ; then
"${lang}_switch_${action}_${optfunc}" "$action"
handled=true
elif is_function "${lang}_param_any_${optfunc}" ; then
next_param="${lang}_param_any_${optfunc}"
handled=true
elif is_function "${lang}_param_${action}_${optfunc}" ; then
next_param="${lang}_param_${action}_${optfunc}"
handled=true
fi
else
# We were given a parameter, so we handle this now
if is_function "${lang}_param_any_${optfunc}" ; then
"${lang}_param_any_${optfunc}" "${value}"
handled=true
elif is_function "${lang}_param_${action}_${optfunc}" ; then
"${lang}_param_${action}_${optfunc}" "${value}"
handled=true
fi
fi
done
if ! "$handled" ; then
echo "Option '$opt' is not understood" >&2
exit 1
fi
fi
else
if [[ "$action" == '' ]] ; then
action="$param"
else
echo "Positional parameters are not supported ('$param')" >&2
exit 1
fi
fi
done
if [[ "$action" == '' || "$action" == 'help' ]] ; then
help_message
fi
# Begin the action, and set up any variables needed.
if ! is_function "all_setup_${action}" ; then
echo "Action '${action}' not known" >&2
exit 1
fi
"all_setup_${action}"
for lang in $languages ; do
if active "$lang" "$action" ; then
# 1. Build and use an environment in which the tests can function.
# This may be CPAN, VirtualEnv, or some other specialism.
call_if_function "${lang}_environment_${action}"
# 2. Configure the tests, with environment variables, clear up
# any left-over resources that we do not want.
call_if_function "${lang}_setup_${action}"
# 3. Execute the action, and update the state variables used by
# the 'all_setup_*' functions.
call_if_function "${lang}_run_${action}"
# 4. Process the results of the action, generating reports,
# compressing files, or other operations that are needed.
call_if_function "${lang}_process_${action}"
fi
done
# Report the results of the action, and call exit appropriately.
"all_end_${action}"
}
# Load the scripts for the languages
languages=""
for file in $(expand_filenames "${scripts}"'/common.*') ; do
languages="$languages ${file##*.}"
source "${scripts}/common.${file##*.}"
done
actions="test coverage lint docs"
common.perl 0 → 100644
View file @ b694baaf
#!/bin/false /bin/bash
##
# Common functions for Perl.
#
set -eo pipefail
##
# Is perl active for this action?
#
# @param $1 the action we are attempting to perform
function perl_active() {
local action="$1"
local files=''
if [[ "$action" == 'test' || "$action" == 'coverage' ]] ; then
files="$(config -e test_perl_files '*-test.pl')"
elif [[ "$action" == 'lint' ]] ; then
files="$(config -e lint_perl_files '*.pl *.pm')"
else
# Unrecognised type
return 1
fi
if [[ -n "$files" ]] ; then
return 0
fi
return 1
}
##
# Help messages
function perl_help_lint() {
echo "--checks List all the enabled perlcritic checks"
}
##
# List what options would be enabled
function perl_switch_lint_checks() {
perl_environment_lint
echo "Perl Critic checks enabled:"
perlcritic -profile "${root}/perlcriticrc" --list-enabled \
| sed 's/^/ /'
exit
}
##
# Set up the environment for Perl.
#
# @param $@ CPAN requirements to install (either filenames, or names of modules preceeded by a '+')
function perl_environment() {
"${scripts}/perl-env-setup" -e "${root}/perllib" "$@" < /dev/null
source "${scripts}/perl-env" -e "${root}/perllib"
}
function perl_environment_test() {
perl_environment +PadWalker@2.2 \
"${root}/cpan.txt" \
"${root}/cpan-test.txt" \
"${scripts}/cpan-test.txt"
}
function perl_environment_coverage() {
perl_environment +PadWalker@2.2 +Devel::Cover@1.25 \
"${root}/cpan.txt" \
"${root}/cpan-test.txt" \
"${root}/cpan-coverage.txt" \
"${scripts}/cpan-test.txt" \
"${scripts}/cpan-coverage.txt"
}
function perl_environment_lint() {
perl_environment +PadWalker@2.2 +Perl::Critic@1.126 +PPI@1.220 \
"${root}/cpan.txt" \
"${root}/cpan-lint.txt" \
"${scripts}/cpan-lint.txt"
}
function perl_setup_test() {
: # Nothing to do
}
function perl_setup_coverage() {
local ignore
local ignore_options=''
local pattern
ignore="$(config coverage_perl_ignore '-test JFTest')"
for pattern in ${ignore} ; do
pattern=${pattern//+/[+]}
pattern=${pattern//-/[-]}
ignore_options="${ignore_options},+ignore_re,${pattern}"
done
export PERL5OPT="MDevel::Cover=-db,cover_db$ignore_options,-silent,1"
# Clear away anything left.
if [[ "$(uname -s)" == 'Darwin' ]] ; then
# On OSX, '--one-file-system' does not exist.
rm -rf cover_db test_results
else
rm -rf --one-file-system cover_db test_results
fi
# Ensure that the results directory exists.
mkdir test_results
}
function perl_setup_lint() {
: # Nothing to do
}
function perl_run_test() {
for test in $(config -e test_perl_files '*-test.pl') ; do
echo " Test '$test'..."
if perl "$test" | sed 's/^/ /' ; then
tests_passed=$(( tests_passed + 1 ))
else
echo " Test '$test' failed"
tests_failed=$(( tests_failed + 1 ))
fi
tests_total=$(( tests_total + 1 ))
done
}
# Running the actual coverage is just calling the tests
function perl_run_coverage() {
perl_run_test "$@"
}
function perl_run_lint() {
for file in $(config -e lint_perl_files '*.pl *.pm') ; do
if ! perl -c "$file" 2>&1 | sed 's/^/ /'; then
lint_failed=$(( lint_failed+1 ))
else
lint_passed=$(( lint_passed+1 ))
fi
done
if ! perlcritic --profile perlcriticrc \
--color \
$(config -e lint_perl_files '*.pl *.pm') \
2>&1 | sed 's/^/ /'; then
lint_failed=$(( lint_failed+1 ))
else
lint_passed=$(( lint_passed+1 ))
fi
lint_total=$(( lint_total+1 ))
}
function perl_process_test() {