Commit 27a8d121 authored by Charles Ferguson's avatar Charles Ferguson
Browse files

Initial import of the streamedinput module.

Provides callback functions which are supplied with data as it is
received from a subprocess.
parents
# All text files should be LFs
* text=auto eol=lf
*.pyc
build-docs
/.project
venv
venv3
lint:
stage: build
script:
- bash scripts/lint
python2:
stage: test
script:
- bash scripts/test
python3:
stage: test
script:
- bash scripts/test -3
docs:
stage: build
script:
- bash scripts/docs
artifacts:
paths:
- build-docs
stages:
- build
- test
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Profiled execution.
profile=no
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Pickle collected data for later comparisons.
persistent=no
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable all messages by default.
enable=all
# Disable all messages we disagree with.
# N.B. Although the documentation suggests we can safely use multiple lines for 'enable' and 'disable' keywords, this is
# not true: they must all be listed on one long line.
disable=missing-docstring, too-few-public-methods, too-many-locals, too-many-branches, too-many-public-methods, fixme, too-many-statements, too-many-return-statements, too-many-instance-attributes, too-many-arguments, too-many-lines, invalid-name, locally-disabled, suppressed-message, superfluous-parens
# TODO We need to re-enable the following checks at some point:
# - bad-continuation
# - too-many-return-statements
# - too-many-instance-attributes
# - too-many-arguments
# - missing-docstring
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Add a comment according to your evaluation note. This is used by the global
# evaluation report (RP0004).
comment=no
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
msg-template={msg_id}:{line:3d},{column}: {obj}: {msg} [{symbol}]
[BASIC]
# Required attributes for module, separated by a comma
required-attributes=
# List of builtins function names that should not be used, separated by a comma
bad-functions=map, filter, input, assert
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_,e,log,m,maxDiff,fh,venv
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=yes
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,40}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,40}$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{2,40}$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,40}$
# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,40}$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,40}$
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,40}$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,40}$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,40}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,40}|(__.*__))$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,40}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,40}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=__.*__
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=120
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis
ignored-modules=
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
ignored-classes=SQLObject
# When zope mode is activated, add a predefined set of Zope acquired attributes
# to generated-members.
zope=no
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
generated-members=REQUEST,acl_users,aq_parent
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=_$|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
[CLASSES]
# List of interface methods to ignore, separated by a comma. This is used for
# instance to not check methods defines in Zope's Interface base class.
ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
# Use:
# {2:only in Python2}
# {3:only in Python3}
#!/bin/bash
##
# Create a virtualenv environment containing the environment that we need to run
# build and test code in.
#
# Common usage:
# scripts/bootstrap-env _venv requirements.txt ...
#
# The requirements files may use the following format for their lines:
# any-normal-requirement
# - Requirement used for both Python 2 and Python 3
# {2:requirement-for-python2}
# - Requirement only to be used in Python 2
# {3:requirement-for-python3}
# - Requirement only to be used in Python 3
#
function normalize() {
echo -n "$( cd "$1" && pwd )"
}
function usage() {
script_name=$(basename "${BASH_SOURCE[0]}")
echo "Usage: $script_name [-v] <environment> <requirements>
Script for configuring and updating a virtualenv based Python environment.
optional arguments:
-v verbose; be more noisy about what is being done
-3 build the environment for Python 3
-h show this help message and exit
positional arguments:
environment path to the directory in which to create the environment
requirements path to the pip formatted (requirements.txt) Python requirements
"
}
function error() {
echo "
Please install pip and virtualenv:
easy_install --user pip
easy_install --user virtualenv
You may also need to add the easy_install directory to your path:
export PATH=~/Library/Python/2.7/bin:\$PATH
" 1>&2
}
python_tool=python2.7
pip_tool=pip
sed_rules='s/\{[^2]:(.*)\}//g; s/\{2:(.*)\}/\1/g;'
# Options.
quiet=-q
while getopts ":h3v" opt ; do
case $opt in
v)
quiet=
;;
3)
python_tool=python3
pip_tool=pip3
sed_rules='s/\{[^3]:(.*)\}//g; s/\{3:(.*)\}/\1/g;'
;;
h)
usage
exit
;;
esac
done
# Check the arguments.
if [[ $# -lt $((OPTIND+1)) ]] ; then
usage
exit 1
fi
environment="${*:$OPTIND:1}"
# Our temporary requirements file
tmprequirements="$TMPDIR/requirements.$$.txt"
function cleanup() {
rm "$tmprequirements"
}
trap cleanup EXIT
# Apply the transformation rules to all the files in the parameters.
sed -E "${sed_rules}" "${@:$OPTIND+1}" | sort -u > "${tmprequirements}"
# Activate the virtualenv.
if [[ ! -e "${environment}/bin/activate" ]]; then
if ! virtualenv -p "${python_tool}" --no-site-packages "${environment}"; then
error
exit 1
fi
fi
environment=$( normalize "${environment}" )
source "${environment}/bin/activate" || exit 1
echo Installing packages with PIP ...
if ! "${pip_tool}" install \
--disable-pip-version-check \
${quiet:-} \
-r "${tmprequirements}" ; then
error
exit 1
fi
echo Checking for any packages that are not in the requirements...
# The echos in the second argument ensure that PIP and Easy_Install are not removed
# from the virtualenv.
to_remove=($(diff <("${pip_tool}" list --disable-pip-version-check | cut -d' ' -f 1 | sort) \
<((echo pip ; echo setuptools; cut -d'=' -f 1 "${tmprequirements}") | sort) \
| grep '^<' \
| cut -d' ' -f2))
if [[ "${#to_remove[@]}" != 0 ]] ; then
echo " - removing ${#to_remove[@]} packages"
if ! "${pip_tool}" uninstall --disable-pip-version-check \
${quiet:-} -y \
"${to_remove[@]}" ; then
echo "Could not remove the unneeded packages. Maybe deleting '${environment}' will help?" >&2
exit 1
fi
else
if [[ -z "${quiet:-}" ]] ; then
echo " - none to remove"
fi
fi
# bash and zsh are sufficiently similar that they are handled by the same
# script. csh and fish, are different enough that they needs their own script.
activate=bin/activate
if [[ "$SHELL" =~ /fish$ ]] ; then
activate=bin/activate.fish
elif [[ "$SHELL" =~ /csh$ ]] ; then
activate=bin/activate.csh
fi
echo "
IMPORTANT: You need to run the following command to activate the virtualenv:
source \"${environment}/${activate}\"
"
#!/usr/bin/env python
"""
Build the Python documentation.
"""
import argparse
import glob
import os
import shutil
import subprocess
import sys
import tarfile
# Configurables
initial_tag = None
files = ['*.py']
usage = "build-python-docs <options>"
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument('--major-version', type=str,
help='Major version number prefix to use')
parser.add_argument('--initial-tag', type=str,
help="Git tag to base commit number against, or 'none' to count all commits", default='1.0')
parser.add_argument('--full-version', type=str,
help='Full version number to use (overrides major-version and initial-tag)')
parser.add_argument('--input-dir', type=str,
help="Directory holding the source to process directory", default='.')
parser.add_argument('--output-dir', type=str,
help="Output directory, relative to the root", default='build-docs')
parser.add_argument('--paths', nargs='*',
help='Python scripts, as a glob to match')
options = parser.parse_args()
if not options.major_version and not options.full_version:
parser.error('Option --major-version or --full-version must be supplied')
if options.initial_tag != 'none':
initial_tag = options.initial_tag
if options.paths:
files = options.paths
if options.input_dir:
options.input_dir = os.path.abspath(options.input_dir)
os.chdir(options.input_dir)
options.output_dir = os.path.abspath(options.output_dir)
# Find the top level, and the repository name
top_git_directory = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).strip()
repository = os.path.basename(top_git_directory)
# Infer version number if not supplied
if options.full_version:
version = options.full_version
else:
list_spec = '%s..HEAD' % (initial_tag,) if initial_tag else 'HEAD'
commit_number = subprocess.check_output(['git', 'rev-list', '--count', list_spec]).strip()
version = "%s.%s" % (options.major_version, commit_number)
name = "%s Python Libraries" % (repository.title(),)
docset = "%s_Python_Libraries_Docset" % (repository.title(),)
print ">>> Building Python documentation for version %s..." % (version,)
docs_root = os.path.join(top_git_directory, options.output_dir)
python_docs_root = os.path.join(docs_root, 'python')
pydoctor_output_root = os.path.join(python_docs_root, name)
dash_output = os.path.join(python_docs_root, '%s.docset' % (name,))
docset_tgz = "%s.tgz" % (os.path.join(docs_root, docset),)
docset_feed = "%s.xml" % (os.path.join(docs_root, docset),)
if os.path.isdir(python_docs_root):
print "Removing python documentation output directory..."
shutil.rmtree(python_docs_root)
os.makedirs(python_docs_root)
# Process through pydoctor
os.chdir(options.input_dir)
command = [
'pydoctor',