forked from orbit-oss/flask
Improved script for automatic extension testing
This commit is contained in:
Armin Ronacher
parent
a7d83a9f80
commit
3a80ecc660
2 changed files with 216 additions and 33 deletions
5
Makefile
5
Makefile
|
|
@ -1,10 +1,13 @@
|
||||||
.PHONY: clean-pyc test upload-docs docs
|
.PHONY: clean-pyc ext-test test upload-docs docs
|
||||||
|
|
||||||
all: clean-pyc test
|
all: clean-pyc test
|
||||||
|
|
||||||
test:
|
test:
|
||||||
python setup.py test
|
python setup.py test
|
||||||
|
|
||||||
|
ext-test:
|
||||||
|
python tests/flaskext_test.py --browse
|
||||||
|
|
||||||
release:
|
release:
|
||||||
python setup.py release sdist upload
|
python setup.py release sdist upload
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,14 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
import tempfile, subprocess, urllib2, os
|
import os
|
||||||
|
import sys
|
||||||
|
import shutil
|
||||||
|
import urllib2
|
||||||
|
import tempfile
|
||||||
|
import subprocess
|
||||||
|
import argparse
|
||||||
|
from cStringIO import StringIO
|
||||||
|
|
||||||
from flask import json
|
from flask import json
|
||||||
|
|
||||||
|
|
@ -19,24 +26,136 @@ from setuptools.package_index import PackageIndex
|
||||||
from setuptools.archive_util import unpack_archive
|
from setuptools.archive_util import unpack_archive
|
||||||
|
|
||||||
flask_svc_url = 'http://flask.pocoo.org/extensions/'
|
flask_svc_url = 'http://flask.pocoo.org/extensions/'
|
||||||
tdir = tempfile.mkdtemp()
|
|
||||||
|
if sys.platform == 'darwin':
|
||||||
|
_tempdir = '/private/tmp'
|
||||||
|
else:
|
||||||
|
_tempdir = tempfile.gettempdir()
|
||||||
|
tdir = _tempdir + '/flaskext-test'
|
||||||
|
flaskdir = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||||
|
'..'))
|
||||||
|
|
||||||
|
|
||||||
def run_tests(checkout_dir):
|
RESULT_TEMPATE = u'''\
|
||||||
cmd = ['tox']
|
<!doctype html>
|
||||||
return subprocess.call(cmd, cwd=checkout_dir,
|
<title>Flask-Extension Test Results</title>
|
||||||
stdout=open(os.path.join(tdir, 'tox.log'), 'w'),
|
<style type=text/css>
|
||||||
stderr=subprocess.STDOUT)
|
body { font-family: 'Georgia', serif; font-size: 17px; color: #000; }
|
||||||
|
a { color: #004B6B; }
|
||||||
|
a:hover { color: #6D4100; }
|
||||||
|
h1, h2, h3 { font-family: 'Garamond', 'Georgia', serif; font-weight: normal; }
|
||||||
|
h1 { font-size: 30px; margin: 15px 0 5px 0; }
|
||||||
|
h2 { font-size: 24px; margin: 15px 0 5px 0; }
|
||||||
|
h3 { font-size: 19px; margin: 15px 0 5px 0; }
|
||||||
|
textarea, code,
|
||||||
|
pre { font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono',
|
||||||
|
'Bitstream Vera Sans Mono', monospace!important; font-size: 15px;
|
||||||
|
background: #eee; }
|
||||||
|
pre { padding: 7px 15px; line-height: 1.3; }
|
||||||
|
p { line-height: 1.4; }
|
||||||
|
table { border: 1px solid black; border-collapse: collapse;
|
||||||
|
margin: 15px 0; }
|
||||||
|
td, th { border: 1px solid black; padding: 4px 10px;
|
||||||
|
text-align: left; }
|
||||||
|
th { background: #eee; font-weight: normal; }
|
||||||
|
tr.success { background: #D3F5CC; }
|
||||||
|
tr.failed { background: #F5D2CB; }
|
||||||
|
</style>
|
||||||
|
<h1>Flask-Extension Test Results</h1>
|
||||||
|
<p>
|
||||||
|
This page contains the detailed test results for the test run of
|
||||||
|
all {{ 'approved' if approved }} Flask extensions.
|
||||||
|
<h2>Summary</h2>
|
||||||
|
<table class=results>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Extension
|
||||||
|
<th>Version
|
||||||
|
<th>Author
|
||||||
|
<th>License
|
||||||
|
<th>Outcome
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{%- for result in results %}
|
||||||
|
{% set outcome = 'success' if result.success else 'failed' %}
|
||||||
|
<tr class={{ outcome }}>
|
||||||
|
<th>{{ result.name }}
|
||||||
|
<td>{{ result.version }}
|
||||||
|
<td>{{ result.author }}
|
||||||
|
<td>{{ result.license }}
|
||||||
|
<td>{{ outcome }}
|
||||||
|
</tr>
|
||||||
|
{%- endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h2>Test Logs</h2>
|
||||||
|
<p>Detailed test logs for all tests on all platforms:
|
||||||
|
{%- for result in results %}
|
||||||
|
{%- for iptr, log in result.logs|dictsort %}
|
||||||
|
<h3>{{ result.name }} - {{ result.version }} [{{ iptr }}]</h3>
|
||||||
|
<pre>{{ log }}</pre>
|
||||||
|
{%- endfor %}
|
||||||
|
{%- endfor %}
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg, *args):
|
||||||
|
print '[EXTTEST]', msg % args
|
||||||
|
|
||||||
|
|
||||||
|
class TestResult(object):
|
||||||
|
|
||||||
|
def __init__(self, name, folder, statuscode, interpreters):
|
||||||
|
intrptr = os.path.join(folder, '.tox/%s/bin/python'
|
||||||
|
% interpreters[0])
|
||||||
|
self.statuscode = statuscode
|
||||||
|
self.folder = folder
|
||||||
|
self.success = statuscode == 0
|
||||||
|
|
||||||
|
def fetch(field):
|
||||||
|
try:
|
||||||
|
c = subprocess.Popen([intrptr, 'setup.py',
|
||||||
|
'--' + field], cwd=folder,
|
||||||
|
stdout=subprocess.PIPE)
|
||||||
|
return c.communicate()[0].strip()
|
||||||
|
except OSError:
|
||||||
|
return '?'
|
||||||
|
self.name = name
|
||||||
|
self.license = fetch('license')
|
||||||
|
self.author = fetch('author')
|
||||||
|
self.version = fetch('version')
|
||||||
|
|
||||||
|
self.logs = {}
|
||||||
|
for interpreter in interpreters:
|
||||||
|
logfile = os.path.join(folder, '.tox/%s/log/test.log'
|
||||||
|
% interpreter)
|
||||||
|
if os.path.isfile(logfile):
|
||||||
|
self.logs[interpreter] = open(logfile).read()
|
||||||
|
else:
|
||||||
|
self.logs[interpreter] = ''
|
||||||
|
|
||||||
|
|
||||||
|
def create_tdir():
|
||||||
|
try:
|
||||||
|
shutil.rmtree(tdir)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
os.mkdir(tdir)
|
||||||
|
|
||||||
|
|
||||||
|
def package_flask():
|
||||||
|
distfolder = tdir + '/.flask-dist'
|
||||||
|
c = subprocess.Popen(['python', 'setup.py', 'sdist', '--formats=gztar',
|
||||||
|
'--dist', distfolder], cwd=flaskdir)
|
||||||
|
c.wait()
|
||||||
|
return os.path.join(distfolder, os.listdir(distfolder)[0])
|
||||||
|
|
||||||
|
|
||||||
def get_test_command(checkout_dir):
|
def get_test_command(checkout_dir):
|
||||||
files = set(os.listdir(checkout_dir))
|
if os.path.isfile(checkout_dir + '/Makefile'):
|
||||||
if 'Makefile' in files:
|
|
||||||
return 'make test'
|
return 'make test'
|
||||||
elif 'conftest.py' in files:
|
return 'python setup.py test'
|
||||||
return 'py.test'
|
|
||||||
else:
|
|
||||||
return 'nosetests'
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_extensions_list():
|
def fetch_extensions_list():
|
||||||
|
|
@ -47,50 +166,111 @@ def fetch_extensions_list():
|
||||||
yield ext
|
yield ext
|
||||||
|
|
||||||
|
|
||||||
def checkout_extension(ext):
|
def checkout_extension(name):
|
||||||
name = ext['name']
|
log('Downloading extension %s to temporary folder', name)
|
||||||
root = os.path.join(tdir, name)
|
root = os.path.join(tdir, name)
|
||||||
os.mkdir(root)
|
os.mkdir(root)
|
||||||
checkout_path = PackageIndex().download(ext['name'], root)
|
checkout_path = PackageIndex().download(name, root)
|
||||||
|
|
||||||
unpack_archive(checkout_path, root)
|
unpack_archive(checkout_path, root)
|
||||||
path = None
|
path = None
|
||||||
for fn in os.listdir(root):
|
for fn in os.listdir(root):
|
||||||
path = os.path.join(root, fn)
|
path = os.path.join(root, fn)
|
||||||
if os.path.isdir(path):
|
if os.path.isdir(path):
|
||||||
break
|
break
|
||||||
|
log('Downloaded to %s', path)
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
tox_template = """[tox]
|
tox_template = """[tox]
|
||||||
envlist=py26
|
envlist=%(env)s
|
||||||
|
|
||||||
[testenv]
|
[testenv]
|
||||||
commands=
|
deps=%(deps)s
|
||||||
%s
|
commands=bash flaskext-runtest.sh {envlogdir}/test.log
|
||||||
downloadcache=
|
downloadcache=%(cache)s
|
||||||
%s
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def create_tox_ini(checkout_path):
|
|
||||||
|
def create_tox_ini(checkout_path, interpreters, flask_dep):
|
||||||
tox_path = os.path.join(checkout_path, 'tox.ini')
|
tox_path = os.path.join(checkout_path, 'tox.ini')
|
||||||
if not os.path.exists(tox_path):
|
if not os.path.exists(tox_path):
|
||||||
with open(tox_path, 'w') as f:
|
with open(tox_path, 'w') as f:
|
||||||
f.write(tox_template % (get_test_command(checkout_path), tdir))
|
f.write(tox_template % {
|
||||||
|
'env': ','.join(interpreters),
|
||||||
|
'cache': tdir,
|
||||||
|
'deps': flask_dep
|
||||||
|
})
|
||||||
|
|
||||||
# XXX command line
|
|
||||||
only_approved = True
|
|
||||||
|
|
||||||
def test_all_extensions(only_approved=only_approved):
|
def iter_extensions(only_approved=True):
|
||||||
for ext in fetch_extensions_list():
|
for ext in fetch_extensions_list():
|
||||||
if ext['approved'] or not only_approved:
|
if ext['approved'] or not only_approved:
|
||||||
checkout_path = checkout_extension(ext)
|
yield ext['name']
|
||||||
create_tox_ini(checkout_path)
|
|
||||||
ret = run_tests(checkout_path)
|
|
||||||
yield ext['name'], ret
|
def test_extension(name, interpreters, flask_dep):
|
||||||
|
checkout_path = checkout_extension(name)
|
||||||
|
log('Running tests with tox in %s', checkout_path)
|
||||||
|
|
||||||
|
# figure out the test command and write a wrapper script. We
|
||||||
|
# can't write that directly into the tox ini because tox does
|
||||||
|
# not invoke the command from the shell so we have no chance
|
||||||
|
# to pipe the output into a logfile
|
||||||
|
test_command = get_test_command(checkout_path)
|
||||||
|
log('Test command: %s', test_command)
|
||||||
|
f = open(checkout_path + '/flaskext-runtest.sh', 'w')
|
||||||
|
f.write(test_command + ' &> "$1"\n')
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
create_tox_ini(checkout_path, interpreters, flask_dep)
|
||||||
|
rv = subprocess.call(['tox'], cwd=checkout_path)
|
||||||
|
return TestResult(name, checkout_path, rv, interpreters)
|
||||||
|
|
||||||
|
|
||||||
|
def run_tests(interpreters, only_approved=True):
|
||||||
|
results = {}
|
||||||
|
create_tdir()
|
||||||
|
log('Packaging Flask')
|
||||||
|
flask_dep = package_flask()
|
||||||
|
log('Running extension tests')
|
||||||
|
log('Temporary Environment: %s', tdir)
|
||||||
|
for name in iter_extensions(only_approved):
|
||||||
|
log('Testing %s', name)
|
||||||
|
result = test_extension(name, interpreters, flask_dep)
|
||||||
|
if result.success:
|
||||||
|
log('Extension test succeeded')
|
||||||
|
else:
|
||||||
|
log('Extension test failed')
|
||||||
|
results[name] = result
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def render_results(results, approved):
|
||||||
|
from jinja2 import Template
|
||||||
|
items = results.values()
|
||||||
|
items.sort(key=lambda x: x.name.lower())
|
||||||
|
rv = Template(RESULT_TEMPATE, autoescape=True).render(results=items,
|
||||||
|
approved=approved)
|
||||||
|
fd, filename = tempfile.mkstemp(suffix='.html')
|
||||||
|
os.fdopen(fd, 'w').write(rv.encode('utf-8') + '\n')
|
||||||
|
return filename
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
for name, ret in test_all_extensions():
|
parser = argparse.ArgumentParser(description='Runs Flask extension tests')
|
||||||
print name, ret
|
parser.add_argument('--all', dest='all', action='store_true',
|
||||||
|
help='run against all extensions, not just approved')
|
||||||
|
parser.add_argument('--browse', dest='browse', action='store_true',
|
||||||
|
help='show browser with the result summary')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
results = run_tests(['py26'], not args.all)
|
||||||
|
filename = render_results(results, not args.all)
|
||||||
|
if args.browse:
|
||||||
|
import webbrowser
|
||||||
|
webbrowser.open('file:///' + filename.lstrip('/'))
|
||||||
|
print 'Results written to', filename
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue