From f4fd76de885a2eef6593bc7d7fe9c37988260c84 Mon Sep 17 00:00:00 2001 From: Andrey Kartashov Date: Sat, 12 Dec 2015 20:59:56 -0500 Subject: [PATCH] initial travis --- .travis.yml | 24 ++++++ test/cwltest.py | 199 ++++++++++++++++++++++++++++++++++++++++++++++++ test/test.sh | 44 +++++++++++ 3 files changed, 267 insertions(+) create mode 100644 .travis.yml create mode 100755 test/cwltest.py create mode 100755 test/test.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..c83172036 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +sudo: required + +language: python + +python: + - "2.7" + +# command to install dependencies +install: + - pip install PyYAML +# - git clone https://github.com/SciDAP/cwltool.git && cd cwltool && sudo python setup.py install && cd .. + - pip install cwltool + - pip install cwl-runner + +services: + - docker + +before_install: +# - docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + - docker pull commonworkflowlanguage/nodejs-engine + +script: + - test/test.sh + diff --git a/test/cwltest.py b/test/cwltest.py new file mode 100755 index 000000000..eca0c3173 --- /dev/null +++ b/test/cwltest.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +import argparse +import json +import os +import subprocess +import sys +import shutil +import tempfile +import yaml +import pipes +import logging + +_logger = logging.getLogger("cwltool") +_logger.addHandler(logging.StreamHandler()) +_logger.setLevel(logging.INFO) + +UNSUPPORTED_FEATURE = 33 + + +class CompareFail(Exception): + pass + + +def compare(a, b, i=None): + try: + if isinstance(a, dict): + if a.get("class") == "File": + if not b["path"].endswith("/" + a["path"]): + raise CompareFail("%s does not end with %s" % (b["path"], a["path"])) + # ignore empty collections + b = {k: v for k, v in b.iteritems() + if not isinstance(v, (list, dict)) or len(v) > 0} + if len(a) != len(b): + raise CompareFail("expected %s\ngot %s" % ( + json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) + for c in a: + if i and c in i: + continue + if a.get("class") != "File" or c != "path": + if c not in b: + raise CompareFail("%s not in %s" % (c, b)) + if not compare(a[c], b[c], i): + return False + return True + elif isinstance(a, list): + if len(a) != len(b): + raise CompareFail("expected %s\ngot %s" % ( + json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) + for c in xrange(0, len(a)): + if not compare(a[c], b[c], i): + return False + return True + else: + if a != b: + raise CompareFail("%s != %s" % (a, b)) + else: + return True + except Exception as e: + raise CompareFail(str(e)) + + +def run_test(args, i, t): + out = {} + outdir = None + try: + if "output" in t and not args.conformance_test: + test_command = [args.tool] + + if "outdir" in t: + outdir = t["outdir"] + else: + # Add prefixes if running on MacOSX so that boot2docker writes to /Users + if 'darwin' in sys.platform: + outdir = tempfile.mkdtemp(prefix=os.path.abspath(os.path.curdir)) + else: + outdir = tempfile.mkdtemp() + if args.tmp: + test_command.extend(["--tmp-outdir-prefix={}".format(outdir), "--tmpdir-prefix={}".format(outdir)]) + + if args.push_image: + test_command.extend(["--push-image"]) + + test_command.extend(["--outdir={}".format(outdir), + t["tool"], + t["job"]]) + outstr = subprocess.check_output(test_command) + out = {"output": json.loads(outstr)} + else: + test_command = [args.tool, + "--conformance-test", + "--basedir=" + args.basedir, + "--no-container", + "--quiet", + t["tool"], + t["job"]] + outstr = subprocess.check_output(test_command) + out = yaml.load(outstr) + except ValueError as v: + _logger.error(v) + _logger.error(outstr) + except subprocess.CalledProcessError as err: + if err.returncode == UNSUPPORTED_FEATURE: + return UNSUPPORTED_FEATURE + else: + _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command])) + _logger.error(t.get("doc")) + _logger.error("Returned non-zero") + return 1 + except yaml.scanner.ScannerError as e: + _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command])) + _logger.error(outstr) + _logger.error("Parse error %s", str(e)) + + pwd = os.path.abspath(os.path.dirname(t["job"])) + # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"]) + # if "stdin" in t: + # t["stdin"] = t["stdin"].replace("$PWD", pwd) + + failed = False + if "output" in t and not args.conformance_test: + checkkeys = ["output"] + else: + checkkeys = ["args", "stdin", "stdout", "createfiles"] + ignore = None + if "ignore_keys" in t: + ignore = t["ignore_keys"] + for key in checkkeys: + try: + compare(t.get(key), out.get(key),ignore) + except CompareFail as ex: + _logger.warn("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command])) + _logger.warn(t.get("doc")) + _logger.warn("%s expected %s\n got %s", key, + json.dumps(t.get(key), indent=4, sort_keys=True), + json.dumps(out.get(key), indent=4, sort_keys=True)) + _logger.warn("Compare failure %s", ex) + failed = True + + # if outdir: + # shutil.rmtree(outdir, True) + + if failed: + return 1 + else: + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True) + parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".") + parser.add_argument("-n", type=int, default=None, help="Run a specific test") + parser.add_argument("--tmp", default=False, action="store_true", help="Enable --tmp params for the tool") + parser.add_argument("--push-image", action="store_true", default=False, + help="Push a Docker image after build from Dockerfile", + dest="push_image") + parser.add_argument("--conformance-test", action="store_true") + parser.add_argument("--tool", type=str, default="cwl-runner", + help="CWL runner executable to use (default 'cwl-runner'") + args = parser.parse_args() + + if not args.test: + parser.print_help() + return 1 + + with open(args.test) as f: + tests = yaml.load(f) + + failures = 0 + unsupported = 0 + + if args.n is not None: + sys.stderr.write("\rTest [%i/%i] " % (args.n, len(tests))) + rt = run_test(args, args.n - 1, tests[args.n - 1]) + if rt == 1: + failures += 1 + elif rt == UNSUPPORTED_FEATURE: + unsupported += 1 + else: + for i, t in enumerate(tests): + sys.stderr.write("\rTest [%i/%i] " % (i + 1, len(tests))) + sys.stderr.flush() + rt = run_test(args, i, t) + if rt == 1: + failures += 1 + elif rt == UNSUPPORTED_FEATURE: + unsupported += 1 + + if failures == 0 and unsupported == 0: + _logger.info("All tests passed") + return 0 + else: + _logger.warn("%i failures, %i unsupported features", failures, unsupported) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 000000000..5243971c8 --- /dev/null +++ b/test/test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -ev + +# run a conformance test for all files in the tools/ +o_pwd=$(pwd) +cd test/ +mkdir -p test-files/dm3 +chmod 777 test-files/dm3 + +for i in ../tools/*.cwl; do + bn=`basename ${i} .cwl` + + if [ "$(cat "${i}"|egrep -e "^class:\s+CommandLineTool$")" = "" ]; then + continue; + fi + + echo "Testing: ${bn}" + + if [ -f ${bn}-test.yaml ]; then + ./cwltest.py --tool "cwltool" --conformance-test --test ${bn}-test.yaml + else + echo "fail" + fi + +done + +#PUSH_DOCKER="" +# +#if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then +# PUSH_DOCKER="--push-image" +#fi +# +# +#echo "STAR real run indexing genome/ reads alignment" +#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test STAR-test.yaml +# +#echo "samtools-index indexing BAM" +#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test samtools-index-test.yaml +# +#echo "bedtools-genomecov genomecov bedGraph" +#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test bedtools-genomecov-test.yaml + + +cd ${o_pwd}