initial travis

.travis.yml

@@ -0,0 +1,24 @@
+sudo: required
+
+language: python
+
+python:
+  - "2.7"
+
+# command to install dependencies
+install:
+  - pip install PyYAML
+#  - git clone https://github.com/SciDAP/cwltool.git && cd cwltool && sudo python setup.py install && cd ..
+  - pip install cwltool
+  - pip install cwl-runner
+
+services:
+  - docker
+
+before_install:
+#  - docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+  - docker pull commonworkflowlanguage/nodejs-engine
+
+script:
+  - test/test.sh
+

test/cwltest.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import shutil
+import tempfile
+import yaml
+import pipes
+import logging
+
+_logger = logging.getLogger("cwltool")
+_logger.addHandler(logging.StreamHandler())
+_logger.setLevel(logging.INFO)
+
+UNSUPPORTED_FEATURE = 33
+
+
+class CompareFail(Exception):
+    pass
+
+
+def compare(a, b, i=None):
+    try:
+        if isinstance(a, dict):
+            if a.get("class") == "File":
+                if not b["path"].endswith("/" + a["path"]):
+                    raise CompareFail("%s does not end with %s" % (b["path"], a["path"]))
+                # ignore empty collections
+                b = {k: v for k, v in b.iteritems()
+                     if not isinstance(v, (list, dict)) or len(v) > 0}
+            if len(a) != len(b):
+                raise CompareFail("expected %s\ngot %s" % (
+                json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
+            for c in a:
+                if i and c in i:
+                    continue
+                if a.get("class") != "File" or c != "path":
+                    if c not in b:
+                        raise CompareFail("%s not in %s" % (c, b))
+                    if not compare(a[c], b[c], i):
+                        return False
+            return True
+        elif isinstance(a, list):
+            if len(a) != len(b):
+                raise CompareFail("expected %s\ngot %s" % (
+                json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True)))
+            for c in xrange(0, len(a)):
+                if not compare(a[c], b[c], i):
+                    return False
+            return True
+        else:
+            if a != b:
+                raise CompareFail("%s != %s" % (a, b))
+            else:
+                return True
+    except Exception as e:
+        raise CompareFail(str(e))
+
+
+def run_test(args, i, t):
+    out = {}
+    outdir = None
+    try:
+        if "output" in t and not args.conformance_test:
+            test_command = [args.tool]
+
+            if "outdir" in t:
+                outdir = t["outdir"]
+            else:
+                # Add prefixes if running on MacOSX so that boot2docker writes to /Users
+                if 'darwin' in sys.platform:
+                    outdir = tempfile.mkdtemp(prefix=os.path.abspath(os.path.curdir))
+                else:
+                    outdir = tempfile.mkdtemp()
+            if args.tmp:
+                test_command.extend(["--tmp-outdir-prefix={}".format(outdir), "--tmpdir-prefix={}".format(outdir)])
+
+            if args.push_image:
+                test_command.extend(["--push-image"])
+
+            test_command.extend(["--outdir={}".format(outdir),
+                                 t["tool"],
+                                 t["job"]])
+            outstr = subprocess.check_output(test_command)
+            out = {"output": json.loads(outstr)}
+        else:
+            test_command = [args.tool,
+                            "--conformance-test",
+                            "--basedir=" + args.basedir,
+                            "--no-container",
+                            "--quiet",
+                            t["tool"],
+                            t["job"]]
+            outstr = subprocess.check_output(test_command)
+            out = yaml.load(outstr)
+    except ValueError as v:
+        _logger.error(v)
+        _logger.error(outstr)
+    except subprocess.CalledProcessError as err:
+        if err.returncode == UNSUPPORTED_FEATURE:
+            return UNSUPPORTED_FEATURE
+        else:
+            _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+            _logger.error(t.get("doc"))
+            _logger.error("Returned non-zero")
+            return 1
+    except yaml.scanner.ScannerError as e:
+        _logger.error("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+        _logger.error(outstr)
+        _logger.error("Parse error %s", str(e))
+
+    pwd = os.path.abspath(os.path.dirname(t["job"]))
+    # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"])
+    # if "stdin" in t:
+    #     t["stdin"] = t["stdin"].replace("$PWD", pwd)
+
+    failed = False
+    if "output" in t and not args.conformance_test:
+        checkkeys = ["output"]
+    else:
+        checkkeys = ["args", "stdin", "stdout", "createfiles"]
+    ignore = None
+    if "ignore_keys" in t:
+        ignore = t["ignore_keys"]
+    for key in checkkeys:
+        try:
+            compare(t.get(key), out.get(key),ignore)
+        except CompareFail as ex:
+            _logger.warn("""Test failed: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+            _logger.warn(t.get("doc"))
+            _logger.warn("%s expected %s\n got %s", key,
+                         json.dumps(t.get(key), indent=4, sort_keys=True),
+                         json.dumps(out.get(key), indent=4, sort_keys=True))
+            _logger.warn("Compare failure %s", ex)
+            failed = True
+
+    # if outdir:
+    #     shutil.rmtree(outdir, True)
+
+    if failed:
+        return 1
+    else:
+        return 0
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True)
+    parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".")
+    parser.add_argument("-n", type=int, default=None, help="Run a specific test")
+    parser.add_argument("--tmp", default=False, action="store_true", help="Enable --tmp params for the tool")
+    parser.add_argument("--push-image", action="store_true", default=False,
+                        help="Push a Docker image after build from Dockerfile",
+                        dest="push_image")
+    parser.add_argument("--conformance-test", action="store_true")
+    parser.add_argument("--tool", type=str, default="cwl-runner",
+                        help="CWL runner executable to use (default 'cwl-runner'")
+    args = parser.parse_args()
+
+    if not args.test:
+        parser.print_help()
+        return 1
+
+    with open(args.test) as f:
+        tests = yaml.load(f)
+
+    failures = 0
+    unsupported = 0
+
+    if args.n is not None:
+        sys.stderr.write("\rTest [%i/%i] " % (args.n, len(tests)))
+        rt = run_test(args, args.n - 1, tests[args.n - 1])
+        if rt == 1:
+            failures += 1
+        elif rt == UNSUPPORTED_FEATURE:
+            unsupported += 1
+    else:
+        for i, t in enumerate(tests):
+            sys.stderr.write("\rTest [%i/%i] " % (i + 1, len(tests)))
+            sys.stderr.flush()
+            rt = run_test(args, i, t)
+            if rt == 1:
+                failures += 1
+            elif rt == UNSUPPORTED_FEATURE:
+                unsupported += 1
+
+    if failures == 0 and unsupported == 0:
+        _logger.info("All tests passed")
+        return 0
+    else:
+        _logger.warn("%i failures, %i unsupported features", failures, unsupported)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/test.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -ev
+
+# run a conformance test for all files in the tools/
+o_pwd=$(pwd)
+cd test/
+mkdir -p test-files/dm3
+chmod 777 test-files/dm3
+
+for i in ../tools/*.cwl; do
+ bn=`basename ${i} .cwl`
+
+ if [ "$(cat "${i}"|egrep -e "^class:\s+CommandLineTool$")" = "" ]; then
+    continue;
+ fi
+
+ echo "Testing: ${bn}"
+
+ if [ -f ${bn}-test.yaml ]; then
+     ./cwltest.py --tool "cwltool" --conformance-test --test ${bn}-test.yaml
+ else
+    echo "fail"
+ fi
+
+done
+
+#PUSH_DOCKER=""
+#
+#if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then
+#    PUSH_DOCKER="--push-image"
+#fi
+#
+#
+#echo "STAR real run indexing genome/ reads alignment"
+#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test STAR-test.yaml
+#
+#echo "samtools-index indexing BAM"
+#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test samtools-index-test.yaml
+#
+#echo "bedtools-genomecov genomecov bedGraph"
+#./cwltest.py ${PUSH_DOCKER} --tool "cwltool" --test bedtools-genomecov-test.yaml
+
+
+cd ${o_pwd}