Add 'compiler/ddlog/' from commit '6305811bd620751e5f1c97665de76bb6f1…

…5dbd51'

git-subtree-dir: compiler/ddlog
git-subtree-mainline: ef7399b
git-subtree-split: 6305811

compiler/ddlog/.gitignore

@@ -0,0 +1,7 @@
+/ddlog.jar
+target
+
+/test/*-test/**/*.bats
+/test/*-test/**/*.actual
+
+/.idea

compiler/ddlog/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/bats"]
+	path = test/bats
+	url = https://github.com/sstephenson/bats.git

compiler/ddlog/.travis.yml

@@ -0,0 +1,21 @@
+language: scala
+scala:
+    - "2.11.7"
+jdk:
+    - oraclejdk8
+
+sudo: false  # to use container-based infrastructure
+addons:
+    apt:
+        packages:
+            - wdiff
+cache:
+    directories:
+        - $HOME/.ivy2/cache
+
+install:
+    - make test-build
+script:
+    - make test
+after_success:
+    - sbt coverageReport coveralls

compiler/ddlog/Makefile

@@ -0,0 +1,70 @@
+# Makefile for DDlog compiler
+
+
+### Build & Clean #############################################################
+
+.PHONY: build
+build: scala-build
+
+# build a standalone jar
+JAR = ddlog.jar
+.PHONY: package
+package: $(JAR)
+$(JAR): scala-assembly-jar
+	ln -sfn $(SCALA_ASSEMBLY_JAR) $@
+	touch $@
+
+.PHONY: clean
+clean: scala-clean
+	# clean test artifacts
+	rm -f $(JAR) $(wildcard test/*/*/*.actual)
+	find test/ -name '*.bats' -type l -exec rm -f {} +
+
+PATH := $(shell pwd)/project/sbt:$(PATH)
+export PATH
+ifeq ($(shell uname),Darwin)
+SHELL := /bin/bash  # Mac requires SHELL to be forced to bash
+endif
+
+include scala.mk  # defines scala-build, scala-test-build, scala-assembly-jar, scala-clean targets
+
+
+
+### Test ######################################################################
+
+# test prep for built classes or assembly jar
+test-package: $(JAR)
+	$(MAKE) test TEST_JAR=1
+ifndef TEST_JAR
+test-build: scala-test-build
+test: export CLASSPATH = $(shell cat $(SCALA_TEST_CLASSPATH_EXPORTED))
+
+else # ifdef TEST_JAR
+.PHONY: test-package
+test-build: $(JAR)
+test: export CLASSPATH = $(realpath $(JAR))
+
+endif # TEST_JAR
+
+actual-expected:
+	for expected in test/*/*/*.expected; do \
+	    case $$expected in \
+	        *-error.expected) actual=$${expected%-error.expected}.actual;; \
+	        *)                actual=$${expected%.expected}.actual      ;; \
+	    esac; \
+	    [[ "$$actual" -nt "$$expected" ]] && \
+	    ! diff -q --ignore-all-space "$$actual" "$$expected" || continue; \
+	    cp -vf "$$actual" "$$expected"; \
+	done
+
+# test coverage report
+.PHONY: test-coverage coveralls
+test-coverage:
+	-$(MAKE) test
+	sbt coverageReport
+coveralls: test-coverage
+	# submit coverage data to https://coveralls.io/r/HazyResearch/ddlog
+	# (Make sure you have set COVERALLS_REPO_TOKEN=...)
+	sbt coveralls
+
+include test/bats.mk  # defines test, test-build, test-list targets

compiler/ddlog/README.md

@@ -0,0 +1,68 @@
+DeepDiveLog [![Build Status](https://travis-ci.org/HazyResearch/ddlog.svg)](https://travis-ci.org/HazyResearch/ddlog) [![Coverage Status](https://coveralls.io/repos/HazyResearch/ddlog/badge.svg)](https://coveralls.io/r/HazyResearch/ddlog)
+===================
+
+A Datalog-like language for writing DeepDive apps.
+
+## Building
+The following command builds all Scala code under src/.
+
+```bash
+make
+```
+
+
+## Testing
+
+### Interactive Tests
+To play with DDlog compiler's command-line interface interactively, use the following script after running `make`.
+
+```bash
+test/ddlog
+```
+
+This provides a quick way to see the effects from the command-line after changing the source code as it directly runs the compiled classes under target/ rather than having to assemble a jar every time.
+After running `make` at least once, you can solely rely on an IDE such as [IntelliJ IDEA][] to compile individual .scala files and test it with this script.
+
+### Automated Tests
+The following command runs all tests under test/.
+Tests are written in [BATS][].
+
+```bash
+make test
+```
+
+To run tests selectively, see the long make command produced by:
+
+```bash
+make test-list
+```
+
+### Test Coverage
+The following command produces a test coverage report.
+
+```bash
+make test-coverage
+```
+
+
+## Deploying
+
+### Assembling a standalone jar
+The following command produces a standalone jar that contains the DDlog compiler.
+
+```bash
+make ddlog.jar
+```
+
+### Running the standalone jar
+The following command generates a deepdive.conf for the [spouse example in DeepDive's tutorial](http://deepdive.stanford.edu/doc/basics/walkthrough/walkthrough.html).
+
+```bash
+mkdir -p examples/spouse_example
+java -jar ddlog.jar compile examples/spouse_example.ddl >examples/spouse_example/deepdive.conf
+```
+
+
+
+[BATS]: https://github.com/sstephenson/bats#readme "Bash Automated Testing System"
+[IntelliJ IDEA]: https://www.jetbrains.com/idea/

compiler/ddlog/build.sbt

@@ -0,0 +1,11 @@
+scalaVersion := "2.11.7"
+
+scalacOptions ++= Seq("-feature")
+
+libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4"
+
+libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.4"
+
+libraryDependencies += "com.github.scopt" %% "scopt" % "3.3.0"
+
+resolvers += Resolver.sonatypeRepo("public")

compiler/ddlog/examples/chunking.ddlog

@@ -0,0 +1,84 @@
+chunk?(
+    @key sent_id bigint,
+    @key word_id bigint,
+    tag text
+).
+tags(tag text).
+
+# input data ##################################################################
+words_raw(
+  sent_id bigint,
+  word_id bigint,
+  word text,
+  pos text,
+  true_tag text
+).
+
+words(
+  sent_id bigint,
+  word_id bigint,
+  word text,
+  pos text,
+  true_tag text,
+  tag text
+).
+
+# supervision / scope of tags #################################################
+chunk(s, word, tag) = (
+  if tag = tag_supervised then TRUE
+                          else NULL
+  end
+) :-
+  words(s, word, _, _, _, tag_supervised),
+  tags(tag).
+
+# TODO the following if-then-else can go directly to the supervision rule once DDlog supports true unification (tag_supervised = if ... end in the body)
+words(sent_id, word_id, word, pos, true_tag,
+             if true_tag = "B-UCP"        then NULL
+        else if true_tag = "I-UCP"        then NULL
+        else if strpos(true_tag, "-") > 0 then split_part(true_tag, "-", 2)
+        else if true_tag = "O"            then "O"
+        else                                   NULL
+        end) :-
+  words_raw(sent_id, word_id, word, pos, true_tag).
+
+# features ####################################################################
+word_features(
+  sent_id bigint,
+  word_id bigint,
+  feature text
+).
+
+function ext_features
+  over (sent_id bigint, word_id1 bigint, word1 text, pos1 text, word2 text, pos2 text)
+  returns rows like word_features
+  implementation "udf/ext_features.py" handles tsv lines.
+
+word_features +=
+  ext_features(s, w1, word1, pos1, word2, pos2) :-
+  words(s, w1, word1, pos1, _, _),
+  words(s, w2, word2, pos2, _, _),
+  [w1 = w2 + 1],
+  word1 IS NOT NULL.
+
+@weight(f, tag)
+@feature_value(f)
+chunk(s, w, tag) :-
+                      words(s, w, _, _, _, _), tags(tag), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  word_features(s, w, f).
+
+# linear chains ###############################################################
+@name("LINEAR  @   chain")
+@weight(tag1, tag2)
+chunk(s, w1, tag1) ^ chunk(s, w2, tag2) :-
+                      words(s, w1, _, _, _, _), words(s, w2, _, _, _, _), tags(tag1), tags(tag2), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  w2 = w1 + 1.
+
+# skip chain ##################################################################
+@name("phony rule")
+@weight(tag1, tag2)
+chunk(s, w1, tag1) ^ chunk(s, w2, tag2) :-
+  words(s, w1, _, _, _, tag),
+                      words(s, w2, _, _, _, _), tags(tag1), tags(tag2), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  tag IS NOT NULL,
+  w1 < w2.

compiler/ddlog/examples/ocr.ddlog

@@ -0,0 +1,27 @@
+// ocr example from deepdive
+// https://github.com/HazyResearch/deepdive/tree/master/examples/ocr
+
+features(
+  id          BIGSERIAL, 
+  word_id     INT, 
+  feature_id  INT, 
+  feature_val BOOLEAN).
+
+label1(
+  wid   INT, 
+  val   BOOLEAN).
+
+label2(
+  wid   INT, 
+  val   BOOLEAN).
+
+q1?(wid INT).
+q2?(wid INT).
+
+q1(wid) = val :- label1(wid, val).
+q2(wid) = val :- label2(wid, val).
+
+@weight(fid)
+q1(wid) :- features(id, wid, fid, fval).
+@weight(fid)
+q2(wid) :- features(id, wid, fid, fval).

compiler/ddlog/examples/smoke.ddlog

@@ -0,0 +1,40 @@
+// smoke example from deepdive
+// https://github.com/HazyResearch/deepdive/tree/master/examples/smoke
+
+person (
+    person_id bigint,
+    name text
+).
+
+person_has_cancer (
+    person_id bigint,
+    has_cancer boolean
+).
+
+person_smokes (
+    person_id bigint,
+    smokes boolean
+).
+
+friends (
+    person_id bigint,
+    friend_id bigint
+).
+
+smoke? (
+    person_id bigint
+).
+
+cancer? (
+    person_id bigint
+).
+
+smoke(pid) = l :- person_smokes(pid, l).
+
+cancer(pid) = l :- person_has_cancer(pid, l).
+
+@weight(0.5)
+smoke(pid) => cancer(pid) :- person_smokes(pid, l).
+
+@weight(0.4)
+smoke(pid1) => smoke(pid) :- friends(pid1, pid).