Merge pull request #626 from HazyResearch/fold-submodules

DeepDive swallows DimmWitted and DDlog submodules

.gitmodules

@@ -1,15 +1,15 @@
-[submodule "inference/dimmwitted"]
-	path = inference/dimmwitted
-	url = https://github.com/HazyResearch/sampler.git
 [submodule "util/mindbender"]
 	path = util/mindbender
 	url = https://github.com/netj/mindbender.git
-[submodule "compiler/ddlog"]
-	path = compiler/ddlog
-	url = https://github.com/HazyResearch/ddlog.git
 [submodule "test/bats"]
 	path = test/bats
 	url = https://github.com/sstephenson/bats.git
+[submodule "inference/dimmwitted/test/bats"]
+        path = inference/dimmwitted/test/bats
+	url = https://github.com/sstephenson/bats.git
+[submodule "compiler/ddlog/test/bats"]
+	path = compiler/ddlog/test/bats
+	url = https://github.com/sstephenson/bats.git
 [submodule "extern/buildkit"]
 	path = extern/buildkit
 	url = https://github.com/netj/buildkit.git

Makefile

@@ -137,6 +137,16 @@ build:
 	# record version and build info
 	util/build/generate-build-info.sh >$(BUILD_INFO)
 
+# build DimmWitted
+.PHONY build: inference/dimmwitted/dw
+inference/dimmwitted/dw:
+	$(@D).build.sh
+
+# build DDlog compiler
+.PHONY build: compiler/ddlog/ddlog.jar
+compiler/ddlog/ddlog.jar:
+	$(@D).build.sh
+
 # ensure some submodules required by stage.sh is there
 build: \
     extern/buildkit/install-shared-libraries-required-by \
@@ -184,21 +194,19 @@ checkstyle:
 
 # submodules to build and the files to copy out from each of them
 include util/build/build-submodules.mk
-$(call BUILD_SUBMODULE_AND_COPY, inference/dimmwitted, dw                                                   )
-$(call BUILD_SUBMODULE_AND_COPY, compiler/ddlog      , target/scala-2.11/ddlog-assembly-0.1-SNAPSHOT.jar    )
 $(call BUILD_SUBMODULE_AND_COPY, runner/mkmimo       , mkmimo                                               )
 ifndef NO_MINDBENDER
 $(call BUILD_SUBMODULE_AND_COPY, util/mindbender     , @prefix@/                                            )
 endif
 
 # XXX legacy targets kept to reduce surprise
 .PHONY: build-sampler
-build-sampler: build-submodule-dimmwitted
+build-sampler: inference/dimmwitted/dw
 .PHONY: build-dimmwitted
-build-dimmwitted: build-submodule-dimmwitted
+build-dimmwitted: inference/dimmwitted/dw
 .PHONY: build-mindbender
 build-mindbender: build-submodule-mindbender
 .PHONY: build-ddlog
-build-ddlog: build-submodule-ddlog
+build-ddlog: compiler/ddlog/ddlog.jar
 .PHONY: build-mkmimo
 build-mkmimo: build-submodule-mkmimo

compiler/ddlog/.gitignore

@@ -0,0 +1,7 @@
+/ddlog.jar
+target
+
+/test/*-test/**/*.bats
+/test/*-test/**/*.actual
+
+/.idea

compiler/ddlog/.travis.yml

@@ -0,0 +1,21 @@
+language: scala
+scala:
+    - "2.11.7"
+jdk:
+    - oraclejdk8
+
+sudo: false  # to use container-based infrastructure
+addons:
+    apt:
+        packages:
+            - wdiff
+cache:
+    directories:
+        - $HOME/.ivy2/cache
+
+install:
+    - make test-build
+script:
+    - make test
+after_success:
+    - sbt coverageReport coveralls

compiler/ddlog/Makefile

@@ -0,0 +1,70 @@
+# Makefile for DDlog compiler
+
+
+### Build & Clean #############################################################
+
+.PHONY: build
+build: scala-build
+
+# build a standalone jar
+JAR = ddlog.jar
+.PHONY: package
+package: $(JAR)
+$(JAR): scala-assembly-jar
+	ln -sfn $(SCALA_ASSEMBLY_JAR) $@
+	touch $@
+
+.PHONY: clean
+clean: scala-clean
+	# clean test artifacts
+	rm -f $(JAR) $(wildcard test/*/*/*.actual)
+	find test/ -name '*.bats' -type l -exec rm -f {} +
+
+PATH := $(shell pwd)/project/sbt:$(PATH)
+export PATH
+ifeq ($(shell uname),Darwin)
+SHELL := /bin/bash  # Mac requires SHELL to be forced to bash
+endif
+
+include scala.mk  # defines scala-build, scala-test-build, scala-assembly-jar, scala-clean targets
+
+
+
+### Test ######################################################################
+
+# test prep for built classes or assembly jar
+test-package: $(JAR)
+	$(MAKE) test TEST_JAR=1
+ifndef TEST_JAR
+test-build: scala-test-build
+test: export CLASSPATH = $(shell cat $(SCALA_TEST_CLASSPATH_EXPORTED))
+
+else # ifdef TEST_JAR
+.PHONY: test-package
+test-build: $(JAR)
+test: export CLASSPATH = $(realpath $(JAR))
+
+endif # TEST_JAR
+
+actual-expected:
+	for expected in test/*/*/*.expected; do \
+	    case $$expected in \
+	        *-error.expected) actual=$${expected%-error.expected}.actual;; \
+	        *)                actual=$${expected%.expected}.actual      ;; \
+	    esac; \
+	    [[ "$$actual" -nt "$$expected" ]] && \
+	    ! diff -q --ignore-all-space "$$actual" "$$expected" || continue; \
+	    cp -vf "$$actual" "$$expected"; \
+	done
+
+# test coverage report
+.PHONY: test-coverage coveralls
+test-coverage:
+	-$(MAKE) test
+	sbt coverageReport
+coveralls: test-coverage
+	# submit coverage data to https://coveralls.io/r/HazyResearch/ddlog
+	# (Make sure you have set COVERALLS_REPO_TOKEN=...)
+	sbt coveralls
+
+include test/bats.mk  # defines test, test-build, test-list targets

compiler/ddlog/README.md

@@ -0,0 +1,68 @@
+DeepDiveLog [![Build Status](https://travis-ci.org/HazyResearch/ddlog.svg)](https://travis-ci.org/HazyResearch/ddlog) [![Coverage Status](https://coveralls.io/repos/HazyResearch/ddlog/badge.svg)](https://coveralls.io/r/HazyResearch/ddlog)
+===================
+
+A Datalog-like language for writing DeepDive apps.
+
+## Building
+The following command builds all Scala code under src/.
+
+```bash
+make
+```
+
+
+## Testing
+
+### Interactive Tests
+To play with DDlog compiler's command-line interface interactively, use the following script after running `make`.
+
+```bash
+test/ddlog
+```
+
+This provides a quick way to see the effects from the command-line after changing the source code as it directly runs the compiled classes under target/ rather than having to assemble a jar every time.
+After running `make` at least once, you can solely rely on an IDE such as [IntelliJ IDEA][] to compile individual .scala files and test it with this script.
+
+### Automated Tests
+The following command runs all tests under test/.
+Tests are written in [BATS][].
+
+```bash
+make test
+```
+
+To run tests selectively, see the long make command produced by:
+
+```bash
+make test-list
+```
+
+### Test Coverage
+The following command produces a test coverage report.
+
+```bash
+make test-coverage
+```
+
+
+## Deploying
+
+### Assembling a standalone jar
+The following command produces a standalone jar that contains the DDlog compiler.
+
+```bash
+make ddlog.jar
+```
+
+### Running the standalone jar
+The following command generates a deepdive.conf for the [spouse example in DeepDive's tutorial](http://deepdive.stanford.edu/doc/basics/walkthrough/walkthrough.html).
+
+```bash
+mkdir -p examples/spouse_example
+java -jar ddlog.jar compile examples/spouse_example.ddl >examples/spouse_example/deepdive.conf
+```
+
+
+
+[BATS]: https://github.com/sstephenson/bats#readme "Bash Automated Testing System"
+[IntelliJ IDEA]: https://www.jetbrains.com/idea/

compiler/ddlog/build.sbt

@@ -0,0 +1,11 @@
+scalaVersion := "2.11.7"
+
+scalacOptions ++= Seq("-feature")
+
+libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4"
+
+libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.4"
+
+libraryDependencies += "com.github.scopt" %% "scopt" % "3.3.0"
+
+resolvers += Resolver.sonatypeRepo("public")

compiler/ddlog/examples/chunking.ddlog

@@ -0,0 +1,84 @@
+chunk?(
+    @key sent_id bigint,
+    @key word_id bigint,
+    tag text
+).
+tags(tag text).
+
+# input data ##################################################################
+words_raw(
+  sent_id bigint,
+  word_id bigint,
+  word text,
+  pos text,
+  true_tag text
+).
+
+words(
+  sent_id bigint,
+  word_id bigint,
+  word text,
+  pos text,
+  true_tag text,
+  tag text
+).
+
+# supervision / scope of tags #################################################
+chunk(s, word, tag) = (
+  if tag = tag_supervised then TRUE
+                          else NULL
+  end
+) :-
+  words(s, word, _, _, _, tag_supervised),
+  tags(tag).
+
+# TODO the following if-then-else can go directly to the supervision rule once DDlog supports true unification (tag_supervised = if ... end in the body)
+words(sent_id, word_id, word, pos, true_tag,
+             if true_tag = "B-UCP"        then NULL
+        else if true_tag = "I-UCP"        then NULL
+        else if strpos(true_tag, "-") > 0 then split_part(true_tag, "-", 2)
+        else if true_tag = "O"            then "O"
+        else                                   NULL
+        end) :-
+  words_raw(sent_id, word_id, word, pos, true_tag).
+
+# features ####################################################################
+word_features(
+  sent_id bigint,
+  word_id bigint,
+  feature text
+).
+
+function ext_features
+  over (sent_id bigint, word_id1 bigint, word1 text, pos1 text, word2 text, pos2 text)
+  returns rows like word_features
+  implementation "udf/ext_features.py" handles tsv lines.
+
+word_features +=
+  ext_features(s, w1, word1, pos1, word2, pos2) :-
+  words(s, w1, word1, pos1, _, _),
+  words(s, w2, word2, pos2, _, _),
+  [w1 = w2 + 1],
+  word1 IS NOT NULL.
+
+@weight(f, tag)
+@feature_value(f)
+chunk(s, w, tag) :-
+                      words(s, w, _, _, _, _), tags(tag), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  word_features(s, w, f).
+
+# linear chains ###############################################################
+@name("LINEAR  @   chain")
+@weight(tag1, tag2)
+chunk(s, w1, tag1) ^ chunk(s, w2, tag2) :-
+                      words(s, w1, _, _, _, _), words(s, w2, _, _, _, _), tags(tag1), tags(tag2), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  w2 = w1 + 1.
+
+# skip chain ##################################################################
+@name("phony rule")
+@weight(tag1, tag2)
+chunk(s, w1, tag1) ^ chunk(s, w2, tag2) :-
+  words(s, w1, _, _, _, tag),
+                      words(s, w2, _, _, _, _), tags(tag1), tags(tag2), # XXX just to make it compile. remove after fixing https://github.com/HazyResearch/ddlog/issues/72
+  tag IS NOT NULL,
+  w1 < w2.

compiler/ddlog/examples/ocr.ddlog

@@ -0,0 +1,27 @@
+// ocr example from deepdive
+// https://github.com/HazyResearch/deepdive/tree/master/examples/ocr
+
+features(
+  id          BIGSERIAL, 
+  word_id     INT, 
+  feature_id  INT, 
+  feature_val BOOLEAN).
+
+label1(
+  wid   INT, 
+  val   BOOLEAN).
+
+label2(
+  wid   INT, 
+  val   BOOLEAN).
+
+q1?(wid INT).
+q2?(wid INT).
+
+q1(wid) = val :- label1(wid, val).
+q2(wid) = val :- label2(wid, val).
+
+@weight(fid)
+q1(wid) :- features(id, wid, fid, fval).
+@weight(fid)
+q2(wid) :- features(id, wid, fid, fval).