From 053f5e13c11c1d2ff82903198c9c3d88babde09b Mon Sep 17 00:00:00 2001 From: Jaeho Shin Date: Sat, 7 Jan 2017 22:31:29 -0800 Subject: [PATCH] Removes obsolete "pipeline" --- compiler/README.md | 2 +- .../compile-check-1-compiled_dependencies_correct | 2 -- compiler/compile-code/compile-code-Makefile | 7 ------- compiler/compile-code/compile-code-dataflow_dot | 1 - .../compile-config/compile-config-0.00-init_objects | 2 -- .../compile-config-1.00-qualified_names | 12 ------------ .../compile-config/compile-config-2.01-grounding | 2 +- compiler/deepdive-compile | 2 +- doc/configuration.md | 2 +- 9 files changed, 4 insertions(+), 28 deletions(-) diff --git a/compiler/README.md b/compiler/README.md index 8e9dfddd2..2bd8be678 100644 --- a/compiler/README.md +++ b/compiler/README.md @@ -8,7 +8,7 @@ Here's a brief summary of how the compilation is done. The HOCON syntax used by `deepdive.conf` is interpreted by `hocon2json` and everything is converted into a single JSON config object that holds everything under the key "deepdive". 2. The config object is first extended with some implied extractors, such as initializing the database and loading input tables. - Then, the dependencies of extractors, factors, pipelines are normalized, and their names are qualified with corresponding prefixes (by `compile-config_normalized`) to make it easier and clearer to produce the final code for execution. + Then, the dependencies of extractors are normalized, and their names are qualified with corresponding prefixes (by `compile-config_normalized`) to make it easier and clearer to produce the final code for execution. DeepDive's built-in processes for variables and factors, such as grounding, learning, inference, and calibration, are added to the config object after the normalization. User's original config is kept intact under "deepdive" while the normalized one is created under a different key, "deepdive_". diff --git a/compiler/compile-check/compile-check-1-compiled_dependencies_correct b/compiler/compile-check/compile-check-1-compiled_dependencies_correct index 5fda7f825..cb3ddfb42 100755 --- a/compiler/compile-check/compile-check-1-compiled_dependencies_correct +++ b/compiler/compile-check/compile-check-1-compiled_dependencies_correct @@ -18,8 +18,6 @@ $deepdive.execution.processes | to_entries[] | [ .dependencies_[]? | select( (startswith("process/") and in($deepdive.execution.processes) - # TODO factor/ aren't really used, so remove? - or startswith("factor/") and (ltrimstr("factor/") | in($deepdive.inference.factors_byName)) or startswith("data/") and (ltrimstr("data/") | in($deepdive.schema.relations) # XXX assume user is doing the right thing if schema.json is empty or ($deepdive.schema.relations | length == 0)) diff --git a/compiler/compile-code/compile-code-Makefile b/compiler/compile-code/compile-code-Makefile index aa79d1ec3..630bf9ee5 100755 --- a/compiler/compile-code/compile-code-Makefile +++ b/compiler/compile-code/compile-code-Makefile @@ -15,8 +15,6 @@ def mktargets: mktargets("done"); # default commands define CMD_data endef -define CMD_pipeline -endef define CMD_process endef define CMD_factor @@ -57,11 +55,6 @@ reset: \(keys | mktargets("reset")) $(RESET) \(.key | mktarget("done")) " -# pipelines are special -, if .key | startswith("pipeline/") then " -.PHONY: \(.key | mktarget) -" else empty end - ) ] | join("") } diff --git a/compiler/compile-code/compile-code-dataflow_dot b/compiler/compile-code/compile-code-dataflow_dot index d19be8b15..562baa070 100755 --- a/compiler/compile-code/compile-code-dataflow_dot +++ b/compiler/compile-code/compile-code-dataflow_dot @@ -28,7 +28,6 @@ def nodeType: sub("/.*$"; ""); # edge attributes by [srcType][dstType] ( { "" : { "": "color=\"#999999\"" } - , pipeline : { "": "style=dotted arrowhead=odiamond" } } * ($deepdiveDotConfig.edge_attrs // {}) ) as $edge_attrs | diff --git a/compiler/compile-config/compile-config-0.00-init_objects b/compiler/compile-config/compile-config-0.00-init_objects index 003172ad7..1941dc331 100755 --- a/compiler/compile-config/compile-config-0.00-init_objects +++ b/compiler/compile-config/compile-config-0.00-init_objects @@ -13,8 +13,6 @@ | .deepdive_.extraction.extractors |= . + {} | .deepdive_.inference |= . + {} | .deepdive_.inference.factors |= . + {} -| .deepdive_.pipeline |= . + {} -| .deepdive_.pipeline.pipelines |= . + {} # make sure our intermediate representation for execution plan set up | .deepdive_.execution |= . + {} diff --git a/compiler/compile-config/compile-config-1.00-qualified_names b/compiler/compile-config/compile-config-1.00-qualified_names index ce78ad6d9..144bcd47f 100755 --- a/compiler/compile-config/compile-config-1.00-qualified_names +++ b/compiler/compile-config/compile-config-1.00-qualified_names @@ -3,7 +3,6 @@ # # - extractor names are prefixed with process/* # - factor names are prefixed with factor/* -# - pipeline names are prefixed with pipeline/* # - output_relation names are prefixed with data/* and kept under a new key output_ # - dependencies are rewritten with qualified names under a new key dependencies_ ## @@ -31,16 +30,5 @@ ) ) -# qualify names in pipelines -| .deepdive_.pipeline.pipelines |= with_entries - ( .key as $p - | .key |= "pipeline/\(.)" - | .value |= map( if $deepdive.extraction.extractors[.] then "process/\(.)" - elif $deepdive.inference.factors[.] then "factor/\(.)" - else error("\(.): Neither an extractor or inference rule in pipeline \($p)") - end - ) - ) - # turn all extractors into processes in the execution plan under compilation | .deepdive_.execution.processes += .deepdive_.extraction.extractors diff --git a/compiler/compile-config/compile-config-2.01-grounding b/compiler/compile-config/compile-config-2.01-grounding index 6cf62835e..75452a496 100755 --- a/compiler/compile-config/compile-config-2.01-grounding +++ b/compiler/compile-config/compile-config-2.01-grounding @@ -702,7 +702,7 @@ def factorWeightDescriptionSqlExpr: ## from_grounding # A nominal process to make it easy to redo the grounding -# TODO remove this once deepdive-do supports process groups or pipelines +# TODO remove this once deepdive-do supports process groups | .deepdive_.execution.processes += { "process/grounding/from_grounding": { style: "cmd_extractor", cmd: ": no-op" diff --git a/compiler/deepdive-compile b/compiler/deepdive-compile index aa14868db..2f306dfb6 100755 --- a/compiler/deepdive-compile +++ b/compiler/deepdive-compile @@ -148,7 +148,7 @@ deepdive-check -a -c "$PWD"/config.json 'compiled_*' 2>&1 | sed 's/^/ /' >&2 ############################################################################### STEP "Compiling executable code into:" -# compile extractors and factors under process/ and factor/ +# compile extractors under process/ pids=(--) for cc in "$DEEPDIVE_HOME"/util/compile-code/compile-code-*; do [[ -x "$cc" ]] || continue diff --git a/doc/configuration.md b/doc/configuration.md index 810a9544a..4adf65bb0 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -45,7 +45,7 @@ deepdive { } ``` -In this template, the global section `deepdive` contains following major sections: `db`, `schema`, `extraction`, `inference`, `calibration`. Other optional sections are `sampler` and `pipeline`. +In this template, the global section `deepdive` contains following major sections: `db`, `schema`, `extraction`, `inference`, `calibration`. Other optional sections are `sampler` and `execution`. Links to these sections: