add more targets to Makefile

.gitignore

@@ -23,6 +23,12 @@ _testmain.go
 
 backup-conf.json
 
+backup-conf.yaml
+conf.yaml
+
+backup-conf.yml
+conf.yml
+
 *.exe
 *.gob
 *.bolt

Makefile

@@ -1,19 +1,36 @@
-build: build-indexer build-crawler build-spider build-tocrawl build-mergedocs build-server 
+.PHONY: all
+
+GIT_BRANCH := $(subst heads/,,$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null))
+
+BIND_DIR := "dist"
+SRCS = $(shell git ls-files '*.go' | grep -v '^vendor/')
 
-build-indexer:
-	go build -o $(PWD)/bin/indexer $(PWD)/cmd/indexer/*.go
+TARGETS := "indexer crawler spider tocrawl mergedocs server"
 
-build-crawler:
-	go build -o $(PWD)/bin/crawler $(PWD)/cmd/crawler/*.go
+print-%: ; @echo $*=$($*)
+
+build: build-indexer build-crawler build-spider build-tocrawl build-mergedocs build-server 
 
-build-tocrawl:
-	go build -o $(PWD)/bin/crawler $(PWD)/cmd/tocrawl/*.go
+build-%:
+	go build -o $(PWD)/bin/$* $(PWD)/cmd/$*/*.go
 
-build-mergedocs:
-	go build -o $(PWD)/bin/server $(PWD)/cmd/mergedocs/*.go
+run-%:
+	# @echo $*=$($*)
+	# @echo "$@"
+	cd $(PWD)/cmd/$* && go run -race *.go
 
-build-spider:
-	go build -o $(PWD)/bin/server $(PWD)/cmd/spider/*.go
+info:
+	@echo "Important: Create a basic \`conf.json\` file, limiting the crawler to a one minute run: \`{ \"crawler\": { \"due_per_run\": \"1m\" } }\`"
+	@echo ""
+	@echo "Available targets for build/run"
+	@echo " 1. tocrawl: Run the package finder: \`go run tocrawl/*.go\`"
+	@echo " 2. crawler: Run the crawler: \`go run crawler/*.go\`"
+	@echo " 3. mergedocs: Merge the crawled docs: \`go run mergedocs/*.go\`"
+	@echo " 4. indexer: Run the indexer: \`go run indexer/*.go\`"
+	@echo " 5. server: Run the server: \`go run server/*.go\`"
+	@echo ""
+	@echo "Visit [http://localhost:8080](http://localhost:8080) in your browser"
+	@echo ""
 
-build-server:
-	go build -o $(PWD)/bin/server $(PWD)/cmd/server/*.go
+help: ## this help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

README.md

@@ -21,11 +21,11 @@ Development
 You'll need to perform the following steps to get a basic server running:
 
   1. Create a basic `conf.json` file, limiting the crawler to a one minute run: `{ "crawler": { "due_per_run": "1m" } }`
-  1. Run the package finder: `go run tocrawl/*.go`
-  1. Run the crawler: `go run crawler/*.go`
-  1. Merge the crawled docs: `go run mergedocs/*.go`
-  1. Run the indexer: `go run indexer/*.go`
-  1. Run the server: ` go run server/*.go`
+  1. Run the package finder: `go run cmd/tocrawl/*.go`
+  1. Run the crawler: `go run cmd/crawler/*.go`
+  1. Merge the crawled docs: `go run cmd/mergedocs/*.go`
+  1. Run the indexer: `go run cmd/indexer/*.go`
+  1. Run the server: ` go run cmd/server/*.go`
   1. Visit [http://localhost:8080](http://localhost:8080) in your browser
 
 

cmd/crawler/main.go

@@ -97,7 +97,7 @@ func (crawlerMapper) MapEnd(c []sophie.Collector) error {
 }
 
 func cleanTempDir() {
-	tmpFn := villa.Path("./shared/data/gddo")
+	tmpFn := villa.Path("../shared/data/gddo")
 	if err := tmpFn.RemoveAll(); err != nil {
 		log.Printf("Delete %v failed: %v", tmpFn, err)
 	}

cmd/server/shared/data/imports/0/links.json

@@ -0,0 +1 @@
+["bitbucket.org/phlyingpenguin/collectinator","bitbucket.org/phlyingpenguin/collectinator/account","bitbucket.org/phlyingpenguin/collectinator/blog","bitbucket.org/phlyingpenguin/collectinator/common","code.google.com/p/sadbox/sessions/mongodb","git.300brand.com/coverage","git.300brand.com/coverage/storage/mongo","github.com/GeertJohan/outyet","github.com/MG-RAST/AWE/core","github.com/MG-RAST/Shock/shock-server","github.com/MG-RAST/Shock/shock-server/controller/node","github.com/MG-RAST/Shock/shock-server/node","github.com/MG-RAST/Shock/shock-server/preauth","github.com/MG-RAST/Shock/shock-server/user","github.com/abiosoft/gopages-sample/store","github.com/adnaan/hamster","github.com/alouca/MongoQueue","github.com/araddon/loges","github.com/araddon/mgou","github.com/arbaal/go-gridfs-serve","github.com/athom/tenpu","github.com/bketelsen/skynet/client","github.com/bketelsen/skynet/rpc/bsonrpc","github.com/bketelsen/skynet/service","github.com/cgrates/cgrates/rater","github.com/chrissexton/alepale/bot","github.com/chrissexton/alepale/plugins","github.com/drevell/mgou","github.com/edsrzf/mgo","github.com/emicklei/landskape/dao","github.com/fluffle/sp0rkle/collections/conf","github.com/fluffle/sp0rkle/collections/factoids","github.com/fluffle/sp0rkle/collections/karma","github.com/fluffle/sp0rkle/collections/markov","github.com/fluffle/sp0rkle/collections/quotes","github.com/fluffle/sp0rkle/collections/reminders","github.com/fluffle/sp0rkle/collections/seen","github.com/fluffle/sp0rkle/collections/stats","github.com/fluffle/sp0rkle/collections/urls","github.com/fluffle/sp0rkle/drivers/factdriver","github.com/fluffle/sp0rkle/drivers/quotedriver","github.com/fluffle/sp0rkle/drivers/reminddriver","github.com/fluffle/sp0rkle/drivers/urldriver","github.com/globocom/gandalf/api","github.com/globocom/gandalf/repository","github.com/globocom/gandalf/user","github.com/globocom/mongoapi","github.com/globocom/tsuru/api","github.com/globocom/tsuru/app","github.com/globocom/tsuru/auth","github.com/globocom/tsuru/collector","github.com/globocom/tsuru/provision/docker","github.com/globocom/tsuru/provision/juju","github.com/globocom/tsuru/provision/lxc","github.com/globocom/tsuru/quota","github.com/globocom/tsuru/service","github.com/godfried/impendulo/db","github.com/godfried/impendulo/processing","github.com/godfried/impendulo/processing/monitor","github.com/godfried/impendulo/project","github.com/godfried/impendulo/server","github.com/godfried/impendulo/server/web","github.com/godfried/impendulo/tool","github.com/godfried/impendulo/tool/javac","github.com/godfried/impendulo/tool/jpf","github.com/godfried/impendulo/tool/junit","github.com/godfried/impendulo/util","github.com/gosexy/db/mongo","github.com/gregworley/koalab-golang-api","github.com/isaiah/tsuru_service","github.com/jasonmoo/gearman-go/client","github.com/jbaikge/coverage","github.com/jbaikge/est","github.com/jmcvetta/jfu","github.com/jmoiron/monet/app","github.com/jmoiron/monet/blog","github.com/jmoiron/monet/db","github.com/jmoiron/monet/gallery","github.com/johnwesonga/gotodolist","github.com/jordanorelli/go-instagram","github.com/jordanorelli/twitter","github.com/kidstuff/mtoy","github.com/kidstuff/mtoy/mgoauth","github.com/kidstuff/mtoy/mgosessions","github.com/liudian/mogogo/src/mogogo","github.com/lukegb/irclogsme","github.com/lukegb/irclogsme/logger","github.com/lukegb/irclogsme/server","github.com/mdennebaum/mgomodel","github.com/melvinmt/startupreader.com","github.com/mikespook/gearman-go/client","github.com/miraclesu/service","github.com/monnand/bully","github.com/mschoch/tuq/datasources/mongodb","github.com/msurdi/alf/db","github.com/netbrain/gonk/examples/authentication/app/role","github.com/netbrain/gonk/examples/authentication/app/user","github.com/nono/koalab-golang-api","github.com/nstott/mongobench","github.com/nvcnvn/glog","github.com/nvcnvn/glog/dbctx","github.com/nvcnvn/gorms","github.com/nvcnvn/gorms/dbctx","github.com/openvn/toys/secure/membership","github.com/openvn/toys/secure/membership/sessions","github.com/opesun/hypecms/model/basic","github.com/opesun/hypecms/model/patterns","github.com/opesun/hypecms/model/scut","github.com/opesun/hypecms/modules/admin/model","github.com/opesun/hypecms/modules/bootstrap","github.com/opesun/hypecms/modules/bootstrap/model","github.com/opesun/hypecms/modules/content","github.com/opesun/hypecms/modules/content/model","github.com/opesun/hypecms/modules/custom_actions","github.com/opesun/hypecms/modules/custom_actions/model","github.com/opesun/hypecms/modules/display_editor","github.com/opesun/hypecms/modules/display_editor/model","github.com/opesun/hypecms/modules/skeleton","github.com/opesun/hypecms/modules/template_editor","github.com/opesun/hypecms/modules/template_editor/model","github.com/opesun/hypecms/modules/user/model","github.com/opesun/nocrud/frame/impl/set/mongodb","github.com/opesun/nocrud/frame/misc/convert","github.com/opesun/nocrud/modules/fulltext","github.com/opesun/resolver","github.com/pavel-paulau/blurr/databases","github.com/pjvds/httpcallback.io/data/mongo","github.com/pjvds/httpcallback.io/model","github.com/prinsmike/GoVHostLog","github.com/prinsmike/gohome","github.com/reiver/turtledq","github.com/retzkek/transfat","github.com/rif/gocmd","github.com/rwynn/gtm","github.com/scottcagno/netkit","github.com/scottferg/goat","github.com/shawnps/mappuri","github.com/skelterjohn/bsonrpc","github.com/stretchr/codecs/bson","github.com/sunfmin/batchbuy/model","github.com/sunfmin/mgodb","github.com/sunfmin/tenpu/gridfs","github.com/sunfmin/tenpu/thumbnails","github.com/surma/importalias","github.com/tanema/mgorx","github.com/trevex/golem_examples","github.com/ungerik/go-start/model","github.com/ungerik/go-start/mongo","github.com/ungerik/go-start/mongomedia","github.com/ungerik/go-start/user","github.com/vbatts/imgsrv","github.com/wendyeq/iweb","github.com/wesnow/qufadai/src","github.com/xing4git/chirp/dao","github.com/xing4git/chirp/dao/redisdao","github.com/xing4git/chirp/model","github.com/xing4git/chirp/service/feedservice","github.com/zeebo/est","github.com/zeebo/goci/app/entities","github.com/zeebo/goci/app/frontend","github.com/zeebo/goci/app/httputil","github.com/zeebo/goci/app/notifications","github.com/zeebo/goci/app/response","github.com/zeebo/goci/app/tracker","github.com/zeebo/goci/app/workqueue","github.com/zeebo/gostbook","labix.org/v2/mgo","labix.org/v2/mgo/txn","launchpad.net/hockeypuck/mgo","launchpad.net/juju-core/charm","launchpad.net/juju-core/state","launchpad.net/juju-core/state/presence","launchpad.net/juju-core/state/watcher","launchpad.net/juju-core/store","launchpad.net/juju-core/version","launchpad.net/mgo/v2","github.com/AsherBond/docker","github.com/ChaosCloud/docker","github.com/CodeNow/docker","github.com/DanielBryan/docker","github.com/Jukkrapong/docker","github.com/ToothlessGear/docker","github.com/Vladimiroff/docker","github.com/ZeissS/docker","github.com/amaudy/docker","github.com/anachronistic/docker","github.com/apatil/docker-cpuset-cpus","github.com/apatil/docker-lxc-options","github.com/aybabtme/docker","github.com/bdon/docker","github.com/benoitc/docker","github.com/billyoung/docker","github.com/bits/docker","github.com/bpo/docker","github.com/bradobro/docker","github.com/c4milo/docker","github.com/calavera/docker","github.com/carlosdp/docker","github.com/cespare/docker","github.com/crosbymichael/docker","github.com/dhrp/docker","github.com/dillera/docker","github.com/dlintw/docker","github.com/dotcloud/docker","github.com/dr-strangecode/docker","github.com/dsissitka/docker","github.com/dynport/docker","github.com/ehazlett/docker","github.com/errnoh/docker","github.com/fmd/docker","github.com/fsouza/docker","github.com/fsouza/go-dockerclient","github.com/gaffo/docker","github.com/gale320/docker","github.com/hantuo/docker","github.com/hukeli/docker","github.com/irr/docker","github.com/ismell/docker","github.com/jaepil/docker","github.com/jamtur01/docker","github.com/jbardin/docker","github.com/jmcvetta/docker","github.com/johnbellone/docker","github.com/johnnydtan/docker","github.com/junk16/docker","github.com/justone/docker","github.com/kencochrane/docker","github.com/kisielk/docker","github.com/kmindg/docker","github.com/kpelykh/docker","github.com/kstaken/docker","github.com/lopter/docker","github.com/mars9/docker","github.com/maxhodak/docker","github.com/metalivedev/docker","github.com/mewpkg/docker","github.com/mhennings/docker","github.com/mindreframer/docker","github.com/monnand/docker","github.com/ndarilek/docker","github.com/nickstenning/docker","github.com/offby1/docker","github.com/ooyala/docker","github.com/oss17888/docker","github.com/petar/gocircuit-docker","github.com/philips/docker","github.com/pjvds/docker","github.com/rhoml/docker","github.com/richo/docker","github.com/ryfow/docker","github.com/sabzil/docker","github.com/shin-/docker","github.com/silpion/docker","github.com/sinhalabs/docker","github.com/sleekslush/docker","github.com/sridatta/docker","github.com/stevedomin/docker","github.com/steveruckdashel/docker","github.com/stfp/docker","github.com/synack/docker","github.com/timcubb/docker","github.com/titanous/docker","github.com/twmb/docker","github.com/unclejack/docker","github.com/vagmi/docker","github.com/zimbatm/docker","github.com/zsol/docker"]

cmd/tools/countdocs/main.go

@@ -12,8 +12,8 @@ import (
 )
 
 func main() {
-	//	path := "data/docs"
-	path := "data/docs-updated"
+	//	path := "../../shared/data/docs"
+	path := "../../shared/data/docs-updated"
 	kvDir := kv.DirInput(sophie.LocalFsPath(path))
 
 	cnt, err := kvDir.PartCount()

cmd/tools/exps/main.go

@@ -30,7 +30,7 @@ func main() {
 	countAll, countReadme, countHasSents := 0, 0, 0
 	countSents := 0
 
-	f, err := villa.Path("exps/notfound.txt").Create()
+	f, err := villa.Path("../../shared/data/exps/notfound.txt").Create()
 	if err != nil {
 		log.Fatal(err)
 	}

conf.yaml-example

@@ -0,0 +1,21 @@
+---
+web:
+  addr: ":8080"
+  root: "./server/"
+  loadtemplatepass: ''
+  autoloadtemplate: false
+back:
+  dbroot: "./shared/data/"
+crawler:
+  due_per_run: 1h
+  godoc: true
+  github_update: true
+  noncrawl_hosts: []
+  github:
+    clientid: ''
+    clientsecret: ''
+docdb:
+  nonstore_regexps: []
+bi:
+  data_path: "./shared/data/gcse.bolt"
+  web_path: "/bi"

pkg/configs/configs.go

@@ -30,12 +30,12 @@ const (
 
 var (
 	ServerAddr = ":8080"
-	ServerRoot = villa.Path("./server/")
+	ServerRoot = villa.Path("./server")
 
 	LoadTemplatePass = ""
 	AutoLoadTemplate = false
 
-	DataRoot = villa.Path("./shared/data/")
+	DataRoot = villa.Path("../shared/data/")
 
 	// producer: server, consumer: crawler
 	ImportPath villa.Path
@@ -60,7 +60,7 @@ var (
 func init() {
 	log.SetFlags(log.Flags() | log.Lshortfile)
 
-	conf, err := ljconf.Load("conf.json")
+	conf, err := ljconf.Load("./conf.json")
 	if err != nil {
 		// we must make sure configuration exist
 		log.Fatal(err)
@@ -91,7 +91,7 @@ func init() {
 
 	NonStorePackageRegexps = conf.StringList("docdb.nonstore_regexps", nil)
 
-	bi.DataPath = conf.String("bi.data_path", "/tmp/gcse.bolt")
+	bi.DataPath = conf.String("bi.data_path", "./shared/data/gcse.bolt")
 	BiWebPath = conf.String("bi.web_path", BiWebPath)
 }