diff --git a/.gitignore b/.gitignore index 5842223..d0f8363 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,17 @@ _obj _test +./tocrawl +./indexer +./server +./spider +./mergedocs +./countdocs +./dump +./fillfound +./fixcrawldb +./exps + # Architecture specific extensions/prefixes *.[568vq] [568vq].out diff --git a/Makefile b/Makefile index 5897070..9a0fd15 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,11 @@ print-%: ; @echo $*=$($*) build: build-indexer build-crawler build-spider build-tocrawl build-mergedocs build-server +build-dist-%: + go build -o $(PWD)/dist/$* $(PWD)/cmd/$*/*.go + build-%: - go build -o $(PWD)/bin/$* $(PWD)/cmd/$*/*.go + go build -o $(PWD)/$* $(PWD)/cmd/$*/*.go run-%: # @echo $*=$($*) diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index b6025cd..0f3d331 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -97,7 +97,7 @@ func (crawlerMapper) MapEnd(c []sophie.Collector) error { } func cleanTempDir() { - tmpFn := villa.Path("../shared/data/gddo") + tmpFn := villa.Path("./shared/data/gddo") if err := tmpFn.RemoveAll(); err != nil { log.Printf("Delete %v failed: %v", tmpFn, err) } diff --git a/cmd/server/shared/data/imports/1/.done b/cmd/server/shared/data/imports/1/.done new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/shared/data/imports/1/links.json b/cmd/server/shared/data/imports/1/links.json new file mode 100644 index 0000000..fe539f2 --- /dev/null +++ b/cmd/server/shared/data/imports/1/links.json @@ -0,0 +1 @@ +["github.com/TechCatsLab/Andariel"] diff --git a/cmd/server/shared/data/imports/2/.done b/cmd/server/shared/data/imports/2/.done new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/shared/data/imports/2/links.json b/cmd/server/shared/data/imports/2/links.json new file mode 100644 index 0000000..fe539f2 --- /dev/null +++ b/cmd/server/shared/data/imports/2/links.json @@ -0,0 +1 @@ +["github.com/TechCatsLab/Andariel"] diff --git a/cmd/server/shared/data/imports/3/.done b/cmd/server/shared/data/imports/3/.done new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/shared/data/imports/3/links.json b/cmd/server/shared/data/imports/3/links.json new file mode 100644 index 0000000..fe539f2 --- /dev/null +++ b/cmd/server/shared/data/imports/3/links.json @@ -0,0 +1 @@ +["github.com/TechCatsLab/Andariel"] diff --git a/cmd/server/shared/data/imports/4/.done b/cmd/server/shared/data/imports/4/.done new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/shared/data/imports/4/links.json b/cmd/server/shared/data/imports/4/links.json new file mode 100644 index 0000000..2130bd7 --- /dev/null +++ b/cmd/server/shared/data/imports/4/links.json @@ -0,0 +1 @@ +["bitbucket.org/phlyingpenguin/collectinator","bitbucket.org/phlyingpenguin/collectinator/account","bitbucket.org/phlyingpenguin/collectinator/blog","bitbucket.org/phlyingpenguin/collectinator/common","code.google.com/p/sadbox/sessions/mongodb","git.300brand.com/coverage","git.300brand.com/coverage/storage/mongo","github.com/GeertJohan/outyet","github.com/MG-RAST/AWE/core","github.com/MG-RAST/Shock/shock-server","github.com/MG-RAST/Shock/shock-server/controller/node","github.com/MG-RAST/Shock/shock-server/node","github.com/MG-RAST/Shock/shock-server/preauth","github.com/MG-RAST/Shock/shock-server/user","github.com/abiosoft/gopages-sample/store","github.com/adnaan/hamster","github.com/alouca/MongoQueue","github.com/araddon/loges","github.com/araddon/mgou","github.com/arbaal/go-gridfs-serve","github.com/athom/tenpu","github.com/bketelsen/skynet/client","github.com/bketelsen/skynet/rpc/bsonrpc","github.com/bketelsen/skynet/service","github.com/cgrates/cgrates/rater","github.com/chrissexton/alepale/bot","github.com/chrissexton/alepale/plugins","github.com/drevell/mgou","github.com/edsrzf/mgo","github.com/emicklei/landskape/dao","github.com/fluffle/sp0rkle/collections/conf","github.com/fluffle/sp0rkle/collections/factoids","github.com/fluffle/sp0rkle/collections/karma","github.com/fluffle/sp0rkle/collections/markov","github.com/fluffle/sp0rkle/collections/quotes","github.com/fluffle/sp0rkle/collections/reminders","github.com/fluffle/sp0rkle/collections/seen","github.com/fluffle/sp0rkle/collections/stats","github.com/fluffle/sp0rkle/collections/urls","github.com/fluffle/sp0rkle/drivers/factdriver","github.com/fluffle/sp0rkle/drivers/quotedriver","github.com/fluffle/sp0rkle/drivers/reminddriver","github.com/fluffle/sp0rkle/drivers/urldriver","github.com/globocom/gandalf/api","github.com/globocom/gandalf/repository","github.com/globocom/gandalf/user","github.com/globocom/mongoapi","github.com/globocom/tsuru/api","github.com/globocom/tsuru/app","github.com/globocom/tsuru/auth","github.com/globocom/tsuru/collector","github.com/globocom/tsuru/provision/docker","github.com/globocom/tsuru/provision/juju","github.com/globocom/tsuru/provision/lxc","github.com/globocom/tsuru/quota","github.com/globocom/tsuru/service","github.com/godfried/impendulo/db","github.com/godfried/impendulo/processing","github.com/godfried/impendulo/processing/monitor","github.com/godfried/impendulo/project","github.com/godfried/impendulo/server","github.com/godfried/impendulo/server/web","github.com/godfried/impendulo/tool","github.com/godfried/impendulo/tool/javac","github.com/godfried/impendulo/tool/jpf","github.com/godfried/impendulo/tool/junit","github.com/godfried/impendulo/util","github.com/gosexy/db/mongo","github.com/gregworley/koalab-golang-api","github.com/isaiah/tsuru_service","github.com/jasonmoo/gearman-go/client","github.com/jbaikge/coverage","github.com/jbaikge/est","github.com/jmcvetta/jfu","github.com/jmoiron/monet/app","github.com/jmoiron/monet/blog","github.com/jmoiron/monet/db","github.com/jmoiron/monet/gallery","github.com/johnwesonga/gotodolist","github.com/jordanorelli/go-instagram","github.com/jordanorelli/twitter","github.com/kidstuff/mtoy","github.com/kidstuff/mtoy/mgoauth","github.com/kidstuff/mtoy/mgosessions","github.com/liudian/mogogo/src/mogogo","github.com/lukegb/irclogsme","github.com/lukegb/irclogsme/logger","github.com/lukegb/irclogsme/server","github.com/mdennebaum/mgomodel","github.com/melvinmt/startupreader.com","github.com/mikespook/gearman-go/client","github.com/miraclesu/service","github.com/monnand/bully","github.com/mschoch/tuq/datasources/mongodb","github.com/msurdi/alf/db","github.com/netbrain/gonk/examples/authentication/app/role","github.com/netbrain/gonk/examples/authentication/app/user","github.com/nono/koalab-golang-api","github.com/nstott/mongobench","github.com/nvcnvn/glog","github.com/nvcnvn/glog/dbctx","github.com/nvcnvn/gorms","github.com/nvcnvn/gorms/dbctx","github.com/openvn/toys/secure/membership","github.com/openvn/toys/secure/membership/sessions","github.com/opesun/hypecms/model/basic","github.com/opesun/hypecms/model/patterns","github.com/opesun/hypecms/model/scut","github.com/opesun/hypecms/modules/admin/model","github.com/opesun/hypecms/modules/bootstrap","github.com/opesun/hypecms/modules/bootstrap/model","github.com/opesun/hypecms/modules/content","github.com/opesun/hypecms/modules/content/model","github.com/opesun/hypecms/modules/custom_actions","github.com/opesun/hypecms/modules/custom_actions/model","github.com/opesun/hypecms/modules/display_editor","github.com/opesun/hypecms/modules/display_editor/model","github.com/opesun/hypecms/modules/skeleton","github.com/opesun/hypecms/modules/template_editor","github.com/opesun/hypecms/modules/template_editor/model","github.com/opesun/hypecms/modules/user/model","github.com/opesun/nocrud/frame/impl/set/mongodb","github.com/opesun/nocrud/frame/misc/convert","github.com/opesun/nocrud/modules/fulltext","github.com/opesun/resolver","github.com/pavel-paulau/blurr/databases","github.com/pjvds/httpcallback.io/data/mongo","github.com/pjvds/httpcallback.io/model","github.com/prinsmike/GoVHostLog","github.com/prinsmike/gohome","github.com/reiver/turtledq","github.com/retzkek/transfat","github.com/rif/gocmd","github.com/rwynn/gtm","github.com/scottcagno/netkit","github.com/scottferg/goat","github.com/shawnps/mappuri","github.com/skelterjohn/bsonrpc","github.com/stretchr/codecs/bson","github.com/sunfmin/batchbuy/model","github.com/sunfmin/mgodb","github.com/sunfmin/tenpu/gridfs","github.com/sunfmin/tenpu/thumbnails","github.com/surma/importalias","github.com/tanema/mgorx","github.com/trevex/golem_examples","github.com/ungerik/go-start/model","github.com/ungerik/go-start/mongo","github.com/ungerik/go-start/mongomedia","github.com/ungerik/go-start/user","github.com/vbatts/imgsrv","github.com/wendyeq/iweb","github.com/wesnow/qufadai/src","github.com/xing4git/chirp/dao","github.com/xing4git/chirp/dao/redisdao","github.com/xing4git/chirp/model","github.com/xing4git/chirp/service/feedservice","github.com/zeebo/est","github.com/zeebo/goci/app/entities","github.com/zeebo/goci/app/frontend","github.com/zeebo/goci/app/httputil","github.com/zeebo/goci/app/notifications","github.com/zeebo/goci/app/response","github.com/zeebo/goci/app/tracker","github.com/zeebo/goci/app/workqueue","github.com/zeebo/gostbook","labix.org/v2/mgo","labix.org/v2/mgo/txn","launchpad.net/hockeypuck/mgo","launchpad.net/juju-core/charm","launchpad.net/juju-core/state","launchpad.net/juju-core/state/presence","launchpad.net/juju-core/state/watcher","launchpad.net/juju-core/store","launchpad.net/juju-core/version","launchpad.net/mgo/v2","github.com/AsherBond/docker","github.com/ChaosCloud/docker","github.com/CodeNow/docker","github.com/DanielBryan/docker","github.com/Jukkrapong/docker","github.com/ToothlessGear/docker","github.com/Vladimiroff/docker","github.com/ZeissS/docker","github.com/amaudy/docker","github.com/anachronistic/docker","github.com/apatil/docker-cpuset-cpus","github.com/apatil/docker-lxc-options","github.com/aybabtme/docker","github.com/bdon/docker","github.com/benoitc/docker","github.com/billyoung/docker","github.com/bits/docker","github.com/bpo/docker","github.com/bradobro/docker","github.com/c4milo/docker","github.com/calavera/docker","github.com/carlosdp/docker","github.com/cespare/docker","github.com/crosbymichael/docker","github.com/dhrp/docker","github.com/dillera/docker","github.com/dlintw/docker","github.com/dotcloud/docker","github.com/dr-strangecode/docker","github.com/dsissitka/docker","github.com/dynport/docker","github.com/ehazlett/docker","github.com/errnoh/docker","github.com/fmd/docker","github.com/fsouza/docker","github.com/fsouza/go-dockerclient","github.com/gaffo/docker","github.com/gale320/docker","github.com/hantuo/docker","github.com/hukeli/docker","github.com/irr/docker","github.com/ismell/docker","github.com/jaepil/docker","github.com/jamtur01/docker","github.com/jbardin/docker","github.com/jmcvetta/docker","github.com/johnbellone/docker","github.com/johnnydtan/docker","github.com/junk16/docker","github.com/justone/docker","github.com/kencochrane/docker","github.com/kisielk/docker","github.com/kmindg/docker","github.com/kpelykh/docker","github.com/kstaken/docker","github.com/lopter/docker","github.com/mars9/docker","github.com/maxhodak/docker","github.com/metalivedev/docker","github.com/mewpkg/docker","github.com/mhennings/docker","github.com/mindreframer/docker","github.com/monnand/docker","github.com/ndarilek/docker","github.com/nickstenning/docker","github.com/offby1/docker","github.com/ooyala/docker","github.com/oss17888/docker","github.com/petar/gocircuit-docker","github.com/philips/docker","github.com/pjvds/docker","github.com/rhoml/docker","github.com/richo/docker","github.com/ryfow/docker","github.com/sabzil/docker","github.com/shin-/docker","github.com/silpion/docker","github.com/sinhalabs/docker","github.com/sleekslush/docker","github.com/sridatta/docker","github.com/stevedomin/docker","github.com/steveruckdashel/docker","github.com/stfp/docker","github.com/synack/docker","github.com/timcubb/docker","github.com/titanous/docker","github.com/twmb/docker","github.com/unclejack/docker","github.com/vagmi/docker","github.com/zimbatm/docker","github.com/zsol/docker"] diff --git a/cmd/tocrawl/main.go b/cmd/tocrawl/main.go index ab3413d..5767527 100644 --- a/cmd/tocrawl/main.go +++ b/cmd/tocrawl/main.go @@ -199,6 +199,7 @@ func main() { cDB = gcse.LoadCrawlerDB() // load pkgUTs + log.Println("pkgUTs: ", sophie.LocalFsPath(configs.DocsDBPath())) pkgUTs, err := loadPackageUpdateTimes(sophie.LocalFsPath(configs.DocsDBPath())) if err != nil { log.Fatalf("loadPackageUpdateTimes failed: %v", err) diff --git a/conf.json-example b/conf.json-example index 510c733..4a23778 100644 --- a/conf.json-example +++ b/conf.json-example @@ -1,7 +1,7 @@ { web: { // addr: ":8080" - // root: "./server/" + // root: "./cmd/server/" // loadtemplatepass: "" // autoloadtemplate: false } diff --git a/conf.yaml-example b/conf.yaml-example index 367de70..01484ea 100644 --- a/conf.yaml-example +++ b/conf.yaml-example @@ -1,7 +1,7 @@ --- web: addr: ":8080" - root: "./server/" + root: "./cmd/server/" loadtemplatepass: '' autoloadtemplate: false back: diff --git a/crawler b/crawler new file mode 100755 index 0000000..96cd8b6 Binary files /dev/null and b/crawler differ diff --git a/bin/dockerfile.dist-alpine3.6 b/dist/dockerfile.dist-alpine3.6 similarity index 100% rename from bin/dockerfile.dist-alpine3.6 rename to dist/dockerfile.dist-alpine3.6 diff --git a/bin/dockerfile.dist-scratch b/dist/dockerfile.dist-scratch similarity index 100% rename from bin/dockerfile.dist-scratch rename to dist/dockerfile.dist-scratch diff --git a/indexer b/indexer new file mode 100755 index 0000000..353211a Binary files /dev/null and b/indexer differ diff --git a/mergedocs b/mergedocs new file mode 100755 index 0000000..232b1d7 Binary files /dev/null and b/mergedocs differ diff --git a/pkg/configs/configs.go b/pkg/configs/configs.go index fdd3fac..0c7eeb8 100644 --- a/pkg/configs/configs.go +++ b/pkg/configs/configs.go @@ -30,12 +30,12 @@ const ( var ( ServerAddr = ":8080" - ServerRoot = villa.Path("./server") + ServerRoot = villa.Path("./") LoadTemplatePass = "" AutoLoadTemplate = false - DataRoot = villa.Path("../shared/data/") + DataRoot = villa.Path("./shared/data/") // producer: server, consumer: crawler ImportPath villa.Path @@ -46,7 +46,7 @@ var ( // configures of crawler CrawlByGodocApi = true CrawlGithubUpdate = true - CrawlerDuePerRun = 1 * time.Hour + CrawlerDuePerRun = 1 * time.Minute CrawlerGithubClientID = "" CrawlerGithubClientSecret = "" CrawlerGithubPersonal = "" diff --git a/server b/server new file mode 100755 index 0000000..7399f34 Binary files /dev/null and b/server differ diff --git a/shared/data/dbout/.gitignore b/shared/data/dbout/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/dbout/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/docs/.gitignore b/shared/data/docs/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/docs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/exps/.gitignore b/shared/data/exps/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/exps/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/imports/.gitignore b/shared/data/imports/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/imports/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/imports/0/.done b/shared/data/imports/0/.done new file mode 100644 index 0000000..e69de29 diff --git a/shared/data/imports/0/links.json b/shared/data/imports/0/links.json new file mode 100644 index 0000000..fe539f2 --- /dev/null +++ b/shared/data/imports/0/links.json @@ -0,0 +1 @@ +["github.com/TechCatsLab/Andariel"] diff --git a/shared/data/index/.gitignore b/shared/data/index/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/index/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/newdocs/.gitignore b/shared/data/newdocs/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/newdocs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/package/.gitignore b/shared/data/package/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/package/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/person/.gitignore b/shared/data/person/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/person/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/store/.gitignore b/shared/data/store/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/store/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/shared/data/tocrawl/.gitignore b/shared/data/tocrawl/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/shared/data/tocrawl/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/spider b/spider new file mode 100755 index 0000000..6aa4ebe Binary files /dev/null and b/spider differ