only tocrawl github

daviddengcn · Apr 15, 2018 · d320a1b · d320a1b
1 parent 7e73bad
commit d320a1b
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pipelines/tocrawl/tocrawl.go b/pipelines/tocrawl/tocrawl.go
@@ -98,6 +98,10 @@ func generateCrawlEntries(db *gcse.MemDB, hostFromID func(id string) string, out
 		}
 		host := hostFromID(id)
 
+		if host != "github.com" {
+			return nil
+		}
+
 		// check host black list
 		if configs.NonCrawlHosts.Contain(host) {
 			return nil