diff --git a/pipelines/tocrawl/tocrawl.go b/pipelines/tocrawl/tocrawl.go index b82c89e..adc8263 100644 --- a/pipelines/tocrawl/tocrawl.go +++ b/pipelines/tocrawl/tocrawl.go @@ -98,6 +98,10 @@ func generateCrawlEntries(db *gcse.MemDB, hostFromID func(id string) string, out } host := hostFromID(id) + if host != "github.com" { + return nil + } + // check host black list if configs.NonCrawlHosts.Contain(host) { return nil