From 28ee850afe0399433c35eda4bd5f4a2c15195eee Mon Sep 17 00:00:00 2001 From: Yi Deng Date: Sun, 15 Apr 2018 16:55:57 -0700 Subject: [PATCH] Move protos to shared/proto --- crawler.go | 13 +- pipelines/crawler/package.go | 23 +- pipelines/spider/spider.go | 18 +- pipelines/spider/spider_test.go | 68 ++-- pipelines/tocrawl/ghup.go | 4 +- pipelines/tocrawl/tocrawl.go | 4 +- proto/store/store.go | 10 - proto/store/store.pb.go | 168 ---------- scripts/gen_proto.gs | 17 +- server/db.go | 6 +- {proto/spider => shared/proto}/spider.go | 2 +- {proto/spider => shared/proto}/spider.pb.go | 330 +++++++++++++++----- {proto/spider => shared/proto}/spider.proto | 4 +- shared/proto/store.go | 8 + shared/proto/store.pb.go | 280 +++++++++++++++++ {proto/store => shared/proto}/store.proto | 18 +- shared/proto/stored.pb.go | 61 ++++ shared/proto/stored.proto | 17 + spider/filecache_test.go | 8 +- spider/github/github.go | 48 +-- spider/ranking.go | 9 +- spider/ranking_test.go | 17 +- store/history.go | 38 +-- store/history_test.go | 49 +-- store/repo.go | 19 +- store/repo_test.go | 18 +- store/store.go | 33 +- store/store_test.go | 33 +- stored/stored.go | 10 + 29 files changed, 848 insertions(+), 485 deletions(-) delete mode 100644 proto/store/store.go delete mode 100644 proto/store/store.pb.go rename {proto/spider => shared/proto}/spider.go (95%) rename {proto/spider => shared/proto}/spider.pb.go (50%) rename {proto/spider => shared/proto}/spider.proto (98%) create mode 100644 shared/proto/store.go create mode 100644 shared/proto/store.pb.go rename {proto/store => shared/proto}/store.proto (67%) create mode 100644 shared/proto/stored.pb.go create mode 100644 shared/proto/stored.proto create mode 100644 stored/stored.go diff --git a/crawler.go b/crawler.go index d61afcc..3cb86a9 100644 --- a/crawler.go +++ b/crawler.go @@ -33,8 +33,7 @@ import ( "github.com/daviddengcn/sophie" "github.com/golang/gddo/gosrc" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" glgddo "github.com/golang/gddo/doc" ) @@ -350,7 +349,7 @@ var GithubSpider *github.Spider const maxRepoInfoAge = 2 * timep.Day -func CrawlRepoInfo(ctx context.Context, site, user, name string) *sppb.RepoInfo { +func CrawlRepoInfo(ctx context.Context, site, user, name string) *gpb.RepoInfo { // Check cache in store. path := user + "/" + name p, err := store.ReadPackage(site, path) @@ -373,7 +372,7 @@ func CrawlRepoInfo(ctx context.Context, site, user, name string) *sppb.RepoInfo } return nil } - if err := store.UpdatePackage(site, path, func(info *stpb.PackageInfo) error { + if err := store.UpdatePackage(site, path, func(info *gpb.PackageInfo) error { info.RepoInfo = ri return nil }); err != nil { @@ -392,7 +391,7 @@ func getGithubStars(ctx context.Context, user, name string) int { return -1 } -func getGithub(ctx context.Context, pkg string) (*doc.Package, []*sppb.FolderInfo, error) { +func getGithub(ctx context.Context, pkg string) (*doc.Package, []*gpb.FolderInfo, error) { parts := strings.SplitN(pkg, "/", 4) for len(parts) < 4 { parts = append(parts, "") @@ -421,7 +420,7 @@ func getGithub(ctx context.Context, pkg string) (*doc.Package, []*sppb.FolderInf }, folders, nil } -func CrawlPackage(ctx context.Context, httpClient doc.HttpClient, pkg string, etag string) (p *Package, folders []*sppb.FolderInfo, err error) { +func CrawlPackage(ctx context.Context, httpClient doc.HttpClient, pkg string, etag string) (p *Package, folders []*gpb.FolderInfo, err error) { defer func() { if perr := recover(); perr != nil { p, folders, err = nil, nil, errorsp.NewWithStacks("Panic when crawling package %s: %v", pkg, perr) @@ -537,7 +536,7 @@ func CrawlPerson(ctx context.Context, httpClient doc.HttpClient, id string) (*Pe for name, ri := range u.Repos { path := user + "/" + name p.Packages = append(p.Packages, "github.com/"+path) - if err := store.UpdatePackage(site, path, func(info *stpb.PackageInfo) error { + if err := store.UpdatePackage(site, path, func(info *gpb.PackageInfo) error { info.RepoInfo = ri return nil }); err != nil { diff --git a/pipelines/crawler/package.go b/pipelines/crawler/package.go index 8646230..a696867 100644 --- a/pipelines/crawler/package.go +++ b/pipelines/crawler/package.go @@ -22,8 +22,7 @@ import ( "github.com/daviddengcn/sophie/kv" "github.com/daviddengcn/sophie/mr" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) const ( @@ -47,17 +46,17 @@ func appendNewPackage(pkg, foundWay string) { cDB.AppendPackage(pkg, allDocsPkgs.Contain) site, path := utils.SplitPackage(pkg) - if err := store.UpdatePackage(site, path, func(*stpb.PackageInfo) error { + if err := store.UpdatePackage(site, path, func(*gpb.PackageInfo) error { return nil }); err != nil { log.Printf("UpdatePackage %s %s failed: %v", site, path, err) } - if err := store.AppendPackageEvent(site, path, foundWay, time.Now(), sppb.HistoryEvent_Action_None); err != nil { + if err := store.AppendPackageEvent(site, path, foundWay, time.Now(), gpb.HistoryEvent_Action_None); err != nil { log.Printf("UpdatePackageHistory %s %s failed: %v", site, path, err) } } -func fillPackageInfo(p *gcse.Package, pi *stpb.PackageInfo) { +func fillPackageInfo(p *gcse.Package, pi *gpb.PackageInfo) { pi.Package = p.Package pi.Name = p.Name pi.Synopsis = p.Synopsis @@ -84,7 +83,7 @@ func fillPackageInfo(p *gcse.Package, pi *stpb.PackageInfo) { } } -func saveRelatedInfo(pi *stpb.PackageInfo) { +func saveRelatedInfo(pi *gpb.PackageInfo) { // append new authors var site, id string if strings.HasPrefix(pi.Package, "github.com/") { @@ -94,7 +93,7 @@ func saveRelatedInfo(pi *stpb.PackageInfo) { } else { return } - if err := store.UpdatePerson(site, id, func(*stpb.PersonInfo) error { + if err := store.UpdatePerson(site, id, func(*gpb.PersonInfo) error { // TODO update history return nil }); err != nil { @@ -192,7 +191,7 @@ func (pc *PackageCrawler) Map(key, val sophie.SophieWriter, c []sophie.Collector if err != nil && errorsp.Cause(err) != gcse.ErrPackageNotModifed { log.Printf("[Part %d] Crawling pkg %s failed: %v", pc.part, pkg, err) if gcse.IsBadPackage(err) { - utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), sppb.HistoryEvent_Action_Invalid), "AppendPackageEvent %v %v failed", site, path) + utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), gpb.HistoryEvent_Action_Invalid), "AppendPackageEvent %v %v failed", site, path) bi.AddValue(bi.Sum, "crawler.package.wrong-package", 1) // a wrong path nda := gcse.NewDocAction{ @@ -202,7 +201,7 @@ func (pc *PackageCrawler) Map(key, val sophie.SophieWriter, c []sophie.Collector cDB.PackageDB.Delete(pkg) log.Printf("[Part %d] Remove wrong package %s", pc.part, pkg) } else { - utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), sppb.HistoryEvent_Action_Failed), "AppendPackageEvent %v %v failed", site, path) + utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), gpb.HistoryEvent_Action_Failed), "AppendPackageEvent %v %v failed", site, path) bi.Inc("crawler.package.failed") if strings.HasPrefix(pkg, "github.com/") { bi.Inc("crawler.package.failed.github") @@ -227,7 +226,7 @@ func (pc *PackageCrawler) Map(key, val sophie.SophieWriter, c []sophie.Collector } return nil } - utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), sppb.HistoryEvent_Action_Success), "AppendPackageEvent %v %v failed", site, path) + utils.LogError(store.AppendPackageEvent(site, path, "", time.Now(), gpb.HistoryEvent_Action_Success), "AppendPackageEvent %v %v failed", site, path) pc.failCount = 0 if errorsp.Cause(err) == gcse.ErrPackageNotModifed { // TODO crawling stars for unchanged project @@ -242,8 +241,8 @@ func (pc *PackageCrawler) Map(key, val sophie.SophieWriter, c []sophie.Collector } log.Printf("[Part %d] Crawled package %s success!", pc.part, pkg) - var pkgInfo *stpb.PackageInfo - if err := store.UpdatePackage(site, path, func(pi *stpb.PackageInfo) error { + var pkgInfo *gpb.PackageInfo + if err := store.UpdatePackage(site, path, func(pi *gpb.PackageInfo) error { fillPackageInfo(p, pi) pkgInfo = pi return nil diff --git a/pipelines/spider/spider.go b/pipelines/spider/spider.go index e9573cd..aa86ad3 100644 --- a/pipelines/spider/spider.go +++ b/pipelines/spider/spider.go @@ -10,20 +10,20 @@ import ( "github.com/golangplus/time" "github.com/daviddengcn/gcse/configs" - "github.com/daviddengcn/gcse/proto/spider" - "github.com/daviddengcn/gcse/proto/store" "github.com/daviddengcn/gcse/spider/github" "github.com/daviddengcn/gcse/store" + + gpb "github.com/daviddengcn/gcse/shared/proto" ) type RepositoryInfo struct { - *stpb.Repository + *gpb.Repository User string Name string } -func needCrawl(r *stpb.Repository) bool { +func needCrawl(r *gpb.Repository) bool { if r.CrawlingInfo == nil { return true } @@ -47,7 +47,7 @@ func selectRepos(site string, maxCrawl int) ([]*RepositoryInfo, error) { repos := heap.NewInterfaces(func(x, y interface{}) bool { return shouldCrawlLater(x.(*RepositoryInfo), y.(*RepositoryInfo)) }, maxCrawl) - if err := store.ForEachRepositoryOfSite(site, func(user, name string, doc *stpb.Repository) error { + if err := store.ForEachRepositoryOfSite(site, func(user, name string, doc *gpb.Repository) error { if !needCrawl(doc) { return nil } @@ -75,7 +75,7 @@ func crawlRepo(ctx context.Context, site string, repo *RepositoryInfo) error { if site != "github.com" { return errorsp.NewWithStacks("Cannot crawl the repository in %v", site) } - repo.CrawlingInfo = &sppb.CrawlingInfo{} + repo.CrawlingInfo = &gpb.CrawlingInfo{} repo.CrawlingInfo.SetCrawlingTime(now()) sha, err := githubSpider.RepoBranchSHA(ctx, repo.User, repo.Name, repo.Branch) @@ -87,8 +87,8 @@ func crawlRepo(ctx context.Context, site string, repo *RepositoryInfo) error { } repo.Signature = sha - repo.Packages = make(map[string]*sppb.Package) - if err := githubSpider.ReadRepo(ctx, repo.User, repo.Name, repo.Signature, func(path string, doc *sppb.Package) error { + repo.Packages = make(map[string]*gpb.Package) + if err := githubSpider.ReadRepo(ctx, repo.User, repo.Name, repo.Signature, func(path string, doc *gpb.Package) error { log.Printf("Package: %v", doc) repo.Packages[path] = doc return nil @@ -106,7 +106,7 @@ func crawlAndSaveRepo(ctx context.Context, site string, repo *RepositoryInfo) er } return err } - return store.UpdateRepository(site, repo.User, repo.Name, func(doc *stpb.Repository) error { + return store.UpdateRepository(site, repo.User, repo.Name, func(doc *gpb.Repository) error { *doc = *repo.Repository return nil }) diff --git a/pipelines/spider/spider_test.go b/pipelines/spider/spider_test.go index 41b88de..905998f 100644 --- a/pipelines/spider/spider_test.go +++ b/pipelines/spider/spider_test.go @@ -11,10 +11,10 @@ import ( "github.com/golangplus/time" "github.com/daviddengcn/gcse/configs" - "github.com/daviddengcn/gcse/proto/spider" - "github.com/daviddengcn/gcse/proto/store" "github.com/daviddengcn/gcse/spider/github" "github.com/daviddengcn/gcse/store" + + gpb "github.com/daviddengcn/gcse/shared/proto" ) func init() { @@ -23,17 +23,17 @@ func init() { func TestShouldCrawlLater(t *testing.T) { assert.True(t, "res", shouldCrawlLater(&RepositoryInfo{ - Repository: &stpb.Repository{ - CrawlingInfo: &sppb.CrawlingInfo{}, + Repository: &gpb.Repository{ + CrawlingInfo: &gpb.CrawlingInfo{}, }, }, &RepositoryInfo{ - Repository: &stpb.Repository{}, + Repository: &gpb.Repository{}, })) assert.False(t, "res", shouldCrawlLater(&RepositoryInfo{ - Repository: &stpb.Repository{}, + Repository: &gpb.Repository{}, }, &RepositoryInfo{ - Repository: &stpb.Repository{ - CrawlingInfo: &sppb.CrawlingInfo{}, + Repository: &gpb.Repository{ + CrawlingInfo: &gpb.CrawlingInfo{}, }, })) } @@ -48,21 +48,21 @@ func TestSelectRepos(t *testing.T) { user3 = "daviddeng" name3 = "go" ) - assert.NoError(t, store.UpdateRepository(site, user1, name1, func(r *stpb.Repository) error { + assert.NoError(t, store.UpdateRepository(site, user1, name1, func(r *gpb.Repository) error { r.Stars = 1 return nil })) now := time.Now() - assert.NoError(t, store.UpdateRepository(site, user2, name2, func(r *stpb.Repository) error { + assert.NoError(t, store.UpdateRepository(site, user2, name2, func(r *gpb.Repository) error { r.Stars = 2 - r.CrawlingInfo = &sppb.CrawlingInfo{} + r.CrawlingInfo = &gpb.CrawlingInfo{} r.CrawlingInfo.SetCrawlingTime(now.Add(-10 * timep.Day)) return nil })) ts3, _ := ptypes.TimestampProto(now.Add(-15 * timep.Day)) - assert.NoError(t, store.UpdateRepository(site, user3, name3, func(r *stpb.Repository) error { + assert.NoError(t, store.UpdateRepository(site, user3, name3, func(r *gpb.Repository) error { r.Stars = 3 - r.CrawlingInfo = &sppb.CrawlingInfo{ + r.CrawlingInfo = &gpb.CrawlingInfo{ CrawlingTime: ts3, } return nil @@ -71,15 +71,15 @@ func TestSelectRepos(t *testing.T) { repos, err := selectRepos(site, 2) assert.NoError(t, err) assert.Equal(t, "repos", repos, []*RepositoryInfo{{ - Repository: &stpb.Repository{ + Repository: &gpb.Repository{ Stars: 1, }, Name: name1, User: user1, }, { - Repository: &stpb.Repository{ + Repository: &gpb.Repository{ Stars: 3, - CrawlingInfo: &sppb.CrawlingInfo{ + CrawlingInfo: &gpb.CrawlingInfo{ CrawlingTime: ts3, }, }, @@ -132,25 +132,25 @@ func TestCrawlRepo(t *testing.T) { ) initSpider() r := &RepositoryInfo{ - Repository: &stpb.Repository{ + Repository: &gpb.Repository{ Branch: "master", - CrawlingInfo: (&sppb.CrawlingInfo{}).SetCrawlingTime(tm.Add(-timep.Day)), + CrawlingInfo: (&gpb.CrawlingInfo{}).SetCrawlingTime(tm.Add(-timep.Day)), }, User: user, Name: repo, } assert.NoError(t, crawlRepo(ctx, "github.com", r)) - assert.Equal(t, "r.Repository", *r.Repository, stpb.Repository{ + assert.Equal(t, "r.Repository", *r.Repository, gpb.Repository{ Branch: "master", Signature: "sha-1", - Packages: map[string]*sppb.Package{ + Packages: map[string]*gpb.Package{ "": { Name: "gcse", Path: "", Imports: []string{"github.com/daviddengcn/go-easybi"}, TestImports: nil, }}, - CrawlingInfo: (&sppb.CrawlingInfo{}).SetCrawlingTime(tm), + CrawlingInfo: (&gpb.CrawlingInfo{}).SetCrawlingTime(tm), }) } @@ -160,10 +160,10 @@ func TestCrawlRepo_Unchanged(t *testing.T) { now = timep.PresetNow(tm) initSpider() r := &RepositoryInfo{ - Repository: &stpb.Repository{ + Repository: &gpb.Repository{ Branch: "master", Signature: "sha-unchanged", - Packages: map[string]*sppb.Package{ + Packages: map[string]*gpb.Package{ "": { Name: "gcse", Path: "", @@ -175,17 +175,17 @@ func TestCrawlRepo_Unchanged(t *testing.T) { Name: "unchanged", } assert.NoError(t, crawlRepo(ctx, "github.com", r)) - assert.Equal(t, "r.Repository", *r.Repository, stpb.Repository{ + assert.Equal(t, "r.Repository", *r.Repository, gpb.Repository{ Branch: "master", Signature: "sha-unchanged", - Packages: map[string]*sppb.Package{ + Packages: map[string]*gpb.Package{ "": { Name: "gcse", Path: "", Imports: []string{"github.com/daviddengcn/go-easybi"}, TestImports: nil, }}, - CrawlingInfo: (&sppb.CrawlingInfo{}).SetCrawlingTime(tm), + CrawlingInfo: (&gpb.CrawlingInfo{}).SetCrawlingTime(tm), }) } @@ -200,14 +200,14 @@ func TestCrawlAndSaveRepo_RepositoryDeleted(t *testing.T) { ) initSpider() r := &RepositoryInfo{ - Repository: &stpb.Repository{ + Repository: &gpb.Repository{ Branch: "master", - CrawlingInfo: (&sppb.CrawlingInfo{}).SetCrawlingTime(tm.Add(-timep.Day)), + CrawlingInfo: (&gpb.CrawlingInfo{}).SetCrawlingTime(tm.Add(-timep.Day)), }, User: user, Name: repo, } - assert.NoError(t, store.UpdateRepository(site, user, repo, func(doc *stpb.Repository) error { + assert.NoError(t, store.UpdateRepository(site, user, repo, func(doc *gpb.Repository) error { *doc = *r.Repository return nil })) @@ -215,7 +215,7 @@ func TestCrawlAndSaveRepo_RepositoryDeleted(t *testing.T) { rp, err := store.ReadRepository(site, user, repo) assert.NoError(t, err) - assert.Equal(t, "rp", rp, &stpb.Repository{}) + assert.Equal(t, "rp", rp, &gpb.Repository{}) } func cleanDatabase(t *testing.T) { @@ -231,7 +231,7 @@ func TestExec(t *testing.T) { repo = "gcse" ) cleanDatabase(t) - assert.NoError(t, store.UpdateRepository(site, user, repo, func(r *stpb.Repository) error { + assert.NoError(t, store.UpdateRepository(site, user, repo, func(r *gpb.Repository) error { r.Branch = "master" return nil })) @@ -239,16 +239,16 @@ func TestExec(t *testing.T) { assert.NoError(t, exec(1, time.Hour)) r, err := store.ReadRepository(site, user, repo) assert.NoError(t, err) - assert.Equal(t, "r", *r, stpb.Repository{ + assert.Equal(t, "r", *r, gpb.Repository{ Branch: "master", Signature: "sha-1", - Packages: map[string]*sppb.Package{ + Packages: map[string]*gpb.Package{ "": { Name: "gcse", Path: "", Imports: []string{"github.com/daviddengcn/go-easybi"}, TestImports: nil, }}, - CrawlingInfo: (&sppb.CrawlingInfo{}).SetCrawlingTime(tm), + CrawlingInfo: (&gpb.CrawlingInfo{}).SetCrawlingTime(tm), }) } diff --git a/pipelines/tocrawl/ghup.go b/pipelines/tocrawl/ghup.go index 09bb104..9a38818 100644 --- a/pipelines/tocrawl/ghup.go +++ b/pipelines/tocrawl/ghup.go @@ -12,7 +12,7 @@ import ( "github.com/daviddengcn/gddo/doc" "github.com/golangplus/strings" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) // touchPackage forces a package to update if it was not crawled before a @@ -53,7 +53,7 @@ func touchByGithubUpdates(ctx context.Context, pkgUTs map[string]time.Time) { continue } touchPackage(fmt.Sprintf("github.com/%s/%s", user, path), r.UpdatedAt.Time, pkgUTs) - if err := store.AppendPackageEvent("github.com", user+"/"+path, "githubhupdate", now, sppb.HistoryEvent_Action_None); err != nil { + if err := store.AppendPackageEvent("github.com", user+"/"+path, "githubhupdate", now, gpb.HistoryEvent_Action_None); err != nil { log.Printf("UpdatePackageHistory %s %s failed: %v", "github.com", user+"/"+path, err) } count++ diff --git a/pipelines/tocrawl/tocrawl.go b/pipelines/tocrawl/tocrawl.go index adc8263..e8264dc 100644 --- a/pipelines/tocrawl/tocrawl.go +++ b/pipelines/tocrawl/tocrawl.go @@ -25,7 +25,7 @@ import ( "github.com/daviddengcn/sophie" "github.com/daviddengcn/sophie/kv" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) var ( @@ -242,7 +242,7 @@ func main() { return ok }) site, path := utils.SplitPackage(pkg) - if err := store.AppendPackageEvent(site, path, "godoc", now, sppb.HistoryEvent_Action_None); err != nil { + if err := store.AppendPackageEvent(site, path, "godoc", now, gpb.HistoryEvent_Action_None); err != nil { log.Printf("UpdatePackageHistory %s %s failed: %v", site, path, err) } } diff --git a/proto/store/store.go b/proto/store/store.go deleted file mode 100644 index e6b0bec..0000000 --- a/proto/store/store.go +++ /dev/null @@ -1,10 +0,0 @@ -package stpb - -import "github.com/daviddengcn/gcse/proto/spider" - -func (m *Repository) PutPackage(path string, pkg *sppb.Package) { - if m.Packages == nil { - m.Packages = make(map[string]*sppb.Package) - } - m.Packages[path] = pkg -} diff --git a/proto/store/store.pb.go b/proto/store/store.pb.go deleted file mode 100644 index 44a69df..0000000 --- a/proto/store/store.pb.go +++ /dev/null @@ -1,168 +0,0 @@ -// Code generated by protoc-gen-go. -// source: github.com/daviddengcn/gcse/proto/store/store.proto -// DO NOT EDIT! - -/* -Package stpb is a generated protocol buffer package. - -It is generated from these files: - github.com/daviddengcn/gcse/proto/store/store.proto - -It has these top-level messages: - PackageInfo - PersonInfo - Repository -*/ -package stpb - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import gcse_spider "github.com/daviddengcn/gcse/proto/spider" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -const _ = proto.ProtoPackageIsVersion1 - -type PackageInfo struct { - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - Package string `protobuf:"bytes,2,opt,name=package" json:"package,omitempty"` - Author string `protobuf:"bytes,3,opt,name=author" json:"author,omitempty"` - Stars int32 `protobuf:"varint,4,opt,name=stars" json:"stars,omitempty"` - Synopsis string `protobuf:"bytes,5,opt,name=synopsis" json:"synopsis,omitempty"` - Description string `protobuf:"bytes,6,opt,name=description" json:"description,omitempty"` - ProjectUrl string `protobuf:"bytes,7,opt,name=project_url,json=projectUrl" json:"project_url,omitempty"` - ReadmeFn string `protobuf:"bytes,8,opt,name=readme_fn,json=readmeFn" json:"readme_fn,omitempty"` - ReadmeData string `protobuf:"bytes,9,opt,name=readme_data,json=readmeData" json:"readme_data,omitempty"` - Imports []string `protobuf:"bytes,10,rep,name=imports" json:"imports,omitempty"` - TestImports []string `protobuf:"bytes,11,rep,name=test_imports,json=testImports" json:"test_imports,omitempty"` - Exported []string `protobuf:"bytes,12,rep,name=exported" json:"exported,omitempty"` - References []string `protobuf:"bytes,18,rep,name=references" json:"references,omitempty"` - CrawlingInfo *gcse_spider.CrawlingInfo `protobuf:"bytes,17,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` - // Available if the package is not the repo's root. - FolderInfo *gcse_spider.FolderInfo `protobuf:"bytes,14,opt,name=folder_info,json=folderInfo" json:"folder_info,omitempty"` - // Available if the package is the repo's root. - RepoInfo *gcse_spider.RepoInfo `protobuf:"bytes,15,opt,name=repo_info,json=repoInfo" json:"repo_info,omitempty"` -} - -func (m *PackageInfo) Reset() { *m = PackageInfo{} } -func (m *PackageInfo) String() string { return proto.CompactTextString(m) } -func (*PackageInfo) ProtoMessage() {} -func (*PackageInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -func (m *PackageInfo) GetCrawlingInfo() *gcse_spider.CrawlingInfo { - if m != nil { - return m.CrawlingInfo - } - return nil -} - -func (m *PackageInfo) GetFolderInfo() *gcse_spider.FolderInfo { - if m != nil { - return m.FolderInfo - } - return nil -} - -func (m *PackageInfo) GetRepoInfo() *gcse_spider.RepoInfo { - if m != nil { - return m.RepoInfo - } - return nil -} - -type PersonInfo struct { - CrawlingInfo *gcse_spider.CrawlingInfo `protobuf:"bytes,1,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` -} - -func (m *PersonInfo) Reset() { *m = PersonInfo{} } -func (m *PersonInfo) String() string { return proto.CompactTextString(m) } -func (*PersonInfo) ProtoMessage() {} -func (*PersonInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } - -func (m *PersonInfo) GetCrawlingInfo() *gcse_spider.CrawlingInfo { - if m != nil { - return m.CrawlingInfo - } - return nil -} - -type Repository struct { - Branch string `protobuf:"bytes,6,opt,name=branch" json:"branch,omitempty"` - Signature string `protobuf:"bytes,7,opt,name=signature" json:"signature,omitempty"` - // map from path to Package - Packages map[string]*gcse_spider.Package `protobuf:"bytes,8,rep,name=packages" json:"packages,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - ReadmeFn string `protobuf:"bytes,2,opt,name=ReadmeFn,json=readmeFn" json:"ReadmeFn,omitempty"` - ReadmeData string `protobuf:"bytes,3,opt,name=ReadmeData,json=readmeData" json:"ReadmeData,omitempty"` - Stars int32 `protobuf:"varint,4,opt,name=stars" json:"stars,omitempty"` - CrawlingInfo *gcse_spider.CrawlingInfo `protobuf:"bytes,5,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` -} - -func (m *Repository) Reset() { *m = Repository{} } -func (m *Repository) String() string { return proto.CompactTextString(m) } -func (*Repository) ProtoMessage() {} -func (*Repository) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } - -func (m *Repository) GetPackages() map[string]*gcse_spider.Package { - if m != nil { - return m.Packages - } - return nil -} - -func (m *Repository) GetCrawlingInfo() *gcse_spider.CrawlingInfo { - if m != nil { - return m.CrawlingInfo - } - return nil -} - -func init() { - proto.RegisterType((*PackageInfo)(nil), "gcse.store.PackageInfo") - proto.RegisterType((*PersonInfo)(nil), "gcse.store.PersonInfo") - proto.RegisterType((*Repository)(nil), "gcse.store.Repository") -} - -var fileDescriptor0 = []byte{ - // 548 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6f, 0x13, 0x3d, - 0x10, 0x56, 0x3e, 0x9b, 0xcc, 0xa6, 0xef, 0x0b, 0xa3, 0x02, 0x26, 0xa0, 0x12, 0x22, 0x0e, 0x88, - 0xc3, 0x46, 0x4a, 0x85, 0x54, 0x71, 0x40, 0x88, 0x8f, 0x4a, 0x95, 0x38, 0x94, 0x95, 0xb8, 0x70, - 0x89, 0x9c, 0x5d, 0x67, 0x63, 0x9a, 0xd8, 0x2b, 0xdb, 0x29, 0xe4, 0x47, 0xf1, 0xc3, 0xf8, 0x17, - 0xf8, 0x2b, 0x5f, 0x2a, 0x07, 0x7a, 0x49, 0x66, 0x9e, 0x67, 0x9e, 0xd9, 0xb1, 0xe7, 0x31, 0x9c, - 0x95, 0xdc, 0xcc, 0x57, 0xd3, 0x34, 0x97, 0xcb, 0x51, 0x41, 0x6f, 0x78, 0x51, 0x30, 0x51, 0xe6, - 0x62, 0x54, 0xe6, 0x9a, 0x8d, 0x2a, 0x25, 0x8d, 0x1c, 0x69, 0x23, 0x15, 0x0b, 0xbf, 0xa9, 0x47, - 0x10, 0x1c, 0x9b, 0x7a, 0xa4, 0xff, 0xfa, 0x1f, 0x1a, 0x54, 0xbc, 0x60, 0x2a, 0xfe, 0x85, 0x16, - 0xc3, 0x5f, 0x4d, 0x48, 0xae, 0x68, 0x7e, 0x4d, 0x4b, 0x76, 0x29, 0x66, 0x12, 0x11, 0x9a, 0x82, - 0x2e, 0x19, 0xa9, 0x0d, 0x6a, 0x2f, 0xbb, 0x99, 0x8f, 0x91, 0xc0, 0x51, 0x15, 0x4a, 0x48, 0xdd, - 0xc3, 0x9b, 0x14, 0x1f, 0x42, 0x9b, 0xae, 0xcc, 0x5c, 0x2a, 0xd2, 0xf0, 0x44, 0xcc, 0xf0, 0x04, - 0x5a, 0xda, 0x50, 0xa5, 0x49, 0xd3, 0xc2, 0xad, 0x2c, 0x24, 0xd8, 0x87, 0x8e, 0x5e, 0x0b, 0x59, - 0x69, 0xae, 0x49, 0xcb, 0xd7, 0x6f, 0x73, 0x1c, 0x40, 0x52, 0x30, 0x9d, 0x2b, 0x5e, 0x19, 0x2e, - 0x05, 0x69, 0x7b, 0x7a, 0x1f, 0xc2, 0x67, 0x90, 0xd8, 0x91, 0xbf, 0xb3, 0xdc, 0x4c, 0x56, 0x6a, - 0x41, 0x8e, 0x7c, 0x05, 0x44, 0xe8, 0xab, 0x5a, 0xe0, 0x13, 0xe8, 0x2a, 0x46, 0x8b, 0x25, 0x9b, - 0xcc, 0x04, 0xe9, 0x84, 0xfe, 0x01, 0xb8, 0xf0, 0xea, 0x48, 0x16, 0xd4, 0x50, 0xd2, 0x0d, 0xea, - 0x00, 0x7d, 0xb4, 0x88, 0x3b, 0x24, 0x5f, 0x56, 0x52, 0x19, 0x4d, 0x60, 0xd0, 0x70, 0x87, 0x8c, - 0x29, 0x3e, 0x87, 0x9e, 0x61, 0xda, 0x4c, 0x36, 0x74, 0xe2, 0xe9, 0xc4, 0x61, 0x97, 0xb1, 0xc4, - 0x9e, 0x8c, 0xfd, 0x74, 0x21, 0x2b, 0x48, 0xcf, 0xd3, 0xdb, 0x1c, 0x4f, 0xc1, 0x7e, 0x66, 0xc6, - 0x14, 0x13, 0x39, 0xd3, 0x04, 0x3d, 0xbb, 0x87, 0xe0, 0x5b, 0x38, 0xce, 0x15, 0xfd, 0xb1, 0xe0, - 0xa2, 0x9c, 0x70, 0xbb, 0x02, 0x72, 0xdf, 0xce, 0x96, 0x8c, 0x1f, 0xa7, 0x61, 0xb9, 0x61, 0x59, - 0x1f, 0x62, 0x85, 0xdb, 0x51, 0xd6, 0xcb, 0xf7, 0x32, 0x3c, 0x87, 0x64, 0x26, 0x17, 0xb6, 0x28, - 0xa8, 0xff, 0xf3, 0xea, 0x47, 0x07, 0xea, 0x0b, 0xcf, 0x7b, 0x2d, 0xcc, 0xb6, 0x31, 0x8e, 0xdd, - 0x85, 0x55, 0x32, 0xe8, 0xfe, 0xf7, 0xba, 0x07, 0x07, 0xba, 0xcc, 0xb2, 0x5e, 0xd5, 0x51, 0x31, - 0x1a, 0x7e, 0x06, 0xb8, 0x62, 0x4a, 0x4b, 0xe1, 0x3b, 0xdc, 0x9a, 0xbd, 0x76, 0xa7, 0xd9, 0x87, - 0xbf, 0xeb, 0x00, 0xee, 0x23, 0x9a, 0x5b, 0x0f, 0xaf, 0x9d, 0x9d, 0xa6, 0x8a, 0x8a, 0x7c, 0x1e, - 0xf7, 0x1f, 0x33, 0x7c, 0x0a, 0x5d, 0xcd, 0x4b, 0x41, 0xcd, 0x4a, 0xb1, 0xb8, 0xf8, 0x1d, 0x80, - 0xef, 0xa0, 0x13, 0xfd, 0xa8, 0xed, 0xda, 0x1b, 0xf6, 0xfb, 0x2f, 0xd2, 0xdd, 0xc3, 0x48, 0x77, - 0xfd, 0xd3, 0x68, 0x74, 0xfd, 0x49, 0x18, 0xb5, 0xce, 0xb6, 0x2a, 0xb7, 0xbe, 0x2c, 0x1a, 0x25, - 0x3a, 0x7c, 0x67, 0x9c, 0x53, 0x37, 0xe1, 0xc6, 0x25, 0xd1, 0xe6, 0xfb, 0xbe, 0xf9, 0xbb, 0xd5, - 0x6f, 0x5d, 0x4c, 0xeb, 0x4e, 0x17, 0xd3, 0xff, 0x02, 0xc7, 0x07, 0xc3, 0xe2, 0x3d, 0x68, 0x5c, - 0xb3, 0x75, 0x7c, 0x96, 0x2e, 0xc4, 0x57, 0xd0, 0xba, 0xa1, 0x8b, 0x55, 0x78, 0x93, 0xc9, 0xf8, - 0xe4, 0xa0, 0x75, 0x14, 0x67, 0xa1, 0xe4, 0x4d, 0xfd, 0xbc, 0xf6, 0xbe, 0xfd, 0xad, 0xa9, 0x4d, - 0x35, 0x9d, 0xb6, 0xfd, 0xc3, 0x3f, 0xfb, 0x13, 0x00, 0x00, 0xff, 0xff, 0x74, 0x07, 0x12, 0x9b, - 0x72, 0x04, 0x00, 0x00, -} diff --git a/scripts/gen_proto.gs b/scripts/gen_proto.gs index ff96fd7..ef895f8 100755 --- a/scripts/gen_proto.gs +++ b/scripts/gen_proto.gs @@ -4,19 +4,8 @@ import "path/filepath" const GCSE = "github.com/daviddengcn/gcse" +protoPath, _ := filepath.Abs("shared/proto/*.proto") gopath := Eval("go", "env", "GOPATH") + "/src" -count := 0 -filepath.Walk(filepath.Join(gopath, GCSE, "proto"), func(path string, info FileInfo, err error) error { - if err != nil || !info.IsDir() { - return nil - } - if ms, err := filepath.Glob(Sprintf("%s/*.proto", path)); len(ms) == 0 || err != nil { - return nil - } - count++ - Printfln("protoc --proto_path %[1]s --go_out %[1]s %s/*.proto", gopath, path) - Bash("protoc --proto_path %[1]s --go_out %[1]s %s/*.proto", gopath, path) - return nil -}) -Printfln("Total %d folders.", count) +Printfln("protoc --proto_path %[1]s --go_out %[1]s %s", gopath, protoPath) +Bash("protoc --proto_path %[1]s --go_out %[1]s %s", gopath, protoPath) diff --git a/server/db.go b/server/db.go index 9ca1321..510ac2b 100644 --- a/server/db.go +++ b/server/db.go @@ -18,7 +18,7 @@ import ( "github.com/daviddengcn/go-easybi" "github.com/daviddengcn/go-index" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) var ( @@ -36,7 +36,7 @@ type database interface { ForEachFullPackage(func(gcse.HitInfo) error) error PackageCountOfToken(field, token string) int Search(q map[string]stringsp.Set, out func(docID int32, data interface{}) error) error - PackageCrawlHistory(pkg string) *sppb.HistoryInfo + PackageCrawlHistory(pkg string) *gpb.HistoryInfo } type searcherDB struct { @@ -126,7 +126,7 @@ func (db *searcherDB) Search(q map[string]stringsp.Set, out func(docID int32, da return db.ts.Search(q, out) } -func (db *searcherDB) PackageCrawlHistory(pkg string) *sppb.HistoryInfo { +func (db *searcherDB) PackageCrawlHistory(pkg string) *gpb.HistoryInfo { site, path := utils.SplitPackage(pkg) info, err := store.ReadPackageHistoryOf(db.storeDB, site, path) if err != nil { diff --git a/proto/spider/spider.go b/shared/proto/spider.go similarity index 95% rename from proto/spider/spider.go rename to shared/proto/spider.go index a76eb79..b30da79 100644 --- a/proto/spider/spider.go +++ b/shared/proto/spider.go @@ -1,4 +1,4 @@ -package sppb +package gcsepb import ( "time" diff --git a/proto/spider/spider.pb.go b/shared/proto/spider.pb.go similarity index 50% rename from proto/spider/spider.pb.go rename to shared/proto/spider.pb.go index f725342..e58b276 100644 --- a/proto/spider/spider.pb.go +++ b/shared/proto/spider.pb.go @@ -1,12 +1,13 @@ -// Code generated by protoc-gen-go. -// source: github.com/daviddengcn/gcse/proto/spider/spider.proto -// DO NOT EDIT! +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: github.com/daviddengcn/gcse/shared/proto/spider.proto /* -Package sppb is a generated protocol buffer package. +Package gcsepb is a generated protocol buffer package. It is generated from these files: - github.com/daviddengcn/gcse/proto/spider/spider.proto + github.com/daviddengcn/gcse/shared/proto/spider.proto + github.com/daviddengcn/gcse/shared/proto/store.proto + github.com/daviddengcn/gcse/shared/proto/stored.proto It has these top-level messages: GoFileInfo @@ -16,8 +17,13 @@ It has these top-level messages: HistoryEvent HistoryInfo Package + PackageInfo + PersonInfo + Repository + PackageCrawlHistoryReq + PackageCrawlHistoryResp */ -package sppb +package gcsepb import proto "github.com/golang/protobuf/proto" import fmt "fmt" @@ -31,7 +37,9 @@ var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. -const _ = proto.ProtoPackageIsVersion1 +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type GoFileInfo_Status int32 @@ -90,7 +98,7 @@ func (HistoryEvent_Action_Enum) EnumDescriptor() ([]byte, []int) { } type GoFileInfo struct { - Status GoFileInfo_Status `protobuf:"varint,1,opt,name=status,enum=gcse.spider.GoFileInfo_Status" json:"status,omitempty"` + Status GoFileInfo_Status `protobuf:"varint,1,opt,name=status,enum=gcse.GoFileInfo_Status" json:"status,omitempty"` Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` Description string `protobuf:"bytes,3,opt,name=description" json:"description,omitempty"` IsTest bool `protobuf:"varint,4,opt,name=is_test,json=isTest" json:"is_test,omitempty"` @@ -102,6 +110,41 @@ func (m *GoFileInfo) String() string { return proto.CompactTextString func (*GoFileInfo) ProtoMessage() {} func (*GoFileInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (m *GoFileInfo) GetStatus() GoFileInfo_Status { + if m != nil { + return m.Status + } + return GoFileInfo_Unknown +} + +func (m *GoFileInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *GoFileInfo) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *GoFileInfo) GetIsTest() bool { + if m != nil { + return m.IsTest + } + return false +} + +func (m *GoFileInfo) GetImports() []string { + if m != nil { + return m.Imports + } + return nil +} + type RepoInfo struct { // The timestamp this repo-info is crawled CrawlingTime *google_protobuf.Timestamp `protobuf:"bytes,1,opt,name=crawling_time,json=crawlingTime" json:"crawling_time,omitempty"` @@ -125,6 +168,27 @@ func (m *RepoInfo) GetCrawlingTime() *google_protobuf.Timestamp { return nil } +func (m *RepoInfo) GetStars() int32 { + if m != nil { + return m.Stars + } + return 0 +} + +func (m *RepoInfo) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *RepoInfo) GetSource() string { + if m != nil { + return m.Source + } + return "" +} + func (m *RepoInfo) GetLastUpdated() *google_protobuf.Timestamp { if m != nil { return m.LastUpdated @@ -149,6 +213,34 @@ func (m *FolderInfo) String() string { return proto.CompactTextString func (*FolderInfo) ProtoMessage() {} func (*FolderInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (m *FolderInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *FolderInfo) GetPath() string { + if m != nil { + return m.Path + } + return "" +} + +func (m *FolderInfo) GetSha() string { + if m != nil { + return m.Sha + } + return "" +} + +func (m *FolderInfo) GetHtmlUrl() string { + if m != nil { + return m.HtmlUrl + } + return "" +} + func (m *FolderInfo) GetCrawlingTime() *google_protobuf.Timestamp { if m != nil { return m.CrawlingTime @@ -174,9 +266,16 @@ func (m *CrawlingInfo) GetCrawlingTime() *google_protobuf.Timestamp { return nil } +func (m *CrawlingInfo) GetEtag() string { + if m != nil { + return m.Etag + } + return "" +} + type HistoryEvent struct { Timestamp *google_protobuf.Timestamp `protobuf:"bytes,1,opt,name=timestamp" json:"timestamp,omitempty"` - Action HistoryEvent_Action_Enum `protobuf:"varint,2,opt,name=action,enum=gcse.spider.HistoryEvent_Action_Enum" json:"action,omitempty"` + Action HistoryEvent_Action_Enum `protobuf:"varint,2,opt,name=action,enum=gcse.HistoryEvent_Action_Enum" json:"action,omitempty"` } func (m *HistoryEvent) Reset() { *m = HistoryEvent{} } @@ -191,6 +290,13 @@ func (m *HistoryEvent) GetTimestamp() *google_protobuf.Timestamp { return nil } +func (m *HistoryEvent) GetAction() HistoryEvent_Action_Enum { + if m != nil { + return m.Action + } + return HistoryEvent_Action_None +} + type HistoryEvent_Action struct { } @@ -235,6 +341,13 @@ func (m *HistoryInfo) GetFoundTime() *google_protobuf.Timestamp { return nil } +func (m *HistoryInfo) GetFoundWay() string { + if m != nil { + return m.FoundWay + } + return "" +} + func (m *HistoryInfo) GetLatestSuccess() *google_protobuf.Timestamp { if m != nil { return m.LatestSuccess @@ -251,18 +364,18 @@ func (m *HistoryInfo) GetLatestFailed() *google_protobuf.Timestamp { type Package struct { // package "name" - Name string `protobuf:"bytes,1,opt,name=Name,json=name" json:"Name,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name" json:"Name,omitempty"` // Relative path to the repository, "" for root repository, "/sub" for a sub package. // Full path: site + "/" + user + "/" + repo + path - Path string `protobuf:"bytes,2,opt,name=Path,json=path" json:"Path,omitempty"` - Synopsis string `protobuf:"bytes,9,opt,name=Synopsis,json=synopsis" json:"Synopsis,omitempty"` - Description string `protobuf:"bytes,3,opt,name=Description,json=description" json:"Description,omitempty"` + Path string `protobuf:"bytes,2,opt,name=Path" json:"Path,omitempty"` + Synopsis string `protobuf:"bytes,9,opt,name=Synopsis" json:"Synopsis,omitempty"` + Description string `protobuf:"bytes,3,opt,name=Description" json:"Description,omitempty"` // No directory info - ReadmeFn string `protobuf:"bytes,4,opt,name=ReadmeFn,json=readmeFn" json:"ReadmeFn,omitempty"` + ReadmeFn string `protobuf:"bytes,4,opt,name=ReadmeFn" json:"ReadmeFn,omitempty"` // Raw content, cound be md, txt, etc. - ReadmeData string `protobuf:"bytes,5,opt,name=ReadmeData,json=readmeData" json:"ReadmeData,omitempty"` - Imports []string `protobuf:"bytes,6,rep,name=Imports,json=imports" json:"Imports,omitempty"` - TestImports []string `protobuf:"bytes,7,rep,name=TestImports,json=testImports" json:"TestImports,omitempty"` + ReadmeData string `protobuf:"bytes,5,opt,name=ReadmeData" json:"ReadmeData,omitempty"` + Imports []string `protobuf:"bytes,6,rep,name=Imports" json:"Imports,omitempty"` + TestImports []string `protobuf:"bytes,7,rep,name=TestImports" json:"TestImports,omitempty"` // URL to the package source code. Url string `protobuf:"bytes,8,opt,name=url" json:"url,omitempty"` } @@ -272,66 +385,133 @@ func (m *Package) String() string { return proto.CompactTextString(m) func (*Package) ProtoMessage() {} func (*Package) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (m *Package) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Package) GetPath() string { + if m != nil { + return m.Path + } + return "" +} + +func (m *Package) GetSynopsis() string { + if m != nil { + return m.Synopsis + } + return "" +} + +func (m *Package) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *Package) GetReadmeFn() string { + if m != nil { + return m.ReadmeFn + } + return "" +} + +func (m *Package) GetReadmeData() string { + if m != nil { + return m.ReadmeData + } + return "" +} + +func (m *Package) GetImports() []string { + if m != nil { + return m.Imports + } + return nil +} + +func (m *Package) GetTestImports() []string { + if m != nil { + return m.TestImports + } + return nil +} + +func (m *Package) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func init() { + proto.RegisterType((*GoFileInfo)(nil), "gcse.GoFileInfo") + proto.RegisterType((*RepoInfo)(nil), "gcse.RepoInfo") + proto.RegisterType((*FolderInfo)(nil), "gcse.FolderInfo") + proto.RegisterType((*CrawlingInfo)(nil), "gcse.CrawlingInfo") + proto.RegisterType((*HistoryEvent)(nil), "gcse.HistoryEvent") + proto.RegisterType((*HistoryEvent_Action)(nil), "gcse.HistoryEvent.Action") + proto.RegisterType((*HistoryInfo)(nil), "gcse.HistoryInfo") + proto.RegisterType((*Package)(nil), "gcse.Package") + proto.RegisterEnum("gcse.GoFileInfo_Status", GoFileInfo_Status_name, GoFileInfo_Status_value) + proto.RegisterEnum("gcse.HistoryEvent_Action_Enum", HistoryEvent_Action_Enum_name, HistoryEvent_Action_Enum_value) +} + func init() { - proto.RegisterType((*GoFileInfo)(nil), "gcse.spider.GoFileInfo") - proto.RegisterType((*RepoInfo)(nil), "gcse.spider.RepoInfo") - proto.RegisterType((*FolderInfo)(nil), "gcse.spider.FolderInfo") - proto.RegisterType((*CrawlingInfo)(nil), "gcse.spider.CrawlingInfo") - proto.RegisterType((*HistoryEvent)(nil), "gcse.spider.HistoryEvent") - proto.RegisterType((*HistoryEvent_Action)(nil), "gcse.spider.HistoryEvent.Action") - proto.RegisterType((*HistoryInfo)(nil), "gcse.spider.HistoryInfo") - proto.RegisterType((*Package)(nil), "gcse.spider.Package") - proto.RegisterEnum("gcse.spider.GoFileInfo_Status", GoFileInfo_Status_name, GoFileInfo_Status_value) - proto.RegisterEnum("gcse.spider.HistoryEvent_Action_Enum", HistoryEvent_Action_Enum_name, HistoryEvent_Action_Enum_value) + proto.RegisterFile("github.com/daviddengcn/gcse/shared/proto/spider.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 738 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x54, 0xdd, 0x6e, 0xd3, 0x4a, - 0x10, 0x3e, 0xf9, 0x73, 0xdc, 0x71, 0xda, 0x13, 0xad, 0x8e, 0xce, 0x49, 0x73, 0xa4, 0xaa, 0x8a, - 0x84, 0xc4, 0x95, 0x23, 0x8a, 0xa8, 0x40, 0x08, 0x41, 0xa1, 0x0d, 0x84, 0x8b, 0xaa, 0x72, 0x5a, - 0x21, 0x71, 0x13, 0x6d, 0xec, 0x8d, 0x63, 0xd5, 0xde, 0xb5, 0xbc, 0xeb, 0x56, 0x79, 0x17, 0xae, - 0x78, 0x1b, 0x5e, 0x81, 0x37, 0xe1, 0x8e, 0xfd, 0x73, 0x93, 0xf2, 0x53, 0x82, 0xc4, 0x95, 0x77, - 0xfe, 0x3c, 0x33, 0xdf, 0xcc, 0x37, 0xf0, 0x28, 0x4e, 0xc4, 0xa2, 0x9c, 0xf9, 0x21, 0xcb, 0x86, - 0x11, 0xbe, 0x4a, 0xa2, 0x88, 0xd0, 0x38, 0xa4, 0xc3, 0x38, 0xe4, 0x64, 0x98, 0x17, 0x4c, 0xb0, - 0x21, 0xcf, 0x93, 0x88, 0x14, 0xf6, 0xe3, 0x6b, 0x1d, 0xf2, 0x94, 0xdd, 0x37, 0xaa, 0xfe, 0xd3, - 0xb5, 0x7f, 0xc4, 0x2c, 0xc5, 0x34, 0x36, 0x91, 0xb3, 0x72, 0x3e, 0xcc, 0xc5, 0x32, 0x27, 0x7c, - 0x28, 0x92, 0x8c, 0x70, 0x81, 0xb3, 0x7c, 0xf5, 0x32, 0x7f, 0x1a, 0x7c, 0xa9, 0x01, 0xbc, 0x66, - 0xa3, 0x24, 0x25, 0x63, 0x3a, 0x67, 0xe8, 0x10, 0x1c, 0x69, 0x15, 0x25, 0xef, 0xd5, 0xf6, 0x6b, - 0xf7, 0x77, 0x0e, 0xf6, 0xfc, 0xb5, 0x4c, 0xfe, 0xca, 0xd1, 0x9f, 0x68, 0xaf, 0xc0, 0x7a, 0x23, - 0x04, 0x4d, 0x8a, 0x33, 0xd2, 0xab, 0xcb, 0xa8, 0xad, 0x40, 0xbf, 0xd1, 0x3e, 0x78, 0x11, 0xe1, - 0x61, 0x91, 0xe4, 0x22, 0x61, 0xb4, 0xd7, 0xd0, 0xa6, 0x75, 0x15, 0xfa, 0x0f, 0xda, 0x09, 0x9f, - 0x0a, 0x59, 0x50, 0xaf, 0x29, 0xad, 0x6e, 0xe0, 0x24, 0xfc, 0x5c, 0x4a, 0xa8, 0x27, 0x0d, 0x59, - 0xce, 0x0a, 0xc1, 0x7b, 0xad, 0xfd, 0x86, 0x0c, 0xab, 0xc4, 0xc1, 0x5b, 0x70, 0x4c, 0x6a, 0xe4, - 0x41, 0xfb, 0x82, 0x5e, 0x52, 0x76, 0x4d, 0xbb, 0x7f, 0xa1, 0x2e, 0x74, 0xce, 0x70, 0xc1, 0xc9, - 0xa4, 0x0c, 0x43, 0xc2, 0x79, 0xb7, 0x86, 0xfe, 0x06, 0x4f, 0x6b, 0x46, 0x58, 0x96, 0x1c, 0x75, - 0xeb, 0xca, 0x65, 0xb2, 0x60, 0x65, 0x1a, 0x8d, 0x63, 0xca, 0x0a, 0xd2, 0x6d, 0x0c, 0x3e, 0xd7, - 0xc0, 0x0d, 0x48, 0xce, 0x74, 0xe7, 0xcf, 0x61, 0x3b, 0x2c, 0xf0, 0x75, 0x9a, 0xd0, 0x78, 0xaa, - 0x40, 0xd2, 0x00, 0x78, 0x07, 0x7d, 0x3f, 0x66, 0x2c, 0x4e, 0x89, 0x5f, 0x41, 0xea, 0x9f, 0x57, - 0x08, 0x06, 0x9d, 0x2a, 0x40, 0xa9, 0xd0, 0x3f, 0xd0, 0x92, 0xea, 0x82, 0x6b, 0x0c, 0x5a, 0x81, - 0x11, 0x36, 0x00, 0xe1, 0x5f, 0x09, 0x39, 0x2b, 0x8b, 0x90, 0xc8, 0x56, 0x95, 0xd1, 0x4a, 0xe8, - 0x19, 0x74, 0x52, 0xcc, 0xc5, 0xb4, 0xcc, 0x23, 0x2c, 0x48, 0xa4, 0x11, 0xba, 0xbb, 0x1e, 0x4f, - 0xf9, 0x5f, 0x18, 0xf7, 0xc1, 0x47, 0x39, 0xd8, 0x11, 0x4b, 0xe5, 0xd8, 0x74, 0x7b, 0xd5, 0x80, - 0x6a, 0x6b, 0x03, 0x92, 0xba, 0x1c, 0x8b, 0x45, 0x35, 0x34, 0xf5, 0x96, 0x28, 0x35, 0xf8, 0x02, - 0xdb, 0x3a, 0xd5, 0x13, 0xed, 0x82, 0xbb, 0x10, 0x59, 0x3a, 0x2d, 0x8b, 0x54, 0xd7, 0x20, 0x87, - 0xa1, 0xe4, 0x8b, 0x22, 0xfd, 0x1e, 0xb3, 0xd6, 0xef, 0x61, 0x36, 0x08, 0xa1, 0xf3, 0xca, 0xca, - 0x7f, 0x66, 0x08, 0xb2, 0x25, 0x22, 0x70, 0x5c, 0xb5, 0xa4, 0xde, 0x83, 0x4f, 0x35, 0xe8, 0xbc, - 0x49, 0xb8, 0x60, 0xc5, 0xf2, 0xe4, 0x8a, 0x50, 0x81, 0x1e, 0xc3, 0xd6, 0x0d, 0x0d, 0x36, 0xc8, - 0xb0, 0x72, 0x96, 0x33, 0x71, 0x70, 0xa8, 0x07, 0x59, 0xd7, 0xf4, 0xb8, 0x77, 0x8b, 0x1e, 0xeb, - 0x49, 0xfc, 0x23, 0xed, 0xe7, 0x9f, 0xd0, 0x32, 0x0b, 0x6c, 0x50, 0xff, 0x05, 0x38, 0x46, 0x3d, - 0x38, 0x84, 0xa6, 0xb2, 0x20, 0x17, 0x9a, 0xa7, 0x8c, 0x12, 0xb9, 0xc1, 0x72, 0x9d, 0x57, 0xcb, - 0x0b, 0xe0, 0xdc, 0xec, 0xad, 0x34, 0x8c, 0xe9, 0x15, 0x4e, 0x93, 0x48, 0xae, 0xec, 0x87, 0x3a, - 0x78, 0x36, 0x8d, 0x06, 0xec, 0x01, 0x38, 0x44, 0xa5, 0x53, 0x7c, 0x6d, 0xc8, 0x3e, 0x76, 0x7f, - 0x5a, 0x50, 0x60, 0x1d, 0xd1, 0x13, 0x80, 0x39, 0x2b, 0x69, 0x64, 0x00, 0xae, 0xff, 0xba, 0x7d, - 0xed, 0xad, 0xd1, 0xfd, 0x1f, 0x8c, 0x30, 0xbd, 0xc6, 0x4b, 0xbb, 0x22, 0xae, 0x56, 0xbc, 0xc3, - 0x4b, 0x74, 0x04, 0x3b, 0x29, 0x56, 0x5c, 0x9e, 0x72, 0xd3, 0xc7, 0x06, 0x1b, 0xbb, 0x6d, 0x22, - 0x6c, 0xe3, 0x6a, 0xfc, 0xf6, 0x17, 0x73, 0xdd, 0xfd, 0x26, 0xfb, 0x64, 0x02, 0x0c, 0x5a, 0xea, - 0x9a, 0xb5, 0xcf, 0x70, 0x78, 0x89, 0x63, 0xbd, 0x0a, 0xa7, 0x3f, 0xd8, 0xf8, 0xb3, 0x6f, 0x37, - 0xbe, 0x0f, 0xee, 0x64, 0x49, 0x59, 0xce, 0x13, 0xde, 0xdb, 0x32, 0x3d, 0x71, 0x2b, 0x2b, 0xf6, - 0x1e, 0xdf, 0xcd, 0xde, 0xbe, 0x3a, 0x21, 0x38, 0xca, 0xc8, 0x88, 0x5a, 0x76, 0xb8, 0x85, 0x95, - 0xd1, 0x1e, 0x80, 0xb1, 0x1d, 0x63, 0x81, 0x2d, 0xbb, 0xa1, 0xb8, 0xd1, 0xa8, 0x2b, 0x37, 0xb6, - 0x57, 0xce, 0xb9, 0x75, 0xe5, 0x54, 0x5e, 0x75, 0x07, 0x2b, 0x6b, 0x5b, 0x5b, 0x3d, 0xb1, 0x52, - 0x29, 0x9e, 0x2a, 0x42, 0xba, 0x86, 0xa7, 0xf2, 0xf9, 0xd2, 0x79, 0xdf, 0xe4, 0x79, 0x3e, 0x9b, - 0x39, 0x1a, 0xa5, 0x87, 0x5f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xee, 0xec, 0x45, 0x2d, 0x5b, 0x06, - 0x00, 0x00, + // 748 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xdd, 0x6a, 0xdb, 0x48, + 0x14, 0x8e, 0xfc, 0x23, 0xcb, 0x47, 0x4e, 0x56, 0x0c, 0xcb, 0x46, 0xeb, 0x85, 0x60, 0x74, 0x65, + 0xf6, 0x42, 0x82, 0x2c, 0x1b, 0x76, 0x59, 0x96, 0x36, 0x6d, 0xe2, 0xd6, 0xbd, 0x08, 0x41, 0x4e, + 0x28, 0xf4, 0xc6, 0x8c, 0xa5, 0xb1, 0x2c, 0x22, 0xcd, 0x08, 0xcd, 0x28, 0xc1, 0x0f, 0xd2, 0x17, + 0xe8, 0xa3, 0xf4, 0x31, 0xfa, 0x18, 0xbd, 0xeb, 0x5d, 0x99, 0x19, 0x29, 0x16, 0x04, 0x5a, 0x17, + 0x7a, 0x77, 0x7e, 0xbe, 0x99, 0x33, 0xe7, 0x3b, 0xe7, 0x1b, 0xf8, 0x3b, 0x49, 0xc5, 0xa6, 0x5a, + 0xf9, 0x11, 0xcb, 0x83, 0x18, 0xdf, 0xa7, 0x71, 0x4c, 0x68, 0x12, 0xd1, 0x20, 0x89, 0x38, 0x09, + 0xf8, 0x06, 0x97, 0x24, 0x0e, 0x8a, 0x92, 0x09, 0x16, 0xf0, 0x22, 0x8d, 0x49, 0xe9, 0x2b, 0x07, + 0xf5, 0x64, 0x7e, 0xfc, 0x5f, 0xeb, 0x70, 0xc2, 0x32, 0x4c, 0x13, 0x8d, 0x5d, 0x55, 0xeb, 0xa0, + 0x10, 0xdb, 0x82, 0xf0, 0x40, 0xa4, 0x39, 0xe1, 0x02, 0xe7, 0xc5, 0xce, 0xd2, 0x57, 0x78, 0x9f, + 0x0d, 0x80, 0x57, 0x6c, 0x96, 0x66, 0x64, 0x4e, 0xd7, 0x0c, 0x05, 0x60, 0x72, 0x81, 0x45, 0xc5, + 0x5d, 0x63, 0x62, 0x4c, 0x8f, 0x4e, 0x8f, 0x7d, 0x59, 0xc2, 0xdf, 0x21, 0xfc, 0x85, 0x4a, 0x87, + 0x35, 0x0c, 0x21, 0xe8, 0x51, 0x9c, 0x13, 0xb7, 0x33, 0x31, 0xa6, 0xc3, 0x50, 0xd9, 0x68, 0x02, + 0x76, 0x4c, 0x78, 0x54, 0xa6, 0x85, 0x48, 0x19, 0x75, 0xbb, 0x2a, 0xd5, 0x0e, 0xa1, 0x63, 0x18, + 0xa4, 0x7c, 0x29, 0x08, 0x17, 0x6e, 0x6f, 0x62, 0x4c, 0xad, 0xd0, 0x4c, 0xf9, 0x0d, 0xe1, 0x02, + 0xb9, 0x30, 0x48, 0xf3, 0x82, 0x95, 0x82, 0xbb, 0xfd, 0x49, 0x77, 0x3a, 0x0c, 0x1b, 0xd7, 0x7b, + 0x03, 0xa6, 0x2e, 0x8d, 0x6c, 0x18, 0xdc, 0xd2, 0x3b, 0xca, 0x1e, 0xa8, 0x73, 0x80, 0x1c, 0x18, + 0x5d, 0xe3, 0x92, 0x93, 0x45, 0x15, 0x45, 0x84, 0x73, 0xc7, 0x40, 0xbf, 0x80, 0xad, 0x22, 0x33, + 0x9c, 0x66, 0x24, 0x76, 0x3a, 0x12, 0xb2, 0xd8, 0xb0, 0x2a, 0x8b, 0xe7, 0x09, 0x65, 0x25, 0x71, + 0xba, 0xde, 0x27, 0x03, 0xac, 0x90, 0x14, 0x4c, 0xb5, 0xfc, 0x0c, 0x0e, 0xa3, 0x12, 0x3f, 0x64, + 0x29, 0x4d, 0x96, 0x92, 0x1d, 0xd5, 0xb9, 0x7d, 0x3a, 0xf6, 0x13, 0xc6, 0x92, 0x8c, 0xf8, 0x0d, + 0x97, 0xfe, 0x4d, 0x43, 0x5d, 0x38, 0x6a, 0x0e, 0xc8, 0x10, 0xfa, 0x15, 0xfa, 0x5c, 0xe0, 0x92, + 0x2b, 0x0e, 0xfa, 0xa1, 0x76, 0xf6, 0x20, 0xe1, 0x37, 0x30, 0x39, 0xab, 0xca, 0x88, 0xb8, 0x7d, + 0x95, 0xac, 0x3d, 0xf4, 0x3f, 0x8c, 0x32, 0xcc, 0xc5, 0xb2, 0x2a, 0x62, 0x2c, 0x48, 0xac, 0x18, + 0xfa, 0xf6, 0x7b, 0x6c, 0x89, 0xbf, 0xd5, 0x70, 0xef, 0x83, 0x01, 0x30, 0x63, 0x59, 0x4c, 0x4a, + 0xd5, 0x5e, 0x33, 0x20, 0xa3, 0x35, 0x20, 0x04, 0xbd, 0x02, 0x8b, 0x4d, 0x33, 0x34, 0x69, 0x23, + 0x07, 0xba, 0x7c, 0x83, 0xeb, 0x77, 0x4a, 0x13, 0xfd, 0x0e, 0xd6, 0x46, 0xe4, 0xd9, 0xb2, 0x2a, + 0x33, 0xf5, 0x86, 0x61, 0x38, 0x90, 0xfe, 0x6d, 0x99, 0x3d, 0xe5, 0xac, 0xff, 0x63, 0x9c, 0x79, + 0x11, 0x8c, 0x5e, 0xd6, 0xfe, 0xcf, 0x19, 0x02, 0x82, 0x1e, 0x11, 0x38, 0x69, 0x5a, 0x92, 0xb6, + 0xf7, 0xd1, 0x80, 0xd1, 0xeb, 0x94, 0x0b, 0x56, 0x6e, 0x2f, 0xef, 0x09, 0x15, 0xe8, 0x1f, 0x18, + 0x3e, 0xee, 0xff, 0x1e, 0x15, 0x76, 0x60, 0x74, 0x06, 0x26, 0x8e, 0xd4, 0x20, 0x3b, 0x4a, 0x17, + 0x27, 0x5a, 0x17, 0xed, 0xdb, 0xfd, 0x73, 0x05, 0xf0, 0x2f, 0x69, 0x95, 0x87, 0x35, 0x7a, 0xfc, + 0x1c, 0x4c, 0x1d, 0xf6, 0xce, 0xa0, 0x27, 0x33, 0xc8, 0x82, 0xde, 0x15, 0xa3, 0xc4, 0x39, 0x90, + 0x7b, 0xbc, 0xdb, 0x5a, 0x00, 0xf3, 0x71, 0x61, 0x6d, 0x18, 0xcc, 0xe9, 0x3d, 0xce, 0xd2, 0xd8, + 0xe9, 0x7a, 0xef, 0x3b, 0x60, 0xd7, 0x65, 0x14, 0x53, 0x7f, 0x82, 0x49, 0x64, 0x39, 0xa9, 0xd0, + 0xee, 0xd4, 0x3e, 0x45, 0x4f, 0x5f, 0x12, 0xd6, 0x08, 0xf4, 0x2f, 0xc0, 0x9a, 0x55, 0x34, 0xd6, + 0x94, 0x76, 0xbe, 0xdf, 0xb0, 0x42, 0x2b, 0x3e, 0xff, 0x00, 0xed, 0x2c, 0x1f, 0xf0, 0xb6, 0x5e, + 0x0a, 0x4b, 0x05, 0xde, 0xe2, 0x2d, 0x3a, 0x87, 0xa3, 0x0c, 0x4b, 0xf5, 0x2e, 0xb9, 0x6e, 0x60, + 0x8f, 0x1d, 0x3d, 0xd4, 0x27, 0xea, 0x8e, 0xe5, 0xc0, 0xeb, 0x2b, 0xd6, 0xaa, 0xed, 0x7d, 0x36, + 0x48, 0x1f, 0xd0, 0x34, 0x79, 0x5f, 0x0c, 0x18, 0x5c, 0xe3, 0xe8, 0x0e, 0x27, 0x6a, 0xf8, 0x57, + 0xad, 0x1d, 0xbf, 0xaa, 0x77, 0xfc, 0xba, 0xb5, 0xe3, 0xd2, 0x46, 0x63, 0xb0, 0x16, 0x5b, 0xca, + 0x0a, 0x9e, 0x72, 0x77, 0xa8, 0x7b, 0x6a, 0x7c, 0xa9, 0xd7, 0x8b, 0xa7, 0x7a, 0x6d, 0x85, 0xe4, + 0xe9, 0x90, 0xe0, 0x38, 0x27, 0x33, 0x5a, 0xeb, 0xe1, 0xd1, 0x47, 0x27, 0x00, 0xda, 0xbe, 0xc0, + 0x02, 0xd7, 0x7a, 0x6e, 0x45, 0xe4, 0xbf, 0x36, 0xaf, 0xff, 0x35, 0x53, 0xff, 0x6b, 0xb5, 0x2b, + 0xeb, 0xca, 0x9f, 0xaf, 0xc9, 0x0e, 0x54, 0xb6, 0x1d, 0x92, 0xca, 0x94, 0x12, 0xb4, 0xb4, 0x32, + 0xab, 0x32, 0x7b, 0x61, 0xbd, 0x33, 0xe5, 0xd0, 0x8b, 0xd5, 0xca, 0x54, 0x3c, 0xfd, 0xf5, 0x35, + 0x00, 0x00, 0xff, 0xff, 0xde, 0x45, 0xb0, 0x53, 0x41, 0x06, 0x00, 0x00, } diff --git a/proto/spider/spider.proto b/shared/proto/spider.proto similarity index 98% rename from proto/spider/spider.proto rename to shared/proto/spider.proto index 497160e..c72d5cc 100644 --- a/proto/spider/spider.proto +++ b/shared/proto/spider.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package gcse.spider; +package gcse; -option go_package = "sppb"; +option go_package = "gcsepb"; import "github.com/golang/protobuf/ptypes/timestamp/timestamp.proto"; diff --git a/shared/proto/store.go b/shared/proto/store.go new file mode 100644 index 0000000..81c7de4 --- /dev/null +++ b/shared/proto/store.go @@ -0,0 +1,8 @@ +package gcsepb + +func (m *Repository) PutPackage(path string, pkg *Package) { + if m.Packages == nil { + m.Packages = make(map[string]*Package) + } + m.Packages[path] = pkg +} diff --git a/shared/proto/store.pb.go b/shared/proto/store.pb.go new file mode 100644 index 0000000..e8931ec --- /dev/null +++ b/shared/proto/store.pb.go @@ -0,0 +1,280 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: github.com/daviddengcn/gcse/shared/proto/store.proto + +package gcsepb + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type PackageInfo struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Package string `protobuf:"bytes,2,opt,name=package" json:"package,omitempty"` + Author string `protobuf:"bytes,3,opt,name=author" json:"author,omitempty"` + Stars int32 `protobuf:"varint,4,opt,name=stars" json:"stars,omitempty"` + Synopsis string `protobuf:"bytes,5,opt,name=synopsis" json:"synopsis,omitempty"` + Description string `protobuf:"bytes,6,opt,name=description" json:"description,omitempty"` + ProjectUrl string `protobuf:"bytes,7,opt,name=project_url,json=projectUrl" json:"project_url,omitempty"` + ReadmeFn string `protobuf:"bytes,8,opt,name=readme_fn,json=readmeFn" json:"readme_fn,omitempty"` + ReadmeData string `protobuf:"bytes,9,opt,name=readme_data,json=readmeData" json:"readme_data,omitempty"` + Imports []string `protobuf:"bytes,10,rep,name=imports" json:"imports,omitempty"` + TestImports []string `protobuf:"bytes,11,rep,name=test_imports,json=testImports" json:"test_imports,omitempty"` + Exported []string `protobuf:"bytes,12,rep,name=exported" json:"exported,omitempty"` + References []string `protobuf:"bytes,18,rep,name=references" json:"references,omitempty"` + CrawlingInfo *CrawlingInfo `protobuf:"bytes,17,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` + // Available if the package is not the repo's root. + FolderInfo *FolderInfo `protobuf:"bytes,14,opt,name=folder_info,json=folderInfo" json:"folder_info,omitempty"` + // Available if the package is the repo's root. + RepoInfo *RepoInfo `protobuf:"bytes,15,opt,name=repo_info,json=repoInfo" json:"repo_info,omitempty"` +} + +func (m *PackageInfo) Reset() { *m = PackageInfo{} } +func (m *PackageInfo) String() string { return proto.CompactTextString(m) } +func (*PackageInfo) ProtoMessage() {} +func (*PackageInfo) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *PackageInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PackageInfo) GetPackage() string { + if m != nil { + return m.Package + } + return "" +} + +func (m *PackageInfo) GetAuthor() string { + if m != nil { + return m.Author + } + return "" +} + +func (m *PackageInfo) GetStars() int32 { + if m != nil { + return m.Stars + } + return 0 +} + +func (m *PackageInfo) GetSynopsis() string { + if m != nil { + return m.Synopsis + } + return "" +} + +func (m *PackageInfo) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *PackageInfo) GetProjectUrl() string { + if m != nil { + return m.ProjectUrl + } + return "" +} + +func (m *PackageInfo) GetReadmeFn() string { + if m != nil { + return m.ReadmeFn + } + return "" +} + +func (m *PackageInfo) GetReadmeData() string { + if m != nil { + return m.ReadmeData + } + return "" +} + +func (m *PackageInfo) GetImports() []string { + if m != nil { + return m.Imports + } + return nil +} + +func (m *PackageInfo) GetTestImports() []string { + if m != nil { + return m.TestImports + } + return nil +} + +func (m *PackageInfo) GetExported() []string { + if m != nil { + return m.Exported + } + return nil +} + +func (m *PackageInfo) GetReferences() []string { + if m != nil { + return m.References + } + return nil +} + +func (m *PackageInfo) GetCrawlingInfo() *CrawlingInfo { + if m != nil { + return m.CrawlingInfo + } + return nil +} + +func (m *PackageInfo) GetFolderInfo() *FolderInfo { + if m != nil { + return m.FolderInfo + } + return nil +} + +func (m *PackageInfo) GetRepoInfo() *RepoInfo { + if m != nil { + return m.RepoInfo + } + return nil +} + +type PersonInfo struct { + CrawlingInfo *CrawlingInfo `protobuf:"bytes,1,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` +} + +func (m *PersonInfo) Reset() { *m = PersonInfo{} } +func (m *PersonInfo) String() string { return proto.CompactTextString(m) } +func (*PersonInfo) ProtoMessage() {} +func (*PersonInfo) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{1} } + +func (m *PersonInfo) GetCrawlingInfo() *CrawlingInfo { + if m != nil { + return m.CrawlingInfo + } + return nil +} + +type Repository struct { + Branch string `protobuf:"bytes,6,opt,name=branch" json:"branch,omitempty"` + Signature string `protobuf:"bytes,7,opt,name=signature" json:"signature,omitempty"` + // map from relative path, e.g. "proto/store", to Package + Packages map[string]*Package `protobuf:"bytes,8,rep,name=packages" json:"packages,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ReadmeFn string `protobuf:"bytes,2,opt,name=ReadmeFn" json:"ReadmeFn,omitempty"` + ReadmeData string `protobuf:"bytes,3,opt,name=ReadmeData" json:"ReadmeData,omitempty"` + Stars int32 `protobuf:"varint,4,opt,name=stars" json:"stars,omitempty"` + CrawlingInfo *CrawlingInfo `protobuf:"bytes,5,opt,name=crawling_info,json=crawlingInfo" json:"crawling_info,omitempty"` +} + +func (m *Repository) Reset() { *m = Repository{} } +func (m *Repository) String() string { return proto.CompactTextString(m) } +func (*Repository) ProtoMessage() {} +func (*Repository) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{2} } + +func (m *Repository) GetBranch() string { + if m != nil { + return m.Branch + } + return "" +} + +func (m *Repository) GetSignature() string { + if m != nil { + return m.Signature + } + return "" +} + +func (m *Repository) GetPackages() map[string]*Package { + if m != nil { + return m.Packages + } + return nil +} + +func (m *Repository) GetReadmeFn() string { + if m != nil { + return m.ReadmeFn + } + return "" +} + +func (m *Repository) GetReadmeData() string { + if m != nil { + return m.ReadmeData + } + return "" +} + +func (m *Repository) GetStars() int32 { + if m != nil { + return m.Stars + } + return 0 +} + +func (m *Repository) GetCrawlingInfo() *CrawlingInfo { + if m != nil { + return m.CrawlingInfo + } + return nil +} + +func init() { + proto.RegisterType((*PackageInfo)(nil), "gcse.PackageInfo") + proto.RegisterType((*PersonInfo)(nil), "gcse.PersonInfo") + proto.RegisterType((*Repository)(nil), "gcse.Repository") +} + +func init() { + proto.RegisterFile("github.com/daviddengcn/gcse/shared/proto/store.proto", fileDescriptor1) +} + +var fileDescriptor1 = []byte{ + // 538 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0x5d, 0x6b, 0x13, 0x41, + 0x14, 0x65, 0x9b, 0x8f, 0x6e, 0xee, 0xb6, 0xb5, 0x0e, 0x22, 0x43, 0x94, 0xba, 0xc6, 0x97, 0x80, + 0x90, 0x60, 0x54, 0x94, 0x3e, 0xaa, 0x2d, 0xd4, 0xa7, 0xb2, 0xe0, 0x8b, 0x2f, 0x61, 0xb2, 0x7b, + 0x77, 0xb3, 0x36, 0x99, 0x59, 0xee, 0x4c, 0xaa, 0xf9, 0x27, 0xfe, 0x31, 0xff, 0x8f, 0xcc, 0x47, + 0xb7, 0x91, 0x22, 0xe4, 0xed, 0x9e, 0x73, 0xee, 0x19, 0xee, 0xcc, 0x3d, 0x03, 0xef, 0xaa, 0xda, + 0x2c, 0x37, 0x8b, 0x49, 0xae, 0xd6, 0xd3, 0x42, 0xdc, 0xd6, 0x45, 0x81, 0xb2, 0xca, 0xe5, 0xb4, + 0xca, 0x35, 0x4e, 0xf5, 0x52, 0x10, 0x16, 0xd3, 0x86, 0x94, 0x51, 0x53, 0x6d, 0x14, 0xe1, 0xc4, + 0xd5, 0xac, 0x6b, 0xe5, 0xe1, 0xfb, 0xfd, 0xbd, 0x4d, 0x5d, 0x20, 0x79, 0xf3, 0xe8, 0x77, 0x17, + 0x92, 0x6b, 0x91, 0xdf, 0x88, 0x0a, 0xaf, 0x64, 0xa9, 0x18, 0x83, 0xae, 0x14, 0x6b, 0xe4, 0x51, + 0x1a, 0x8d, 0x07, 0x99, 0xab, 0x19, 0x87, 0xc3, 0xc6, 0xb7, 0xf0, 0x03, 0x47, 0xdf, 0x41, 0xf6, + 0x14, 0xfa, 0x62, 0x63, 0x96, 0x8a, 0x78, 0xc7, 0x09, 0x01, 0xb1, 0x27, 0xd0, 0xd3, 0x46, 0x90, + 0xe6, 0xdd, 0x34, 0x1a, 0xf7, 0x32, 0x0f, 0xd8, 0x10, 0x62, 0xbd, 0x95, 0xaa, 0xd1, 0xb5, 0xe6, + 0x3d, 0xd7, 0xdf, 0x62, 0x96, 0x42, 0x52, 0xa0, 0xce, 0xa9, 0x6e, 0x4c, 0xad, 0x24, 0xef, 0x3b, + 0x79, 0x97, 0x62, 0x2f, 0x20, 0x69, 0x48, 0xfd, 0xc0, 0xdc, 0xcc, 0x37, 0xb4, 0xe2, 0x87, 0xae, + 0x03, 0x02, 0xf5, 0x8d, 0x56, 0xec, 0x19, 0x0c, 0x08, 0x45, 0xb1, 0xc6, 0x79, 0x29, 0x79, 0xec, + 0xcf, 0xf7, 0xc4, 0xa5, 0x73, 0x07, 0xb1, 0x10, 0x46, 0xf0, 0x81, 0x77, 0x7b, 0xea, 0x8b, 0x30, + 0xc2, 0x5e, 0xb2, 0x5e, 0x37, 0x8a, 0x8c, 0xe6, 0x90, 0x76, 0xec, 0x25, 0x03, 0x64, 0x2f, 0xe1, + 0xc8, 0xa0, 0x36, 0xf3, 0x3b, 0x39, 0x71, 0x72, 0x62, 0xb9, 0xab, 0xd0, 0x32, 0x84, 0x18, 0x7f, + 0xd9, 0x12, 0x0b, 0x7e, 0xe4, 0xe4, 0x16, 0xb3, 0x33, 0x00, 0xc2, 0x12, 0x09, 0x65, 0x8e, 0x9a, + 0x33, 0xa7, 0xee, 0x30, 0xec, 0x03, 0x1c, 0xe7, 0x24, 0x7e, 0xae, 0x6a, 0x59, 0xcd, 0x6b, 0x59, + 0x2a, 0xfe, 0x38, 0x8d, 0xc6, 0xc9, 0x8c, 0x4d, 0xec, 0xe6, 0x26, 0x9f, 0x83, 0x64, 0x97, 0x93, + 0x1d, 0xe5, 0x3b, 0x88, 0xbd, 0x81, 0xa4, 0x54, 0xab, 0x02, 0xc9, 0xdb, 0x4e, 0x9c, 0xed, 0xd4, + 0xdb, 0x2e, 0x9d, 0xe0, 0x4c, 0x50, 0xb6, 0x35, 0x7b, 0x6d, 0x9f, 0xa8, 0x51, 0xde, 0xf0, 0xc8, + 0x19, 0x4e, 0xbc, 0x21, 0xc3, 0x46, 0xb9, 0xf6, 0x98, 0x42, 0x35, 0xba, 0x00, 0xb8, 0x46, 0xd2, + 0x4a, 0x3a, 0xeb, 0x83, 0x31, 0xa3, 0xfd, 0xc6, 0x1c, 0xfd, 0x39, 0x00, 0xb0, 0xa7, 0xeb, 0xda, + 0x28, 0xda, 0xda, 0xc8, 0x2c, 0x48, 0xc8, 0x7c, 0x19, 0x76, 0x1c, 0x10, 0x7b, 0x0e, 0x03, 0x5d, + 0x57, 0x52, 0x98, 0x0d, 0x61, 0x58, 0xee, 0x3d, 0xc1, 0xce, 0x21, 0x0e, 0x99, 0xd3, 0x3c, 0x4e, + 0x3b, 0xe3, 0x64, 0x76, 0x76, 0x3f, 0xb7, 0x3f, 0x79, 0x12, 0x62, 0xac, 0x2f, 0xa4, 0xa1, 0x6d, + 0xd6, 0xf6, 0xdb, 0xe5, 0x64, 0x21, 0x06, 0x21, 0xbf, 0x2d, 0xb6, 0xcb, 0xc9, 0xda, 0x0c, 0x84, + 0x10, 0xef, 0x30, 0xff, 0x09, 0xf2, 0x83, 0xb7, 0xe8, 0xed, 0xf7, 0x16, 0xc3, 0xaf, 0x70, 0xfc, + 0xcf, 0x94, 0xec, 0x14, 0x3a, 0x37, 0xb8, 0x0d, 0xbf, 0xcd, 0x96, 0xec, 0x15, 0xf4, 0x6e, 0xc5, + 0x6a, 0xe3, 0xbf, 0x5a, 0x32, 0x3b, 0xf6, 0x67, 0x06, 0x57, 0xe6, 0xb5, 0xf3, 0x83, 0x8f, 0xd1, + 0xa7, 0xf8, 0x7b, 0xdf, 0x4a, 0xcd, 0x62, 0xd1, 0x77, 0x5f, 0xf9, 0xed, 0xdf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xd7, 0xe5, 0xfe, 0xf4, 0x3f, 0x04, 0x00, 0x00, +} diff --git a/proto/store/store.proto b/shared/proto/store.proto similarity index 67% rename from proto/store/store.proto rename to shared/proto/store.proto index 0b9f68b..bfcd7f8 100644 --- a/proto/store/store.proto +++ b/shared/proto/store.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -package gcse.store; +package gcse; -option go_package = "stpb"; +option go_package = "gcsepb"; -import "github.com/daviddengcn/gcse/proto/spider/spider.proto"; +import "github.com/daviddengcn/gcse/shared/proto/spider.proto"; message PackageInfo { string name = 1; @@ -21,17 +21,17 @@ message PackageInfo { repeated string exported = 12; repeated string references = 18; - gcse.spider.CrawlingInfo crawling_info = 17; + CrawlingInfo crawling_info = 17; // Available if the package is not the repo's root. - gcse.spider.FolderInfo folder_info = 14; + FolderInfo folder_info = 14; // Available if the package is the repo's root. - gcse.spider.RepoInfo repo_info = 15; + RepoInfo repo_info = 15; } message PersonInfo { - gcse.spider.CrawlingInfo crawling_info = 1; + CrawlingInfo crawling_info = 1; } message Repository { @@ -39,11 +39,11 @@ message Repository { string signature = 7; // map from relative path, e.g. "proto/store", to Package - map packages = 8; + map packages = 8; string ReadmeFn = 2; // No directory info string ReadmeData = 3; // Raw content, cound be md, txt, etc. int32 stars = 4; - gcse.spider.CrawlingInfo crawling_info = 5; + CrawlingInfo crawling_info = 5; } diff --git a/shared/proto/stored.pb.go b/shared/proto/stored.pb.go new file mode 100644 index 0000000..db70b8b --- /dev/null +++ b/shared/proto/stored.pb.go @@ -0,0 +1,61 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: github.com/daviddengcn/gcse/shared/proto/stored.proto + +package gcsepb + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type PackageCrawlHistoryReq struct { + Package string `protobuf:"bytes,1,opt,name=package" json:"package,omitempty"` +} + +func (m *PackageCrawlHistoryReq) Reset() { *m = PackageCrawlHistoryReq{} } +func (m *PackageCrawlHistoryReq) String() string { return proto.CompactTextString(m) } +func (*PackageCrawlHistoryReq) ProtoMessage() {} +func (*PackageCrawlHistoryReq) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +func (m *PackageCrawlHistoryReq) GetPackage() string { + if m != nil { + return m.Package + } + return "" +} + +type PackageCrawlHistoryResp struct { +} + +func (m *PackageCrawlHistoryResp) Reset() { *m = PackageCrawlHistoryResp{} } +func (m *PackageCrawlHistoryResp) String() string { return proto.CompactTextString(m) } +func (*PackageCrawlHistoryResp) ProtoMessage() {} +func (*PackageCrawlHistoryResp) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{1} } + +func init() { + proto.RegisterType((*PackageCrawlHistoryReq)(nil), "gcse.PackageCrawlHistoryReq") + proto.RegisterType((*PackageCrawlHistoryResp)(nil), "gcse.PackageCrawlHistoryResp") +} + +func init() { + proto.RegisterFile("github.com/daviddengcn/gcse/shared/proto/stored.proto", fileDescriptor2) +} + +var fileDescriptor2 = []byte{ + // 176 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x32, 0x4d, 0xcf, 0x2c, 0xc9, + 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0x49, 0x2c, 0xcb, 0x4c, 0x49, 0x49, 0xcd, 0x4b, + 0x4f, 0xce, 0xd3, 0x4f, 0x4f, 0x2e, 0x4e, 0xd5, 0x2f, 0xce, 0x48, 0x2c, 0x4a, 0x4d, 0xd1, 0x2f, + 0x28, 0xca, 0x2f, 0xc9, 0xd7, 0x2f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4d, 0xd1, 0x03, 0x73, 0x84, 0x58, + 0x40, 0xf2, 0x4a, 0x46, 0x5c, 0x62, 0x01, 0x89, 0xc9, 0xd9, 0x89, 0xe9, 0xa9, 0xce, 0x45, 0x89, + 0xe5, 0x39, 0x1e, 0x99, 0x20, 0x35, 0x95, 0x41, 0xa9, 0x85, 0x42, 0x12, 0x5c, 0xec, 0x05, 0x10, + 0x19, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x18, 0x57, 0x49, 0x92, 0x4b, 0x1c, 0xab, 0x9e, + 0xe2, 0x02, 0xa3, 0x24, 0x2e, 0x9e, 0x60, 0x90, 0x25, 0xc1, 0xa9, 0x45, 0x65, 0x99, 0xc9, 0xa9, + 0x42, 0x41, 0x5c, 0xc2, 0x58, 0x94, 0x0a, 0xc9, 0xe8, 0x81, 0x2c, 0xd7, 0xc3, 0x6e, 0xb3, 0x94, + 0x2c, 0x1e, 0xd9, 0xe2, 0x02, 0x27, 0x8e, 0x28, 0x36, 0x90, 0x7c, 0x41, 0x52, 0x12, 0x1b, 0xd8, + 0x27, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x20, 0xcf, 0x28, 0x38, 0x02, 0x01, 0x00, 0x00, +} diff --git a/shared/proto/stored.proto b/shared/proto/stored.proto new file mode 100644 index 0000000..8c13ef2 --- /dev/null +++ b/shared/proto/stored.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package gcse; + +option go_package = "gcsepb"; + +message PackageCrawlHistoryReq { + string package = 1; +} + +message PackageCrawlHistoryResp { + // HistoryInfo info = 1; +} + +service StoreService { + rpc PackageCrawlHistory(PackageCrawlHistoryReq) returns (PackageCrawlHistoryResp); +} diff --git a/spider/filecache_test.go b/spider/filecache_test.go index e27cfb6..5f02f18 100644 --- a/spider/filecache_test.go +++ b/spider/filecache_test.go @@ -9,7 +9,7 @@ import ( "github.com/daviddengcn/bolthelper" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) func TestNullFileCache(t *testing.T) { @@ -41,7 +41,7 @@ func TestBoltFileCache(t *testing.T) { sub = "sub" subfolder = "root/sub" ) - fi := &sppb.GoFileInfo{} + fi := &gpb.GoFileInfo{} ////////////////////////////////////////////////////////////// // New file found. @@ -52,14 +52,14 @@ func TestBoltFileCache(t *testing.T) { "crawler.filecache.missed": 1, }) // Set the info. - c.Set(sign1, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore}) + c.Set(sign1, &gpb.GoFileInfo{Status: gpb.GoFileInfo_ShouldIgnore}) assert.Equal(t, "counter", counter, map[string]int{ "crawler.filecache.missed": 1, "crawler.filecache.sign_saved": 1, }) // Now, should fetch the cache assert.True(t, "c.Get", c.Get(sign1, fi)) - assert.Equal(t, "fi", fi, &sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore}) + assert.Equal(t, "fi", fi, &gpb.GoFileInfo{Status: gpb.GoFileInfo_ShouldIgnore}) assert.Equal(t, "counter", counter, map[string]int{ "crawler.filecache.missed": 1, "crawler.filecache.sign_saved": 1, diff --git a/spider/github/github.go b/spider/github/github.go index 6cd3662..6588de5 100644 --- a/spider/github/github.go +++ b/spider/github/github.go @@ -21,7 +21,7 @@ import ( "github.com/google/go-github/github" "golang.org/x/oauth2" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) var ErrInvalidPackage = errors.New("the package is not a Go package") @@ -81,7 +81,7 @@ func NewSpiderWithContents(contents map[string]string) *Spider { } type User struct { - Repos map[string]*sppb.RepoInfo + Repos map[string]*gpb.RepoInfo } func (s *Spider) waitForRate() error { @@ -104,8 +104,8 @@ func (s *Spider) waitForRate() error { // return nil } -func repoInfoFromGithub(repo *github.Repository) *sppb.RepoInfo { - ri := &sppb.RepoInfo{ +func repoInfoFromGithub(repo *github.Repository) *gpb.RepoInfo { + ri := &gpb.RepoInfo{ Description: stringsp.Get(repo.Description), Stars: int32(getInt(repo.StargazersCount)), } @@ -130,14 +130,14 @@ func (s *Spider) ReadUser(ctx context.Context, name string) (*User, error) { continue } if user.Repos == nil { - user.Repos = make(map[string]*sppb.RepoInfo) + user.Repos = make(map[string]*gpb.RepoInfo) } user.Repos[repoName] = repoInfoFromGithub(repo) } return user, nil } -func (s *Spider) ReadRepository(ctx context.Context, user, name string) (*sppb.RepoInfo, error) { +func (s *Spider) ReadRepository(ctx context.Context, user, name string) (*gpb.RepoInfo, error) { s.waitForRate() repo, _, err := s.client.Repositories.Get(ctx, user, name) if err != nil { @@ -192,11 +192,11 @@ func buildIgnored(comments []*ast.CommentGroup) bool { } var ( - goFileInfo_ShouldIgnore = sppb.GoFileInfo{Status: sppb.GoFileInfo_ShouldIgnore} - goFileInfo_ParseFailed = sppb.GoFileInfo{Status: sppb.GoFileInfo_ParseFailed} + goFileInfo_ShouldIgnore = gpb.GoFileInfo{Status: gpb.GoFileInfo_ShouldIgnore} + goFileInfo_ParseFailed = gpb.GoFileInfo{Status: gpb.GoFileInfo_ParseFailed} ) -func parseGoFile(path string, body string, info *sppb.GoFileInfo) { +func parseGoFile(path string, body string, info *gpb.GoFileInfo) { info.IsTest = strings.HasSuffix(path, "_test.go") fs := token.NewFileSet() goF, err := parser.ParseFile(fs, "", body, parser.ImportsOnly|parser.ParseComments) @@ -213,7 +213,7 @@ func parseGoFile(path string, body string, info *sppb.GoFileInfo) { *info = goFileInfo_ShouldIgnore return } - info.Status = sppb.GoFileInfo_ParseSuccess + info.Status = gpb.GoFileInfo_ParseSuccess for _, imp := range goF.Imports { p, _ := strconv.Unquote(imp.Path.Value) info.Imports = append(info.Imports, p) @@ -258,8 +258,8 @@ func isNotFound(err error) bool { return errResp.Response.StatusCode == http.StatusNotFound } -func folderInfoFromGithub(rc *github.RepositoryContent) *sppb.FolderInfo { - return &sppb.FolderInfo{ +func folderInfoFromGithub(rc *github.RepositoryContent) *gpb.FolderInfo { + return &gpb.FolderInfo{ Name: getString(rc.Name), Path: getString(rc.Path), Sha: getString(rc.SHA), @@ -278,7 +278,7 @@ type Package struct { } // Even an error is returned, the folders may still contain useful elements. -func (s *Spider) ReadPackage(ctx context.Context, user, repo, path string) (*Package, []*sppb.FolderInfo, error) { +func (s *Spider) ReadPackage(ctx context.Context, user, repo, path string) (*Package, []*gpb.FolderInfo, error) { s.waitForRate() _, cs, _, err := s.client.Repositories.GetContents(ctx, user, repo, path, nil) if err != nil { @@ -288,7 +288,7 @@ func (s *Spider) ReadPackage(ctx context.Context, user, repo, path string) (*Pac errResp, _ := errorsp.Cause(err).(*github.ErrorResponse) return nil, nil, errorsp.WithStacksAndMessage(err, "GetContents %v %v %v failed: %v", user, repo, path, errResp) } - var folders []*sppb.FolderInfo + var folders []*gpb.FolderInfo for _, c := range cs { if getString(c.Type) != "dir" { continue @@ -310,8 +310,8 @@ func (s *Spider) ReadPackage(ctx context.Context, user, repo, path string) (*Pac cPath := path + "/" + fn switch { case strings.HasSuffix(fn, ".go"): - fi, err := func() (*sppb.GoFileInfo, error) { - fi := &sppb.GoFileInfo{} + fi, err := func() (*gpb.GoFileInfo, error) { + fi := &gpb.GoFileInfo{} if s.FileCache.Get(sha, fi) { log.Printf("Cache for %v found(sha:%q)", calcFullPath(user, repo, path, fn), sha) return fi, nil @@ -334,10 +334,10 @@ func (s *Spider) ReadPackage(ctx context.Context, user, repo, path string) (*Pac if err != nil { return nil, folders, err } - if fi.Status == sppb.GoFileInfo_ParseFailed { + if fi.Status == gpb.GoFileInfo_ParseFailed { return nil, folders, errorsp.WithStacksAndMessage(ErrInvalidPackage, "fi.Status is ParseFailed") } - if fi.Status == sppb.GoFileInfo_ShouldIgnore { + if fi.Status == gpb.GoFileInfo_ShouldIgnore { continue } if fi.IsTest { @@ -424,7 +424,7 @@ func (s *Spider) getTree(ctx context.Context, owner, repo, sha string, recursive // ReadRepo reads all packages of a repository. // For pkg given to f, it will not be reused. // path in f is relative to the repository path. -func (s *Spider) ReadRepo(ctx context.Context, user, repo, sha string, f func(path string, pkg *sppb.Package) error) error { +func (s *Spider) ReadRepo(ctx context.Context, user, repo, sha string, f func(path string, pkg *gpb.Package) error) error { tree, err := s.getTree(ctx, user, repo, sha, true) if err != nil { return err @@ -448,7 +448,7 @@ func (s *Spider) ReadRepo(ctx context.Context, user, repo, sha string, f func(pa } log.Printf("pkgs: %v", pkgs) for d, teList := range pkgs { - pkg := sppb.Package{ + pkg := gpb.Package{ Path: d, } var imports stringsp.Set @@ -459,8 +459,8 @@ func (s *Spider) ReadRepo(ctx context.Context, user, repo, sha string, f func(pa sha := *te.SHA switch { case strings.HasSuffix(fn, ".go"): - fi, err := func() (*sppb.GoFileInfo, error) { - fi := &sppb.GoFileInfo{} + fi, err := func() (*gpb.GoFileInfo, error) { + fi := &gpb.GoFileInfo{} if s.FileCache.Get(sha, fi) { log.Printf("Cache for %v found(sha:%q)", "github.com/"+user+"/"+cPath, sha) return fi, nil @@ -483,10 +483,10 @@ func (s *Spider) ReadRepo(ctx context.Context, user, repo, sha string, f func(pa if err != nil { return err } - if fi.Status == sppb.GoFileInfo_ParseFailed { + if fi.Status == gpb.GoFileInfo_ParseFailed { return errorsp.WithStacksAndMessage(ErrInvalidPackage, "fi.Status is ParseFailed") } - if fi.Status == sppb.GoFileInfo_ShouldIgnore { + if fi.Status == gpb.GoFileInfo_ShouldIgnore { continue } if fi.IsTest { diff --git a/spider/ranking.go b/spider/ranking.go index f1f8a8f..a48fdf9 100644 --- a/spider/ranking.go +++ b/spider/ranking.go @@ -9,8 +9,7 @@ import ( "github.com/golangplus/strings" "github.com/golangplus/time" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) const ( @@ -80,7 +79,7 @@ func (s PackageStatus) String() string { return "-" } -func repoInfoAvailable(info *sppb.RepoInfo) bool { +func repoInfoAvailable(info *gpb.RepoInfo) bool { if info == nil { return false } @@ -88,7 +87,7 @@ func repoInfoAvailable(info *sppb.RepoInfo) bool { return t.After(time.Now().Add(-maxRepoInfoDue)) } -func folderInfoAvailable(info *sppb.FolderInfo) bool { +func folderInfoAvailable(info *gpb.FolderInfo) bool { if info == nil { return false } @@ -96,7 +95,7 @@ func folderInfoAvailable(info *sppb.FolderInfo) bool { return t.After(time.Now().Add(-maxFolderInfoDue)) } -func CheckPackageStatus(pkg *stpb.PackageInfo, repo *sppb.RepoInfo) PackageStatus { +func CheckPackageStatus(pkg *gpb.PackageInfo, repo *gpb.RepoInfo) PackageStatus { if pkg.CrawlingInfo == nil { return OutOfDate } diff --git a/spider/ranking_test.go b/spider/ranking_test.go index e6becb5..53481d3 100644 --- a/spider/ranking_test.go +++ b/spider/ranking_test.go @@ -9,8 +9,7 @@ import ( "github.com/golangplus/testing/assert" "github.com/golangplus/time" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) func TestLikeGoSubFolder(t *testing.T) { @@ -30,26 +29,26 @@ func TestLikeGoSubFolder(t *testing.T) { func TestCheckPackageStatus(t *testing.T) { // No crawling info, new package - assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{}, nil), OutOfDate) + assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&gpb.PackageInfo{}, nil), OutOfDate) pkgCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-5 * timep.Day)) newRepoInfoCrawlTime, _ := ptypes.TimestampProto(time.Now().Add(-3 * timep.Day)) newPkgUpdateTime, _ := ptypes.TimestampProto(time.Now().Add(-4 * timep.Day)) - assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{ - CrawlingInfo: &sppb.CrawlingInfo{ + assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&gpb.PackageInfo{ + CrawlingInfo: &gpb.CrawlingInfo{ CrawlingTime: pkgCrawlTime, }, - }, &sppb.RepoInfo{ + }, &gpb.RepoInfo{ CrawlingTime: newRepoInfoCrawlTime, LastUpdated: newPkgUpdateTime, }), OutOfDate) newPkgUpdateTime, _ = ptypes.TimestampProto(time.Now().Add(-6 * timep.Day)) - assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&stpb.PackageInfo{ - CrawlingInfo: &sppb.CrawlingInfo{ + assert.Equal(t, "CheckPackageStatus", CheckPackageStatus(&gpb.PackageInfo{ + CrawlingInfo: &gpb.CrawlingInfo{ CrawlingTime: pkgCrawlTime, }, - }, &sppb.RepoInfo{ + }, &gpb.RepoInfo{ CrawlingTime: newRepoInfoCrawlTime, LastUpdated: newPkgUpdateTime, }), UpToDate) diff --git a/store/history.go b/store/history.go index b5754ea..c62a525 100644 --- a/store/history.go +++ b/store/history.go @@ -11,7 +11,7 @@ import ( "github.com/daviddengcn/bolthelper" - sppb "github.com/daviddengcn/gcse/proto/spider" + gpb "github.com/daviddengcn/gcse/shared/proto" ) func SaveSnapshot(path string) error { @@ -24,13 +24,13 @@ const ( maxHistoryEvents = 10 ) -func readHistoryOf(box *bh.RefCountBox, root []byte, site, idOrPath string) (*sppb.HistoryInfo, error) { - info := &sppb.HistoryInfo{} +func readHistoryOf(box *bh.RefCountBox, root []byte, site, idOrPath string) (*gpb.HistoryInfo, error) { + info := &gpb.HistoryInfo{} if err := box.View(func(tx bh.Tx) error { return tx.Value([][]byte{historyRoot, root, []byte(site), []byte(idOrPath)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes failed", len(bs)); err != nil { log.Printf("Unmarshal failed: %v", err) - *info = sppb.HistoryInfo{} + *info = gpb.HistoryInfo{} } return nil }) @@ -40,34 +40,34 @@ func readHistoryOf(box *bh.RefCountBox, root []byte, site, idOrPath string) (*sp return info, nil } -func readHistory(root []byte, site, idOrPath string) (*sppb.HistoryInfo, error) { +func readHistory(root []byte, site, idOrPath string) (*gpb.HistoryInfo, error) { return readHistoryOf(box, root, site, idOrPath) } -func ReadPackageHistory(site, path string) (*sppb.HistoryInfo, error) { +func ReadPackageHistory(site, path string) (*gpb.HistoryInfo, error) { return readHistory(pkgsRoot, site, path) } -func ReadPackageHistoryOf(box *bh.RefCountBox, site, path string) (*sppb.HistoryInfo, error) { +func ReadPackageHistoryOf(box *bh.RefCountBox, site, path string) (*gpb.HistoryInfo, error) { return readHistoryOf(box, pkgsRoot, site, path) } -func ReadPersonHistory(site, path string) (*sppb.HistoryInfo, error) { +func ReadPersonHistory(site, path string) (*gpb.HistoryInfo, error) { return readHistory(personsRoot, site, path) } -func updateHistory(root []byte, site, idOrPath string, f func(*sppb.HistoryInfo) error) error { +func updateHistory(root []byte, site, idOrPath string, f func(*gpb.HistoryInfo) error) error { return box.Update(func(tx bh.Tx) error { b, err := tx.CreateBucketIfNotExists([][]byte{historyRoot, root, []byte(site)}) if err != nil { return err } - info := &sppb.HistoryInfo{} + info := &gpb.HistoryInfo{} if err := b.Value([][]byte{[]byte(idOrPath)}, func(bs bytesp.Slice) error { err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes", len(bs)) if err != nil { log.Printf("Unmarshaling failed: %v", err) - *info = sppb.HistoryInfo{} + *info = gpb.HistoryInfo{} } return nil }); err != nil { @@ -84,23 +84,23 @@ func updateHistory(root []byte, site, idOrPath string, f func(*sppb.HistoryInfo) }) } -func UpdatePackageHistory(site, path string, f func(*sppb.HistoryInfo) error) error { +func UpdatePackageHistory(site, path string, f func(*gpb.HistoryInfo) error) error { return updateHistory(pkgsRoot, site, path, f) } -func AppendPackageEvent(site, path, foundWay string, t time.Time, a sppb.HistoryEvent_Action_Enum) error { - return UpdatePackageHistory(site, path, func(hi *sppb.HistoryInfo) error { +func AppendPackageEvent(site, path, foundWay string, t time.Time, a gpb.HistoryEvent_Action_Enum) error { + return UpdatePackageHistory(site, path, func(hi *gpb.HistoryInfo) error { if hi.FoundTime == nil { // The first time the package was found hi.FoundTime, _ = ptypes.TimestampProto(t) hi.FoundWay = foundWay } - if a == sppb.HistoryEvent_Action_None { + if a == gpb.HistoryEvent_Action_None { return nil } // Insert the event tsp, _ := ptypes.TimestampProto(t) - hi.Events = append([]*sppb.HistoryEvent{{ + hi.Events = append([]*gpb.HistoryEvent{{ Action: a, Timestamp: tsp, }}, hi.Events...) @@ -108,16 +108,16 @@ func AppendPackageEvent(site, path, foundWay string, t time.Time, a sppb.History hi.Events = hi.Events[:maxHistoryEvents] } switch a { - case sppb.HistoryEvent_Action_Success: + case gpb.HistoryEvent_Action_Success: hi.LatestSuccess = tsp - case sppb.HistoryEvent_Action_Failed: + case gpb.HistoryEvent_Action_Failed: hi.LatestFailed = tsp } return nil }) } -func UpdatePersonHistory(site, path string, f func(*sppb.HistoryInfo) error) error { +func UpdatePersonHistory(site, path string, f func(*gpb.HistoryInfo) error) error { return updateHistory(personsRoot, site, path, f) } diff --git a/store/history_test.go b/store/history_test.go index e9a00e3..4ff12c8 100644 --- a/store/history_test.go +++ b/store/history_test.go @@ -9,8 +9,9 @@ import ( "github.com/golangplus/testing/assert" "github.com/daviddengcn/bolthelper" - sppb "github.com/daviddengcn/gcse/proto/spider" "github.com/daviddengcn/go-villa" + + gpb "github.com/daviddengcn/gcse/shared/proto" ) func TestUpdateReadDeletePackageHistory(t *testing.T) { @@ -19,20 +20,20 @@ func TestUpdateReadDeletePackageHistory(t *testing.T) { path = "gcse" foundWay = "testing" ) - assert.NoError(t, UpdatePackageHistory(site, path, func(info *sppb.HistoryInfo) error { - assert.Equal(t, "info", info, &sppb.HistoryInfo{}) + assert.NoError(t, UpdatePackageHistory(site, path, func(info *gpb.HistoryInfo) error { + assert.Equal(t, "info", info, &gpb.HistoryInfo{}) info.FoundWay = foundWay return nil })) h, err := ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{FoundWay: foundWay}) assert.NoError(t, DeletePackageHistory(site, path)) h, err = ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{}) } func TestAppendPackageEvent(t *testing.T) { @@ -44,41 +45,41 @@ func TestAppendPackageEvent(t *testing.T) { // Insert a found only event, no action. foundTm := time.Now() foundTs, _ := ptypes.TimestampProto(foundTm) - assert.NoError(t, AppendPackageEvent(site, path, "test", foundTm, sppb.HistoryEvent_Action_None)) + assert.NoError(t, AppendPackageEvent(site, path, "test", foundTm, gpb.HistoryEvent_Action_None)) h, err := ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay, FoundTime: foundTs}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{FoundWay: foundWay, FoundTime: foundTs}) // Inser a Success action succTm := foundTm.Add(time.Hour) succTs, _ := ptypes.TimestampProto(succTm) - assert.NoError(t, AppendPackageEvent(site, path, "non-test", succTm, sppb.HistoryEvent_Action_Success)) + assert.NoError(t, AppendPackageEvent(site, path, "non-test", succTm, gpb.HistoryEvent_Action_Success)) h, err = ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{ + assert.Equal(t, "h", h, &gpb.HistoryInfo{ FoundWay: foundWay, FoundTime: foundTs, - Events: []*sppb.HistoryEvent{{ + Events: []*gpb.HistoryEvent{{ Timestamp: succTs, - Action: sppb.HistoryEvent_Action_Success, + Action: gpb.HistoryEvent_Action_Success, }}, LatestSuccess: succTs, }) // Inser a Failed action failedTm := succTm.Add(time.Hour) failedTs, _ := ptypes.TimestampProto(failedTm) - assert.NoError(t, AppendPackageEvent(site, path, "", failedTm, sppb.HistoryEvent_Action_Failed)) + assert.NoError(t, AppendPackageEvent(site, path, "", failedTm, gpb.HistoryEvent_Action_Failed)) h, err = ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{ + assert.Equal(t, "h", h, &gpb.HistoryInfo{ FoundWay: foundWay, FoundTime: foundTs, - Events: []*sppb.HistoryEvent{{ + Events: []*gpb.HistoryEvent{{ Timestamp: failedTs, - Action: sppb.HistoryEvent_Action_Failed, + Action: gpb.HistoryEvent_Action_Failed, }, { Timestamp: succTs, - Action: sppb.HistoryEvent_Action_Success, + Action: gpb.HistoryEvent_Action_Success, }}, LatestSuccess: succTs, LatestFailed: failedTs, @@ -91,20 +92,20 @@ func TestUpdateReadDeletePersonHistory(t *testing.T) { id = "daviddengcn" foundWay = "testing" ) - assert.NoError(t, UpdatePersonHistory(site, id, func(info *sppb.HistoryInfo) error { - assert.Equal(t, "info", info, &sppb.HistoryInfo{}) + assert.NoError(t, UpdatePersonHistory(site, id, func(info *gpb.HistoryInfo) error { + assert.Equal(t, "info", info, &gpb.HistoryInfo{}) info.FoundWay = foundWay return nil })) h, err := ReadPersonHistory(site, id) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{FoundWay: foundWay}) assert.NoError(t, DeletePersonHistory(site, id)) h, err = ReadPersonHistory(site, id) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{}) } func TestSaveSnapshot(t *testing.T) { @@ -113,14 +114,14 @@ func TestSaveSnapshot(t *testing.T) { path = "gcse" foundWay = "testing" ) - assert.NoError(t, UpdatePackageHistory(site, path, func(info *sppb.HistoryInfo) error { - assert.Equal(t, "info", info, &sppb.HistoryInfo{}) + assert.NoError(t, UpdatePackageHistory(site, path, func(info *gpb.HistoryInfo) error { + assert.Equal(t, "info", info, &gpb.HistoryInfo{}) info.FoundWay = foundWay return nil })) h, err := ReadPackageHistory(site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{FoundWay: foundWay}) outPath := villa.Path(os.TempDir()).Join("TestSaveSnapshot").S() assert.NoError(t, SaveSnapshot(outPath)) @@ -131,5 +132,5 @@ func TestSaveSnapshot(t *testing.T) { } h, err = ReadPackageHistoryOf(box, site, path) assert.NoError(t, err) - assert.Equal(t, "h", h, &sppb.HistoryInfo{FoundWay: foundWay}) + assert.Equal(t, "h", h, &gpb.HistoryInfo{FoundWay: foundWay}) } diff --git a/store/repo.go b/store/repo.go index 16f8a4e..2bdbf0a 100644 --- a/store/repo.go +++ b/store/repo.go @@ -8,17 +8,18 @@ import ( "github.com/golangplus/errors" "github.com/daviddengcn/bolthelper" - "github.com/daviddengcn/gcse/proto/store" + + gpb "github.com/daviddengcn/gcse/shared/proto" ) // Returns an empty (non-nil) PackageInfo if not found. -func ReadRepository(site, user, repo string) (*stpb.Repository, error) { - doc := &stpb.Repository{} +func ReadRepository(site, user, repo string) (*gpb.Repository, error) { + doc := &gpb.Repository{} if err := box.View(func(tx bh.Tx) error { return tx.Value([][]byte{reposRoot, []byte(site), []byte(user), []byte(repo)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, doc), "Unmarshal %d bytes failed", len(bs)); err != nil { log.Printf("Unmarshal failed: %v", err) - *doc = stpb.Repository{} + *doc = gpb.Repository{} } return nil }) @@ -28,17 +29,17 @@ func ReadRepository(site, user, repo string) (*stpb.Repository, error) { return doc, nil } -func UpdateRepository(site, user, repo string, f func(doc *stpb.Repository) error) error { +func UpdateRepository(site, user, repo string, f func(doc *gpb.Repository) error) error { return box.Update(func(tx bh.Tx) error { b, err := tx.CreateBucketIfNotExists([][]byte{reposRoot, []byte(site), []byte(user)}) if err != nil { return err } - doc := &stpb.Repository{} + doc := &gpb.Repository{} if err := b.Value([][]byte{[]byte(repo)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, doc), "Unmarshal %d bytes", len(bs)); err != nil { log.Printf("Unmarshaling failed: %v", err) - *doc = stpb.Repository{} + *doc = gpb.Repository{} } return nil }); err != nil { @@ -73,7 +74,7 @@ func ForEachRepositorySite(f func(string) error) error { }) } -func ForEachRepositoryOfSite(site string, f func(user, name string, doc *stpb.Repository) error) error { +func ForEachRepositoryOfSite(site string, f func(user, name string, doc *gpb.Repository) error) error { return box.View(func(tx bh.Tx) error { return tx.ForEach([][]byte{reposRoot, []byte(site)}, func(b bh.Bucket, user, v bytesp.Slice) error { if v != nil { @@ -85,7 +86,7 @@ func ForEachRepositoryOfSite(site string, f func(user, name string, doc *stpb.Re log.Printf("Unexpected nil value for key %q, ignored", string(name)) return nil } - doc := &stpb.Repository{} + doc := &gpb.Repository{} if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, doc), "Unmarshal %d bytes", len(bs)); err != nil { log.Printf("Unmarshaling value for %v failed, ignored: %v", name, err) return nil diff --git a/store/repo_test.go b/store/repo_test.go index 8aad71c..91cea5a 100644 --- a/store/repo_test.go +++ b/store/repo_test.go @@ -5,7 +5,7 @@ import ( "github.com/golangplus/testing/assert" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) func TestUpdateReadDeleteRepository(t *testing.T) { @@ -14,20 +14,20 @@ func TestUpdateReadDeleteRepository(t *testing.T) { user = "daviddengcn" repo = "gcse" ) - assert.NoError(t, UpdateRepository(site, user, repo, func(doc *stpb.Repository) error { - assert.Equal(t, "doc", doc, &stpb.Repository{}) + assert.NoError(t, UpdateRepository(site, user, repo, func(doc *gpb.Repository) error { + assert.Equal(t, "doc", doc, &gpb.Repository{}) doc.Stars = 10 return nil })) r, err := ReadRepository(site, user, repo) assert.NoError(t, err) - assert.Equal(t, "r", r, &stpb.Repository{Stars: 10}) + assert.Equal(t, "r", r, &gpb.Repository{Stars: 10}) assert.NoError(t, DeleteRepository(site, user, repo)) r, err = ReadRepository(site, user, repo) assert.NoError(t, err) - assert.Equal(t, "r", r, &stpb.Repository{}) + assert.Equal(t, "r", r, &gpb.Repository{}) } func TestForEachRepositorySite(t *testing.T) { @@ -38,7 +38,7 @@ func TestForEachRepositorySite(t *testing.T) { user = "daviddengcn" repo = "gcse" ) - assert.NoError(t, UpdateRepository(site, user, repo, func(doc *stpb.Repository) error { + assert.NoError(t, UpdateRepository(site, user, repo, func(doc *gpb.Repository) error { return nil })) var sites []string @@ -55,14 +55,14 @@ func TestForEachRepositoryOfSite(t *testing.T) { user = "daviddengcn" repo = "gcse" ) - assert.NoError(t, UpdateRepository(site, user, repo, func(doc *stpb.Repository) error { + assert.NoError(t, UpdateRepository(site, user, repo, func(doc *gpb.Repository) error { doc.ReadmeData = "hello" return nil })) - assert.NoError(t, ForEachRepositoryOfSite(site, func(u, r string, doc *stpb.Repository) error { + assert.NoError(t, ForEachRepositoryOfSite(site, func(u, r string, doc *gpb.Repository) error { assert.Equal(t, "user", u, user) assert.Equal(t, "repo", r, repo) - assert.Equal(t, "doc", doc, &stpb.Repository{ReadmeData: "hello"}) + assert.Equal(t, "doc", doc, &gpb.Repository{ReadmeData: "hello"}) return nil })) } diff --git a/store/store.go b/store/store.go index 6b42246..50af9b7 100644 --- a/store/store.go +++ b/store/store.go @@ -13,8 +13,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) var ( @@ -43,7 +42,7 @@ var box = &bh.RefCountBox{ DataPath: configs.StoreBoltPath, } -func RepoInfoAge(r *sppb.RepoInfo) time.Duration { +func RepoInfoAge(r *gpb.RepoInfo) time.Duration { t, _ := ptypes.Timestamp(r.CrawlingTime) return time.Now().Sub(t) } @@ -61,14 +60,14 @@ func ForEachPackageSite(f func(string) error) error { }) } -func ForEachPackageOfSite(site string, f func(string, *stpb.PackageInfo) error) error { +func ForEachPackageOfSite(site string, f func(string, *gpb.PackageInfo) error) error { return box.View(func(tx bh.Tx) error { return tx.ForEach([][]byte{pkgsRoot, []byte(site)}, func(_ bh.Bucket, k, v bytesp.Slice) error { if v == nil { log.Printf("Unexpected nil value for key %q, ignored", string(k)) return nil } - info := &stpb.PackageInfo{} + info := &gpb.PackageInfo{} if err := errorsp.WithStacksAndMessage(proto.Unmarshal(v, info), "Unmarshal %d bytes failed", len(v)); err != nil { log.Printf("Unmarshal failed: %v, ignored", err) return nil @@ -79,13 +78,13 @@ func ForEachPackageOfSite(site string, f func(string, *stpb.PackageInfo) error) } // Returns an empty (non-nil) PackageInfo if not found. -func ReadPackage(site, path string) (*stpb.PackageInfo, error) { - info := &stpb.PackageInfo{} +func ReadPackage(site, path string) (*gpb.PackageInfo, error) { + info := &gpb.PackageInfo{} if err := box.View(func(tx bh.Tx) error { return tx.Value([][]byte{pkgsRoot, []byte(site), []byte(path)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes failed", len(bs)); err != nil { log.Printf("Unmarshal failed: %v", err) - *info = stpb.PackageInfo{} + *info = gpb.PackageInfo{} } return nil }) @@ -95,17 +94,17 @@ func ReadPackage(site, path string) (*stpb.PackageInfo, error) { return info, nil } -func UpdatePackage(site, path string, f func(*stpb.PackageInfo) error) error { +func UpdatePackage(site, path string, f func(*gpb.PackageInfo) error) error { return box.Update(func(tx bh.Tx) error { b, err := tx.CreateBucketIfNotExists([][]byte{pkgsRoot, []byte(site)}) if err != nil { return err } - info := &stpb.PackageInfo{} + info := &gpb.PackageInfo{} if err := b.Value([][]byte{[]byte(path)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes", len(bs)); err != nil { log.Printf("Unmarshaling failed: %v", err) - *info = stpb.PackageInfo{} + *info = gpb.PackageInfo{} } return nil }); err != nil { @@ -128,13 +127,13 @@ func DeletePackage(site, path string) error { }) } -func ReadPerson(site, id string) (*stpb.PersonInfo, error) { - info := &stpb.PersonInfo{} +func ReadPerson(site, id string) (*gpb.PersonInfo, error) { + info := &gpb.PersonInfo{} if err := box.View(func(tx bh.Tx) error { return tx.Value([][]byte{personsRoot, []byte(site), []byte(id)}, func(bs bytesp.Slice) error { if err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes failed", len(bs)); err != nil { log.Printf("Unmarshal failed: %v", err) - *info = stpb.PersonInfo{} + *info = gpb.PersonInfo{} } return nil }) @@ -144,18 +143,18 @@ func ReadPerson(site, id string) (*stpb.PersonInfo, error) { return info, nil } -func UpdatePerson(site, id string, f func(*stpb.PersonInfo) error) error { +func UpdatePerson(site, id string, f func(*gpb.PersonInfo) error) error { return box.Update(func(tx bh.Tx) error { b, err := tx.CreateBucketIfNotExists([][]byte{personsRoot, []byte(site)}) if err != nil { return err } - info := &stpb.PersonInfo{} + info := &gpb.PersonInfo{} if err := b.Value([][]byte{[]byte(id)}, func(bs bytesp.Slice) error { err := errorsp.WithStacksAndMessage(proto.Unmarshal(bs, info), "Unmarshal %d bytes", len(bs)) if err != nil { log.Printf("Unmarshaling failed: %v", err) - *info = stpb.PersonInfo{} + *info = gpb.PersonInfo{} } return nil }); err != nil { diff --git a/store/store_test.go b/store/store_test.go index 6164694..984f925 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -10,8 +10,7 @@ import ( "github.com/daviddengcn/gcse/configs" - sppb "github.com/daviddengcn/gcse/proto/spider" - stpb "github.com/daviddengcn/gcse/proto/store" + gpb "github.com/daviddengcn/gcse/shared/proto" ) func init() { @@ -24,7 +23,7 @@ func cleanDatabase(t *testing.T) { func TestRepoInfoAge(t *testing.T) { ts, _ := ptypes.TimestampProto(time.Now().Add(-time.Hour)) - age := RepoInfoAge(&sppb.RepoInfo{ + age := RepoInfoAge(&gpb.RepoInfo{ CrawlingTime: ts, }) assert.ValueShould(t, "age", age, age >= time.Hour && age < time.Hour+time.Minute, "age out of expected range") @@ -39,10 +38,10 @@ func TestForEachPackageSite(t *testing.T) { path = "gcse" name = "pkgname" ) - assert.NoError(t, UpdatePackage(site1, path, func(info *stpb.PackageInfo) error { + assert.NoError(t, UpdatePackage(site1, path, func(info *gpb.PackageInfo) error { return nil })) - assert.NoError(t, UpdatePackage(site2, path, func(info *stpb.PackageInfo) error { + assert.NoError(t, UpdatePackage(site2, path, func(info *gpb.PackageInfo) error { return nil })) var sites []string @@ -63,16 +62,16 @@ func TestForEachPackageOfSite(t *testing.T) { path2 = "gcse2" name2 = "TestForEachPackageOfSite" ) - assert.NoError(t, UpdatePackage(site, path1, func(info *stpb.PackageInfo) error { + assert.NoError(t, UpdatePackage(site, path1, func(info *gpb.PackageInfo) error { info.Name = name1 return nil })) - assert.NoError(t, UpdatePackage(site, path2, func(info *stpb.PackageInfo) error { + assert.NoError(t, UpdatePackage(site, path2, func(info *gpb.PackageInfo) error { info.Name = name2 return nil })) var paths, names []string - assert.NoError(t, ForEachPackageOfSite(site, func(path string, info *stpb.PackageInfo) error { + assert.NoError(t, ForEachPackageOfSite(site, func(path string, info *gpb.PackageInfo) error { paths = append(paths, path) names = append(names, info.Name) return nil @@ -87,20 +86,20 @@ func TestUpdateReadDeletePackage(t *testing.T) { path = "gcse" name = "pkgname" ) - assert.NoError(t, UpdatePackage(site, path, func(info *stpb.PackageInfo) error { - assert.Equal(t, "info", info, &stpb.PackageInfo{}) + assert.NoError(t, UpdatePackage(site, path, func(info *gpb.PackageInfo) error { + assert.Equal(t, "info", info, &gpb.PackageInfo{}) info.Name = name return nil })) pkg, err := ReadPackage(site, path) assert.NoError(t, err) - assert.Equal(t, "pkg", pkg, &stpb.PackageInfo{Name: name}) + assert.Equal(t, "pkg", pkg, &gpb.PackageInfo{Name: name}) assert.NoError(t, DeletePackage(site, path)) pkg, err = ReadPackage(site, path) assert.NoError(t, err) - assert.Equal(t, "pkg", pkg, &stpb.PackageInfo{}) + assert.Equal(t, "pkg", pkg, &gpb.PackageInfo{}) } func TestUpdateReadDeletePerson(t *testing.T) { @@ -109,20 +108,20 @@ func TestUpdateReadDeletePerson(t *testing.T) { id = "daviddengcn" etag = "tag" ) - assert.NoError(t, UpdatePerson(site, id, func(info *stpb.PersonInfo) error { - assert.Equal(t, "info", info, &stpb.PersonInfo{}) - info.CrawlingInfo = &sppb.CrawlingInfo{ + assert.NoError(t, UpdatePerson(site, id, func(info *gpb.PersonInfo) error { + assert.Equal(t, "info", info, &gpb.PersonInfo{}) + info.CrawlingInfo = &gpb.CrawlingInfo{ Etag: etag, } return nil })) p, err := ReadPerson(site, id) assert.NoError(t, err) - assert.Equal(t, "p", p, &stpb.PersonInfo{CrawlingInfo: &sppb.CrawlingInfo{Etag: etag}}) + assert.Equal(t, "p", p, &gpb.PersonInfo{CrawlingInfo: &gpb.CrawlingInfo{Etag: etag}}) assert.NoError(t, DeletePerson(site, id)) p, err = ReadPerson(site, id) assert.NoError(t, err) - assert.Equal(t, "p", p, &stpb.PersonInfo{}) + assert.Equal(t, "p", p, &gpb.PersonInfo{}) } diff --git a/stored/stored.go b/stored/stored.go new file mode 100644 index 0000000..d64c814 --- /dev/null +++ b/stored/stored.go @@ -0,0 +1,10 @@ +package main + +import ( + "flag" +) + +func main() { + port := flag.Int("port", 8081, "port") + +}