From e22aa0630b61053a013967b05be75dd3856cea12 Mon Sep 17 00:00:00 2001 From: Haruue Icymoon Date: Wed, 21 Feb 2024 16:21:49 +0800 Subject: [PATCH 1/3] fix: geo db load fail after download error now geo db are downloaded to a temp file, have a integrity check by a loading test, and then moved to where it should be. fix: #944 --- app/internal/utils/geoloader.go | 40 +++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/app/internal/utils/geoloader.go b/app/internal/utils/geoloader.go index 56cb205e4f..051773124c 100644 --- a/app/internal/utils/geoloader.go +++ b/app/internal/utils/geoloader.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "io" "net/http" "os" @@ -15,6 +16,7 @@ const ( geoipURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat" geositeFilename = "geosite.dat" geositeURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat" + geoDlTmpPattern = ".hysteria-geoloader.dlpart.*" geoDefaultUpdateInterval = 7 * 24 * time.Hour // 7 days ) @@ -49,7 +51,7 @@ func (l *GeoLoader) shouldDownload(filename string) bool { } } -func (l *GeoLoader) download(filename, url string) error { +func (l *GeoLoader) downloadAndCheck(filename, url string, checkFunc func(filename string) error) error { l.DownloadFunc(filename, url) resp, err := http.Get(url) @@ -59,16 +61,34 @@ func (l *GeoLoader) download(filename, url string) error { } defer resp.Body.Close() - f, err := os.Create(filename) + f, err := os.CreateTemp(".", geoDlTmpPattern) if err != nil { l.DownloadErrFunc(err) return err } - defer f.Close() + defer os.Remove(f.Name()) _, err = io.Copy(f, resp.Body) - l.DownloadErrFunc(err) - return err + if err != nil { + f.Close() + l.DownloadErrFunc(err) + return err + } + f.Close() + + err = checkFunc(f.Name()) + if err != nil { + l.DownloadErrFunc(fmt.Errorf("integrity check failed: %w", err)) + return err + } + + err = os.Rename(f.Name(), filename) + if err != nil { + l.DownloadErrFunc(fmt.Errorf("rename failed: %w", err)) + return err + } + + return nil } func (l *GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) { @@ -82,7 +102,10 @@ func (l *GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) { filename = geoipFilename } if autoDL && l.shouldDownload(filename) { - err := l.download(filename, geoipURL) + err := l.downloadAndCheck(filename, geoipURL, func(filename string) error { + _, err := v2geo.LoadGeoIP(filename) + return err + }) if err != nil { return nil, err } @@ -106,7 +129,10 @@ func (l *GeoLoader) LoadGeoSite() (map[string]*v2geo.GeoSite, error) { filename = geositeFilename } if autoDL && l.shouldDownload(filename) { - err := l.download(filename, geositeURL) + err := l.downloadAndCheck(filename, geositeURL, func(filename string) error { + _, err := v2geo.LoadGeoSite(filename) + return err + }) if err != nil { return nil, err } From 6dea0adb19be7ecf4fd3edfbcc9e8288bcc7fb30 Mon Sep 17 00:00:00 2001 From: Haruue Icymoon Date: Wed, 21 Feb 2024 17:22:45 +0800 Subject: [PATCH 2/3] feat: re-download geo db when autoDL && load fail --- app/internal/utils/geoloader.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/app/internal/utils/geoloader.go b/app/internal/utils/geoloader.go index 051773124c..328be68a44 100644 --- a/app/internal/utils/geoloader.go +++ b/app/internal/utils/geoloader.go @@ -43,6 +43,10 @@ func (l *GeoLoader) shouldDownload(filename string) bool { if os.IsNotExist(err) { return true } + if info.Size() == 0 { + // empty files are loadable by v2geo, but we consider it broken + return true + } dt := time.Now().Sub(info.ModTime()) if l.UpdateInterval == 0 { return dt > geoDefaultUpdateInterval @@ -101,7 +105,15 @@ func (l *GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) { autoDL = true filename = geoipFilename } - if autoDL && l.shouldDownload(filename) { + if autoDL { + if !l.shouldDownload(filename) { + m, err := v2geo.LoadGeoIP(filename) + if err == nil { + l.geoipMap = m + return m, nil + } + // file is broken, download it again + } err := l.downloadAndCheck(filename, geoipURL, func(filename string) error { _, err := v2geo.LoadGeoIP(filename) return err @@ -128,7 +140,15 @@ func (l *GeoLoader) LoadGeoSite() (map[string]*v2geo.GeoSite, error) { autoDL = true filename = geositeFilename } - if autoDL && l.shouldDownload(filename) { + if autoDL { + if !l.shouldDownload(filename) { + m, err := v2geo.LoadGeoSite(filename) + if err == nil { + l.geositeMap = m + return m, nil + } + // file is broken, download it again + } err := l.downloadAndCheck(filename, geositeURL, func(filename string) error { _, err := v2geo.LoadGeoSite(filename) return err From 842b0ab3f7de28f52bf7548c8f29ff5df5304185 Mon Sep 17 00:00:00 2001 From: Haruue Icymoon Date: Thu, 22 Feb 2024 01:32:10 +0800 Subject: [PATCH 3/3] feat: load previous download when download fail --- app/internal/utils/geoloader.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/internal/utils/geoloader.go b/app/internal/utils/geoloader.go index 328be68a44..fa9775ba71 100644 --- a/app/internal/utils/geoloader.go +++ b/app/internal/utils/geoloader.go @@ -119,7 +119,10 @@ func (l *GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) { return err }) if err != nil { - return nil, err + // as long as the previous download exists, fallback to it + if _, serr := os.Stat(filename); os.IsNotExist(serr) { + return nil, err + } } } m, err := v2geo.LoadGeoIP(filename) @@ -154,7 +157,10 @@ func (l *GeoLoader) LoadGeoSite() (map[string]*v2geo.GeoSite, error) { return err }) if err != nil { - return nil, err + // as long as the previous download exists, fallback to it + if _, serr := os.Stat(filename); os.IsNotExist(serr) { + return nil, err + } } } m, err := v2geo.LoadGeoSite(filename)