diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 46e76ac..3ee0f0a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,6 +28,11 @@ jobs: with: name: ctrf-report path: ctrf-report.json + - name: Publish Test Report + uses: ctrf-io/github-test-reporter@v1 + with: + report-path: 'ctrf-report.json' + if: always() - name: Publish Test Summary Results run: npx github-actions-ctrf ctrf-report.json - name: Install goveralls diff --git a/cache_test.go b/cache_test.go index 852deb5..635c65f 100644 --- a/cache_test.go +++ b/cache_test.go @@ -22,8 +22,8 @@ func TestCaching(t *testing.T) { cloudsackID := "cloudsack" providers := map[string]*Provider{ - testProviderID: NewProvider(nil, "", nil, nil, nil, nil, nil, ""), - cloudsackID: NewProvider(nil, "", nil, nil, nil, nil, nil, ""), + testProviderID: NewProvider(nil, "", nil, nil, nil, nil, ""), + cloudsackID: NewProvider(nil, "", nil, nil, nil, nil, ""), } log.Debug("Creating fronted") diff --git a/front.go b/front.go index 8cb285b..e55e5f0 100644 --- a/front.go +++ b/front.go @@ -252,7 +252,7 @@ type SNIConfig struct { } // Create a Provider with the given details -func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, validator ResponseValidator, passthrough []string, frontingSNIs map[string]*SNIConfig, verifyHostname *string, countryCode string) *Provider { +func NewProvider(hosts map[string]string, testURL string, masquerades []*Masquerade, passthrough []string, frontingSNIs map[string]*SNIConfig, verifyHostname *string, countryCode string) *Provider { p := &Provider{ HostAliases: make(map[string]string), TestURL: testURL, diff --git a/front_test.go b/front_test.go index f621432..0496478 100644 --- a/front_test.go +++ b/front_test.go @@ -1,7 +1,6 @@ package fronted import ( - "net/http" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +13,6 @@ func TestNewProvider(t *testing.T) { givenHosts map[string]string givenTestURL string givenMasquerades []*Masquerade - givenValidator ResponseValidator givenPassthrough []string //givenSNIConfig *SNIConfig givenFrontingSNIs map[string]*SNIConfig @@ -39,7 +37,6 @@ func TestNewProvider(t *testing.T) { givenHosts: map[string]string{"host1": "alias1", "host2": "alias2"}, givenTestURL: "http://test.com", givenMasquerades: []*Masquerade{{Domain: "domain1", IpAddress: "127.0.0.1"}}, - givenValidator: func(*http.Response) error { return nil }, givenPassthrough: []string{"passthrough1", "passthrough2"}, givenFrontingSNIs: map[string]*SNIConfig{ "test": &SNIConfig{ @@ -64,7 +61,7 @@ func TestNewProvider(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - actual := NewProvider(tt.givenHosts, tt.givenTestURL, tt.givenMasquerades, tt.givenValidator, tt.givenPassthrough, tt.givenFrontingSNIs, tt.givenVerifyHostname, "test") + actual := NewProvider(tt.givenHosts, tt.givenTestURL, tt.givenMasquerades, tt.givenPassthrough, tt.givenFrontingSNIs, tt.givenVerifyHostname, "test") tt.assert(t, actual) }) } diff --git a/fronted.go b/fronted.go index 82eb9ac..6942ea0 100644 --- a/fronted.go +++ b/fronted.go @@ -14,6 +14,8 @@ import ( "net" "net/http" "net/url" + "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -24,6 +26,7 @@ import ( tls "github.com/refraction-networking/utls" "github.com/getlantern/golog" + "github.com/getlantern/keepcurrent" "github.com/getlantern/ops" "github.com/alitto/pond/v2" @@ -48,6 +51,7 @@ type fronted struct { fronts sortedFronts maxAllowedCachedAge time.Duration maxCacheSize int + cacheFile string cacheSaveInterval time.Duration cacheDirty chan interface{} cacheClosed chan interface{} @@ -61,17 +65,23 @@ type fronted struct { stopCh chan interface{} crawlOnce sync.Once stopped atomic.Bool + countryCode string + httpClient *http.Client + configURL string } +// configURL is the URL from which to continually fetch updated domain fronting configurations. +const configURL = "https://media.githubusercontent.com/media/getlantern/fronted/refs/heads/main/fronted.yaml.gz" + // Interface for sending HTTP traffic over domain fronting. type Fronted interface { http.RoundTripper - // OnNewFrontsConfig updates the set of domain fronts to try from a YAML configuration. - OnNewFrontsConfig(yml []byte, countryCode string) + // onNewFrontsConfig updates the set of domain fronts to try from a YAML configuration. + onNewFrontsConfig(yml []byte) - // OnNewFronts updates the set of domain fronts to try. - OnNewFronts(pool *x509.CertPool, providers map[string]*Provider, countryCode string) + // onNewFronts updates the set of domain fronts to try. + onNewFronts(pool *x509.CertPool, providers map[string]*Provider) // Close closes any resources, such as goroutines that are testing fronts. Close() @@ -80,10 +90,13 @@ type Fronted interface { //go:embed fronted.yaml.gz var embedFS embed.FS +// Option is a functional option type that allows us to configure the fronted client. +type Option func(*fronted) + // NewFronted creates a new Fronted instance with the given cache file. // At this point it does not have the actual IPs, domains, etc of the fronts to try. // defaultProviderID is used when a front without a provider is encountered (eg in a cache file) -func NewFronted(cacheFile string) Fronted { +func NewFronted(options ...Option) Fronted { log.Debug("Creating new fronted") f := &fronted{ @@ -100,69 +113,118 @@ func NewFronted(cacheFile string) Fronted { connectingFronts: newConnectingFronts(4000), stopCh: make(chan interface{}, 10), defaultProviderID: defaultFrontedProviderID, + httpClient: http.DefaultClient, + cacheFile: defaultCacheFilePath(), + configURL: "", } - if cacheFile != "" { - f.initCaching(cacheFile) + for _, opt := range options { + opt(f) } f.readFrontsFromEmbeddedConfig() + f.keepCurrent() return f } -func (f *fronted) readFrontsFromEmbeddedConfig() { - yml, err := embedFS.ReadFile("fronted.yaml.gz") - if err != nil { - slog.Error("Failed to read smart dialer config", "error", err) +// WithHTTPClient sets the HTTP client to use for fetching the fronted configuration. For example, the client +// could be censorship-resistant in some way. +func WithHTTPClient(httpClient *http.Client) Option { + return func(f *fronted) { + f.httpClient = httpClient } - f.OnNewFrontsConfig(yml, "") } -func (f *fronted) OnNewFrontsConfig(gzippedYaml []byte, countryCode string) { - r, gzipErr := gzip.NewReader(bytes.NewReader(gzippedYaml)) - if gzipErr != nil { - slog.Error("Failed to create gzip reader", "error", gzipErr) - return +// WithCacheFile sets the file to use for caching domains that have successfully connected. +func WithCacheFile(file string) Option { + return func(f *fronted) { + f.initCaching(file) } - yml, err := io.ReadAll(r) - if err != nil { - slog.Error("Failed to read gzipped file", "error", err) - return +} + +// WithCountryCode sets the country code to use for fronting, which is particularly relevant for the +// SNI to use when connecting to the fronting domain. +func WithCountryCode(cc string) Option { + return func(f *fronted) { + f.countryCode = cc } - path, err := yaml.PathString("$.providers") - if err != nil { - slog.Error("Failed to create providers dpath", "error", err) - return +} + +// WithConfigURL sets the URL from which to continually fetch updated domain fronting configurations. +func WithConfigURL(configURL string) Option { + return func(f *fronted) { + f.configURL = configURL } - providers := make(map[string]*Provider) - pathErr := path.Read(bytes.NewReader(yml), &providers) - if pathErr != nil { - slog.Error("Failed to read providers", "error", pathErr) +} + +func defaultCacheFilePath() string { + if dir, err := os.UserConfigDir(); err != nil { + log.Errorf("Unable to get user config dir: %v", err) + // Use the temporary directory. + return filepath.Join(os.TempDir(), "fronted_cache.json") + } else { + return filepath.Join(dir, "domainfronting", "fronted_cache.json") + } +} + +func (f *fronted) keepCurrent() { + if f.configURL == "" { + slog.Debug("No config URL provided -- not updating fronting configuration") return } - trustedCAsPath, err := yaml.PathString("$.trustedcas") + source := keepcurrent.FromTarGz( + keepcurrent.FromWebWithClient(configURL, f.httpClient), "fronted.yaml.gz") + chDB := make(chan []byte) + dest := keepcurrent.ToChannel(chDB) + + runner := keepcurrent.NewWithValidator( + f.validator(), + source, + keepcurrent.ToFile("fronted.yaml.gz"), + dest, + ) + + go func() { + for data := range chDB { + f.onNewFrontsConfig(data) + } + }() + + runner.Start(12 * time.Hour) +} + +func (f *fronted) validator() func([]byte) error { + return func(data []byte) error { + _, _, err := processYaml(data) + if err != nil { + return err + } + return nil + } +} + +func (f *fronted) readFrontsFromEmbeddedConfig() { + yml, err := embedFS.ReadFile("fronted.yaml.gz") if err != nil { - slog.Error("Failed to create trusted CA path", "error", err) - return + slog.Error("Failed to read smart dialer config", "error", err) } - var trustedCAs []*CA - trustedCAsErr := trustedCAsPath.Read(bytes.NewReader(yml), &trustedCAs) - if trustedCAsErr != nil { - slog.Error("Failed to read trusted CAs", "error", trustedCAsErr) + f.onNewFrontsConfig(yml) +} + +func (f *fronted) onNewFrontsConfig(gzippedYaml []byte) { + pool, providers, err := processYaml(gzippedYaml) + if err != nil { + log.Errorf("Failed to process fronted config: %v", err) return } - pool := x509.NewCertPool() - for _, ca := range trustedCAs { - pool.AppendCertsFromPEM([]byte(ca.Cert)) - } - f.OnNewFronts(pool, providers, countryCode) + f.onNewFronts(pool, providers) } -// OnNewFronts sets the domain fronts to use, the trusted root CAs and the fronting providers +// onNewFronts sets the domain fronts to use, the trusted root CAs and the fronting providers // (such as Akamai, Cloudfront, etc) -func (f *fronted) OnNewFronts(pool *x509.CertPool, providers map[string]*Provider, countryCode string) { +func (f *fronted) onNewFronts(pool *x509.CertPool, providers map[string]*Provider) { // Make copies just to avoid any concurrency issues with access that may be happening on the // caller side. log.Debug("Updating fronted configuration") @@ -170,7 +232,7 @@ func (f *fronted) OnNewFronts(pool *x509.CertPool, providers map[string]*Provide log.Errorf("No providers configured") return } - providersCopy := copyProviders(providers, countryCode) + providersCopy := copyProviders(providers, f.countryCode) f.addProviders(providersCopy) f.addFronts(loadFronts(providersCopy)) f.certPool.Store(pool) @@ -623,7 +685,7 @@ func (f *fronted) isStopped() bool { func copyProviders(providers map[string]*Provider, countryCode string) map[string]*Provider { providersCopy := make(map[string]*Provider, len(providers)) for key, p := range providers { - providersCopy[key] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, nil, p.PassthroughPatterns, p.FrontingSNIs, p.VerifyHostname, countryCode) + providersCopy[key] = NewProvider(p.HostAliases, p.TestURL, p.Masquerades, p.PassthroughPatterns, p.FrontingSNIs, p.VerifyHostname, countryCode) } return providersCopy } @@ -705,3 +767,45 @@ func Vet(m *Masquerade, pool *x509.CertPool, testURL string) bool { defer conn.Close() return masq.verifyWithPost(conn, testURL) } + +func processYaml(gzippedYaml []byte) (*x509.CertPool, map[string]*Provider, error) { + r, gzipErr := gzip.NewReader(bytes.NewReader(gzippedYaml)) + if gzipErr != nil { + slog.Error("Failed to create gzip reader", "error", gzipErr) + // Wrap the error + return nil, nil, fmt.Errorf("failed to create gzip reader: %w", gzipErr) + } + yml, err := io.ReadAll(r) + if err != nil { + slog.Error("Failed to read gzipped file", "error", err) + return nil, nil, fmt.Errorf("failed to read gzipped file: %w", err) + } + path, err := yaml.PathString("$.providers") + if err != nil { + slog.Error("Failed to create providers path", "error", err) + return nil, nil, fmt.Errorf("failed to create providers path: %w", err) + } + providers := make(map[string]*Provider) + pathErr := path.Read(bytes.NewReader(yml), &providers) + if pathErr != nil { + slog.Error("Failed to read providers", "error", pathErr) + return nil, nil, fmt.Errorf("failed to read providers: %w", pathErr) + } + + trustedCAsPath, err := yaml.PathString("$.trustedcas") + if err != nil { + slog.Error("Failed to create trusted CA path", "error", err) + return nil, nil, fmt.Errorf("failed to create trusted CA path: %w", err) + } + var trustedCAs []*CA + trustedCAsErr := trustedCAsPath.Read(bytes.NewReader(yml), &trustedCAs) + if trustedCAsErr != nil { + slog.Error("Failed to read trusted CAs", "error", trustedCAsErr) + return nil, nil, fmt.Errorf("failed to read trusted CAs: %w", trustedCAsErr) + } + pool := x509.NewCertPool() + for _, ca := range trustedCAs { + pool.AppendCertsFromPEM([]byte(ca.Cert)) + } + return pool, providers, nil +} diff --git a/fronted_test.go b/fronted_test.go index a47ef3c..3396a95 100644 --- a/fronted_test.go +++ b/fronted_test.go @@ -19,27 +19,25 @@ import ( "time" . "github.com/getlantern/waitforserver" - "github.com/goccy/go-yaml" tls "github.com/refraction-networking/utls" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestYamlParsing(t *testing.T) { - // Read fronted.yaml into a map of Provider structs - f, err := os.ReadFile("fronted.yaml") - require.NoError(t, err) - providers := make(map[string]*Provider) - err = yaml.Unmarshal(f, &providers) - require.NoError(t, err) - - // Now write out the struct to a new yaml file. - f2, err := os.Create("fronted2.yaml") + // Disable this if we're running in CI because the file is using git lfs and will just be a pointer. + if os.Getenv("GITHUB_ACTIONS") == "true" { + t.Skip("Skipping test in GitHub Actions because the file is using git lfs and will be a pointer") + } + yamlFile, err := os.ReadFile("fronted.yaml.gz") require.NoError(t, err) - defer f2.Close() - err = yaml.NewEncoder(f2).Encode(providers) + pool, providers, err := processYaml(yamlFile) require.NoError(t, err) + require.NotNil(t, pool) + require.NotNil(t, providers) + // Make sure there are some providers + assert.Greater(t, len(providers), 0) } func TestDirectDomainFrontingWithoutSNIConfig(t *testing.T) { @@ -76,8 +74,8 @@ func TestDirectDomainFrontingWithSNIConfig(t *testing.T) { ArbitrarySNIs: []string{"mercadopago.com", "amazon.com.br", "facebook.com", "google.com", "twitter.com", "youtube.com", "instagram.com", "linkedin.com", "whatsapp.com", "netflix.com", "microsoft.com", "yahoo.com", "bing.com", "wikipedia.org", "github.com"}, }) defaultFrontedProviderID = "akamai" - transport := NewFronted(cacheFile) - transport.OnNewFronts(certs, p, "") + transport := NewFronted(WithCacheFile(cacheFile)) + transport.onNewFronts(certs, p) client := &http.Client{ Transport: transport, @@ -104,8 +102,8 @@ func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtE certs := trustedCACerts(t) p := testProvidersWithHosts(hosts) defaultFrontedProviderID = testProviderID - transport := NewFronted(cacheFile) - transport.OnNewFronts(certs, p, "") + transport := NewFronted(WithCacheFile(cacheFile)) + transport.onNewFronts(certs, p) client := &http.Client{ Transport: transport, @@ -114,8 +112,8 @@ func doTestDomainFronting(t *testing.T, cacheFile string, expectedMasqueradesAtE require.True(t, doCheck(client, http.MethodPost, http.StatusAccepted, pingURL)) defaultFrontedProviderID = testProviderID - transport = NewFronted(cacheFile) - transport.OnNewFronts(certs, p, "") + transport = NewFronted(WithCacheFile(cacheFile)) + transport.onNewFronts(certs, p) client = &http.Client{ Transport: transport, } @@ -223,15 +221,15 @@ func TestHostAliasesBasic(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil, nil, "") + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, nil, nil, "") certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) defaultFrontedProviderID = "cloudsack" - rt := NewFronted("") + rt := NewFronted() - rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p}, "") + rt.onNewFronts(certs, map[string]*Provider{"cloudsack": p}) client := &http.Client{Transport: rt} for _, test := range tests { @@ -321,14 +319,14 @@ func TestHostAliasesMulti(t *testing.T) { "abc.forbidden.com": "abc.cloudsack.biz", "def.forbidden.com": "def.cloudsack.biz", } - p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil, nil, "") + p1 := NewProvider(alias1, "https://ttt.cloudsack.biz/ping", masq1, nil, nil, nil, "") masq2 := []*Masquerade{{Domain: "example.com", IpAddress: sadCloudAddr}} alias2 := map[string]string{ "abc.forbidden.com": "abc.sadcloud.io", "def.forbidden.com": "def.sadcloud.io", } - p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil, nil, "") + p2 := NewProvider(alias2, "https://ttt.sadcloud.io/ping", masq2, nil, nil, nil, "") certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) @@ -340,8 +338,8 @@ func TestHostAliasesMulti(t *testing.T) { } defaultFrontedProviderID = "cloudsack" - rt := NewFronted("") - rt.OnNewFronts(certs, providers, "") + rt := NewFronted() + rt.onNewFronts(certs, providers) client := &http.Client{Transport: rt} @@ -459,14 +457,14 @@ func TestPassthrough(t *testing.T) { masq := []*Masquerade{{Domain: "example.com", IpAddress: cloudSackAddr}} alias := map[string]string{} passthrough := []string{"*.ok.cloudsack.biz", "abc.cloudsack.biz"} - p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, nil, passthrough, nil, nil, "") + p := NewProvider(alias, "https://ttt.cloudsack.biz/ping", masq, passthrough, nil, nil, "") certs := x509.NewCertPool() certs.AddCert(cloudSack.Certificate()) defaultFrontedProviderID = "cloudsack" - rt := NewFronted("") - rt.OnNewFronts(certs, map[string]*Provider{"cloudsack": p}, "") + rt := NewFronted() + rt.onNewFronts(certs, map[string]*Provider{"cloudsack": p}) client := &http.Client{Transport: rt} for _, test := range tests { @@ -508,141 +506,6 @@ func TestPassthrough(t *testing.T) { } } -func TestCustomValidators(t *testing.T) { - - sadCloud, sadCloudAddr, err := newCDN("sadcloud", "sadcloud.io") - if !assert.NoError(t, err, "failed to start sadcloud cdn") { - return - } - defer sadCloud.Close() - - sadCloudCodes := []int{http.StatusPaymentRequired, http.StatusTeapot, http.StatusBadGateway} - sadCloudValidator := NewStatusCodeValidator(sadCloudCodes) - testURL := "https://abc.forbidden.com/quux" - - setup := func(validator ResponseValidator) (Fronted, error) { - masq := []*Masquerade{{Domain: "example.com", IpAddress: sadCloudAddr}} - alias := map[string]string{ - "abc.forbidden.com": "abc.sadcloud.io", - } - p := NewProvider(alias, "https://ttt.sadcloud.io/ping", masq, validator, nil, nil, nil, "") - - certs := x509.NewCertPool() - certs.AddCert(sadCloud.Certificate()) - - providers := map[string]*Provider{ - "sadcloud": p, - } - - defaultFrontedProviderID = "sadcloud" - f := NewFronted("") - f.OnNewFronts(certs, providers, "") - return f, nil - } - - // This error indicates that the validator has discarded all masquerades. - // Each test starts with one masquerade, which is vetted during the - // call to NewDirect. - masqueradesExhausted := fmt.Sprintf(`Get "%v": could not complete request even with retries`, testURL) - - tests := []struct { - name string - responseCode int - validator ResponseValidator - expectedError string - }{ - // with the default validator, only 403s are rejected - { - name: "with default validator, it should reject 403", - responseCode: http.StatusForbidden, - validator: nil, - expectedError: masqueradesExhausted, - }, - { - name: "with default validator, it should accept 202", - responseCode: http.StatusAccepted, - validator: nil, - expectedError: "", - }, - { - name: "with default validator, it should accept 402", - responseCode: http.StatusPaymentRequired, - validator: nil, - expectedError: "", - }, - { - name: "with default validator, it should accept 418", - responseCode: http.StatusTeapot, - validator: nil, - expectedError: "", - }, - { - name: "with default validator, it should accept 502", - responseCode: http.StatusBadGateway, - validator: nil, - expectedError: "", - }, - - // with the custom validator, 403 is allowed, listed codes are rejected - { - name: "with custom validator, it should accept 403", - responseCode: http.StatusForbidden, - validator: sadCloudValidator, - expectedError: "", - }, - { - name: "with custom validator, it should accept 402", - responseCode: http.StatusAccepted, - validator: sadCloudValidator, - expectedError: "", - }, - { - name: "with custom validator, it should reject and return error for 402", - responseCode: http.StatusPaymentRequired, - validator: sadCloudValidator, - expectedError: masqueradesExhausted, - }, - { - name: "with custom validator, it should reject and return error for 418", - responseCode: http.StatusTeapot, - validator: sadCloudValidator, - expectedError: masqueradesExhausted, - }, - { - name: "with custom validator, it should reject and return error for 502", - responseCode: http.StatusBadGateway, - validator: sadCloudValidator, - expectedError: masqueradesExhausted, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - direct, err := setup(test.validator) - require.NoError(t, err) - client := &http.Client{ - Transport: direct, - } - - req, err := http.NewRequest(http.MethodGet, testURL, nil) - require.NoError(t, err) - if test.responseCode != http.StatusAccepted { - val := strconv.Itoa(test.responseCode) - log.Debugf("requesting forced response code %s", val) - req.Header.Set(CDNForceFail, val) - } - - res, err := client.Do(req) - if test.expectedError == "" { - require.NoError(t, err) - assert.Equal(t, test.responseCode, res.StatusCode, "Failed to force response status code") - } else { - assert.EqualError(t, err, test.expectedError) - } - }) - } -} - const ( // set this header to an integer to force response status code CDNForceFail = "X-CDN-Force-Fail" @@ -866,7 +729,7 @@ func TestFindWorkingMasquerades(t *testing.T) { stopCh: make(chan interface{}, 10), } f.providers = make(map[string]*Provider) - f.providers["testProviderId"] = NewProvider(nil, "", nil, nil, nil, nil, nil, "") + f.providers["testProviderId"] = NewProvider(nil, "", nil, nil, nil, nil, "") f.fronts = make(sortedFronts, len(tt.masquerades)) for i, m := range tt.masquerades { f.fronts[i] = m diff --git a/go.mod b/go.mod index 35562c5..b8c7d30 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.22.8 require ( github.com/alitto/pond/v2 v2.1.5 github.com/getlantern/golog v0.0.0-20230503153817-8e72de7e0a65 + github.com/getlantern/keepcurrent v0.0.0-20240126172110-2e0264ca385d github.com/getlantern/keyman v0.0.0-20200819205636-76fef27c39f1 github.com/getlantern/netx v0.0.0-20240814210628-0984f52e2d18 github.com/getlantern/ops v0.0.0-20231025133620-f368ab734534 @@ -14,13 +15,14 @@ require ( github.com/getlantern/waitforserver v1.0.1 github.com/goccy/go-yaml v1.15.13 github.com/refraction-networking/utls v1.6.7 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.10.0 ) require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/getlantern/byteexec v0.0.0-20170405023437-4cfb26ec74f4 // indirect github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect github.com/getlantern/elevate v0.0.0-20200430163644-2881a121236d // indirect @@ -33,15 +35,23 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-stack/stack v1.8.1 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect + github.com/mholt/archiver/v3 v3.5.1 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.27.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c4cd9ca..a96478a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/alitto/pond/v2 v2.1.5 h1:2pp/KAPcb02NSpHsjjnxnrTDzogMLsq+vFf/L0DB84A= github.com/alitto/pond/v2 v2.1.5/go.mod h1:xkjYEgQ05RSpWdfSd1nM3OVv7TBhLdy7rMp3+2Nq+yE= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -8,6 +9,9 @@ github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBS github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/getlantern/byteexec v0.0.0-20170405023437-4cfb26ec74f4 h1:Nqmy8i81dzokjNHpyOg24gnQBeGRF7D51m8HmBRNn0Y= github.com/getlantern/byteexec v0.0.0-20170405023437-4cfb26ec74f4/go.mod h1:4WCQkaCIwta0KlF9bQZA1jYqp8bzIS2PeCqjnef8nZ8= github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY= @@ -37,6 +41,8 @@ github.com/getlantern/hidden v0.0.0-20220104173330-f221c5a24770/go.mod h1:GOQsoD github.com/getlantern/iptool v0.0.0-20210721034953-519bf8ce0147/go.mod h1:hfspzdRcvJ130tpTPL53/L92gG0pFtvQ6ln35ppwhHE= github.com/getlantern/iptool v0.0.0-20230112135223-c00e863b2696 h1:D7wbL2Ww6QN5SblEDMiQcFulqz2jgcvawKaNBTzHLvQ= github.com/getlantern/iptool v0.0.0-20230112135223-c00e863b2696/go.mod h1:hfspzdRcvJ130tpTPL53/L92gG0pFtvQ6ln35ppwhHE= +github.com/getlantern/keepcurrent v0.0.0-20240126172110-2e0264ca385d h1:2/9rPC1xT+jWBnAe4mD6Q0LWkByFYGcTiKsmDWbv2T4= +github.com/getlantern/keepcurrent v0.0.0-20240126172110-2e0264ca385d/go.mod h1:enUAvxkJ15QUtTKOKoO9WJV2L5u33P8YmqkC+iu8iT4= github.com/getlantern/keyman v0.0.0-20200819205636-76fef27c39f1 h1:8qNXKWQBgmMfaXXTNfYs1D6i42etSjvwxfCSlmvHHr8= github.com/getlantern/keyman v0.0.0-20200819205636-76fef27c39f1/go.mod h1:FMf0g72BHs14jVcD8i8ubEk4sMB6JdidBn67d44i3ws= github.com/getlantern/mockconn v0.0.0-20200818071412-cb30d065a848 h1:2MhMMVBTnaHrst6HyWFDhwQCaJ05PZuOv1bE2gN8WFY= @@ -66,25 +72,44 @@ github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/goccy/go-yaml v1.15.13 h1:Xd87Yddmr2rC1SLLTm2MNDcTjeO/GYo0JGiww6gSTDg= github.com/goccy/go-yaml v1.15.13/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/refraction-networking/utls v0.0.0-20190909200633-43c36d3c1f57/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -92,8 +117,14 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= @@ -132,8 +163,8 @@ golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -143,6 +174,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/test_support.go b/test_support.go index 18bc19d..81f7048 100644 --- a/test_support.go +++ b/test_support.go @@ -24,8 +24,8 @@ func ConfigureCachingForTest(t *testing.T, cacheFile string) Fronted { certs := trustedCACerts(t) p := testProviders() defaultFrontedProviderID = testProviderID - f := NewFronted(cacheFile) - f.OnNewFronts(certs, p, "") + f := NewFronted(WithCacheFile(cacheFile)) + f.onNewFronts(certs, p) return f } @@ -33,8 +33,8 @@ func ConfigureHostAlaisesForTest(t *testing.T, hosts map[string]string) Fronted certs := trustedCACerts(t) p := testProvidersWithHosts(hosts) defaultFrontedProviderID = testProviderID - f := NewFronted("") - f.OnNewFronts(certs, p, "") + f := NewFronted() + f.onNewFronts(certs, p) return f } @@ -53,13 +53,13 @@ func trustedCACerts(t *testing.T) *x509.CertPool { func testProviders() map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil, nil, ""), + testProviderID: NewProvider(testHosts, pingTestURL, testMasquerades, nil, nil, nil, ""), } } func testProvidersWithHosts(hosts map[string]string) map[string]*Provider { return map[string]*Provider{ - testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil, nil, ""), + testProviderID: NewProvider(hosts, pingTestURL, testMasquerades, nil, nil, nil, ""), } } func testAkamaiProvidersWithHosts(hosts map[string]string, sniConfig *SNIConfig) map[string]*Provider { @@ -67,6 +67,6 @@ func testAkamaiProvidersWithHosts(hosts map[string]string, sniConfig *SNIConfig) "test": sniConfig, } return map[string]*Provider{ - "akamai": NewProvider(hosts, "https://fronted-ping.dsa.akamai.getiantem.org/ping", DefaultAkamaiMasquerades, nil, nil, frontingSNIs, nil, "test"), + "akamai": NewProvider(hosts, "https://fronted-ping.dsa.akamai.getiantem.org/ping", DefaultAkamaiMasquerades, nil, frontingSNIs, nil, "test"), } }