From 8bfe56332eb3e6d5a48b11bf433e57f61e71ed2c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 17 Dec 2024 16:27:47 +1100 Subject: [PATCH 1/2] chore: upgrade Go to 1.22, adapt to http server URL changes Strange behaviour causing e2e test failures when upgrading to 1.22, diagnosed as a different form that an HTTP server decodes a request and doesn't fully unescape its `Path` when it has special characters in it. This seems to be related to the new ServeMux changes in 1.22 because GODEBUG=httpmuxgo121=1 reverts the behaviour and the test passes. The fix is to explicitly unescape the path before passing it on, go-trustless-utils will re-escape itself when it builds a new request to pass on downstream, but in the meantime we need to know the exact unescaped path so we can traverse it to validate the incoming data as well as pass it on upstream to the remote. --- go.mod | 2 +- pkg/internal/itest/e2e_test.go | 20 ++++++++++++++------ pkg/server/http/ipfs.go | 20 +++++++++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index e57c5007..93158ba0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/lassie -go 1.21 +go 1.22 require ( github.com/benbjohnson/clock v1.3.5 diff --git a/pkg/internal/itest/e2e_test.go b/pkg/internal/itest/e2e_test.go index 41e70a80..08fc8a15 100644 --- a/pkg/internal/itest/e2e_test.go +++ b/pkg/internal/itest/e2e_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -54,11 +55,14 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.NoError(err) // install the indexer to announce to + t.Log("Installing indexer") indexer := filepath.Join(tr.Dir, "storetheindex") - tr.Run("go", "install", "github.com/ipni/storetheindex@latest") + tr.Run("go", "install", "github.com/ipni/storetheindex@v0.8.14") // install the ipni cli to inspect the indexer + t.Log("Installing ipni") ipni := filepath.Join(tr.Dir, "ipni") - tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") + tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@v0.1.8") + t.Log("Installing frisbii") // install frisbii to serve the content frisbii := filepath.Join(tr.Dir, "frisbii") tr.Run("go", "install", "github.com/ipld/frisbii/cmd/frisbii@latest") @@ -67,7 +71,11 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.NoError(err) // initialise and start the indexer and adjust the config + t.Log("Initialising indexer") + t.Log(indexer + "init --store pebble --pubsub-topic /indexer/ingest/mainnet --no-bootstrap") tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") + + t.Log("Starting indexer") indexerReady := test.NewStdoutWatcher(test.IndexerReadyMatch) cmdIndexer := tr.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady)) select { @@ -82,6 +90,7 @@ func TestTrustlessGatewayE2E(t *testing.T) { carPath := trustlesspathing.Unixfs20mVarietyCARPath() // start frisbii with the fixture CAR + t.Logf("Starting frisbii with CAR [%s] and root [%s]", carPath, root) frisbiiReady := test.NewStdoutWatcher("Announce() complete") cmdFrisbii := tr.Start(test.NewExecution(frisbii, "--listen", "localhost:37471", @@ -101,8 +110,6 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.Eventually(func() bool { mh := root.Hash().B58String() findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) - t.Logf("import output:\n%s\n", findOutput) - if bytes.Contains(findOutput, []byte("not found")) { return false } @@ -110,8 +117,6 @@ func TestTrustlessGatewayE2E(t *testing.T) { t.Logf("mh %s: unexpected error: %s", mh, findOutput) return false } - - t.Logf("mh %s: found", mh) return true }, 10*time.Second, time.Second) @@ -172,6 +177,7 @@ func TestTrustlessGatewayE2E(t *testing.T) { if testCase.Path != "" { args[len(args)-1] = args[len(args)-1] + "/" + testCase.Path } + t.Logf("Running lassie %s", strings.Join(args, " ")) tr.Run(lassie, args...) _, err = os.Stat(expectedCarPath) @@ -188,6 +194,8 @@ func TestTrustlessGatewayE2E(t *testing.T) { reqUrl, err := url.Parse("http://localhost:30000/" + testCase.AsQuery()) req.NoError(err) + t.Logf("Fetching %s", reqUrl.String()) + // download and read all body from URL along with Accept:application/vnd.ipld.car header reqReq, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl.String(), nil) req.NoError(err) diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index d67ea47d..ffcb6547 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -24,13 +25,18 @@ import ( func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.ResponseWriter, *http.Request) { return func(res http.ResponseWriter, req *http.Request) { - statusLogger := newStatusLogger(req.Method, req.URL.Path) + unescapedPath, err := url.PathUnescape(req.URL.Path) + if err != nil { + logger.Warnf("error unescaping path: %s", err) + unescapedPath = req.URL.Path + } + statusLogger := newStatusLogger(req.Method, unescapedPath) if !checkGet(req, res, statusLogger) { return } - ok, request := decodeRetrievalRequest(cfg, res, req, statusLogger) + ok, request := decodeRetrievalRequest(cfg, res, req, unescapedPath, statusLogger) if !ok { return } @@ -87,7 +93,7 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response res.Header().Set("Content-Type", trustlesshttp.DefaultContentType().WithDuplicates(request.Duplicates).String()) res.Header().Set("Etag", request.Etag()) res.Header().Set("X-Content-Type-Options", "nosniff") - res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(req.URL.Path)) + res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(unescapedPath)) res.Header().Set("X-Trace-Id", requestId) statusLogger.logStatus(200, "OK") close(bytesWritten) @@ -152,8 +158,8 @@ func checkGet(req *http.Request, res http.ResponseWriter, statusLogger *statusLo return false } -func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *statusLogger) (bool, trustlessutils.Request) { - rootCid, path, err := trustlesshttp.ParseUrlPath(req.URL.Path) +func decodeRequest(res http.ResponseWriter, req *http.Request, unescapedPath string, statusLogger *statusLogger) (bool, trustlessutils.Request) { + rootCid, path, err := trustlesshttp.ParseUrlPath(unescapedPath) if err != nil { if errors.Is(err, trustlesshttp.ErrPathNotFound) { errorResponse(res, statusLogger, http.StatusNotFound, err) @@ -205,8 +211,8 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta } } -func decodeRetrievalRequest(cfg HttpServerConfig, res http.ResponseWriter, req *http.Request, statusLogger *statusLogger) (bool, types.RetrievalRequest) { - ok, request := decodeRequest(res, req, statusLogger) +func decodeRetrievalRequest(cfg HttpServerConfig, res http.ResponseWriter, req *http.Request, unescapedPath string, statusLogger *statusLogger) (bool, types.RetrievalRequest) { + ok, request := decodeRequest(res, req, unescapedPath, statusLogger) if !ok { return false, types.RetrievalRequest{} } From 2aeae59eda496d1bce5d1fc823b109b2c3d2a081 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 17 Dec 2024 16:56:37 +1100 Subject: [PATCH 2/2] Update pkg/internal/itest/e2e_test.go --- pkg/internal/itest/e2e_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/internal/itest/e2e_test.go b/pkg/internal/itest/e2e_test.go index 08fc8a15..b5592253 100644 --- a/pkg/internal/itest/e2e_test.go +++ b/pkg/internal/itest/e2e_test.go @@ -72,7 +72,6 @@ func TestTrustlessGatewayE2E(t *testing.T) { // initialise and start the indexer and adjust the config t.Log("Initialising indexer") - t.Log(indexer + "init --store pebble --pubsub-topic /indexer/ingest/mainnet --no-bootstrap") tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") t.Log("Starting indexer")