From 455bdc63c572bff87e336b7f6608c79792349b8f Mon Sep 17 00:00:00 2001 From: Andrew Phelps Date: Fri, 31 Jan 2025 15:54:33 -0500 Subject: [PATCH] daemon, tests: expose installing offline components in api and test change --- daemon/api_model.go | 45 +++++- daemon/api_model_test.go | 151 ++++++++++++++++++ daemon/api_systems_test.go | 35 +++- .../remodel-with-components-offline/task.yaml | 123 ++++++++++++++ 4 files changed, 344 insertions(+), 10 deletions(-) create mode 100644 tests/nested/manual/remodel-with-components-offline/task.yaml diff --git a/daemon/api_model.go b/daemon/api_model.go index 10241f475ce..d28880883d0 100644 --- a/daemon/api_model.go +++ b/daemon/api_model.go @@ -188,11 +188,15 @@ func startOfflineRemodelChange(st *state.State, newModel *asserts.Model, *pathsToNotRemove = make([]string, 0, len(slInfo.snaps)) localSnaps := make([]snapstate.PathSnap, 0, len(slInfo.snaps)) + localComponents := make([]snapstate.PathComponent, 0) for _, psi := range slInfo.snaps { // Move file to the same name of what a downloaded one would have dest := filepath.Join(dirs.SnapBlobDir, fmt.Sprintf("%s_%s.snap", psi.info.RealName, psi.info.Revision)) - os.Rename(psi.tmpPath, dest) + if err := os.Rename(psi.tmpPath, dest); err != nil { + return nil, InternalError("cannot move uploaded snap file: %v", err) + } + // Avoid trying to remove a file that does not exist anymore *pathsToNotRemove = append(*pathsToNotRemove, psi.tmpPath) @@ -200,14 +204,49 @@ func startOfflineRemodelChange(st *state.State, newModel *asserts.Model, SideInfo: &psi.info.SideInfo, Path: dest, }) + + for _, comp := range psi.components { + dest := filepath.Join(dirs.SnapBlobDir, + fmt.Sprintf("%s_%s.comp", comp.sideInfo.Component, comp.sideInfo.Revision)) + + if err := os.Rename(comp.tmpPath, dest); err != nil { + return nil, InternalError("cannot move uploaded component file: %v", err) + } + + // Avoid trying to remove a file that does not exist anymore + *pathsToNotRemove = append(*pathsToNotRemove, comp.tmpPath) + + localComponents = append(localComponents, snapstate.PathComponent{ + SideInfo: comp.sideInfo, + Path: dest, + }) + } + } + + for _, comp := range slInfo.components { + dest := filepath.Join(dirs.SnapBlobDir, + fmt.Sprintf("%s_%s.comp", comp.sideInfo.Component, comp.sideInfo.Revision)) + + if err := os.Rename(comp.tmpPath, dest); err != nil { + return nil, InternalError("cannot move uploaded component file: %v", err) + } + + // Avoid trying to remove a file that does not exist anymore + *pathsToNotRemove = append(*pathsToNotRemove, comp.tmpPath) + + localComponents = append(localComponents, snapstate.PathComponent{ + SideInfo: comp.sideInfo, + Path: dest, + }) } // Now create and start the remodel change chg, err := devicestateRemodel(st, newModel, devicestate.RemodelOptions{ // since this is the codepath that parses the form, offline is implicit // because local snaps are being provided. - Offline: true, - LocalSnaps: localSnaps, + Offline: true, + LocalSnaps: localSnaps, + LocalComponents: localComponents, }) if err != nil { return nil, BadRequest("cannot remodel device: %v", err) diff --git a/daemon/api_model_test.go b/daemon/api_model_test.go index 00b8acb3de3..6e5924b7dc4 100644 --- a/daemon/api_model_test.go +++ b/daemon/api_model_test.go @@ -28,6 +28,8 @@ import ( "mime/multipart" "net/http" "net/http/httptest" + "os" + "path/filepath" "strconv" "strings" "time" @@ -39,13 +41,17 @@ import ( "github.com/snapcore/snapd/client" "github.com/snapcore/snapd/client/clientutil" "github.com/snapcore/snapd/daemon" + "github.com/snapcore/snapd/dirs" "github.com/snapcore/snapd/overlord/assertstate/assertstatetest" "github.com/snapcore/snapd/overlord/auth" "github.com/snapcore/snapd/overlord/devicestate" "github.com/snapcore/snapd/overlord/devicestate/devicestatetest" "github.com/snapcore/snapd/overlord/hookstate" + "github.com/snapcore/snapd/overlord/snapstate" + "github.com/snapcore/snapd/overlord/snapstate/snapstatetest" "github.com/snapcore/snapd/overlord/state" "github.com/snapcore/snapd/snap" + "github.com/snapcore/snapd/snap/snaptest" ) var modelDefaults = map[string]interface{}{ @@ -674,3 +680,148 @@ func (s *modelSuite) testPostOfflineRemodel(c *check.C, params *testPostOfflineR c.Assert(soon, check.Equals, 1) } } + +func (s *modelSuite) TestPostOfflineRemodelWithComponents(c *check.C) { + s.expectRootAccess() + + oldModel := s.Brands.Model("my-brand", "my-old-model", modelDefaults) + newModel := s.Brands.Model("my-brand", "my-old-model", modelDefaults, map[string]interface{}{ + "revision": "2", + }) + + d := s.daemonWithOverlordMockAndStore() + hookMgr, err := hookstate.Manager(d.Overlord().State(), d.Overlord().TaskRunner()) + c.Assert(err, check.IsNil) + deviceMgr, err := devicestate.Manager(d.Overlord().State(), hookMgr, d.Overlord().TaskRunner(), nil) + c.Assert(err, check.IsNil) + d.Overlord().AddManager(deviceMgr) + + st := d.Overlord().State() + st.Lock() + defer st.Unlock() + + st.Set("seeded", true) + + assertstatetest.AddMany(st, s.StoreSigning.StoreAccountKey("")) + assertstatetest.AddMany(st, s.Brands.AccountsAndKeys("my-brand")...) + s.mockModel(st, oldModel) + + signer := assertstest.NewStoreStack("can0nical", nil) + assertstatetest.AddMany(st, signer.StoreAccountKey("")) + + account := assertstest.NewAccount(signer, "developer1", nil, "") + c.Assert(signer.Add(account), check.IsNil) + + const snapName = "some-snap" + snapID := snaptest.AssertedSnapID(snapName) + snapFormData := make(map[string]string) + + snapPath := snaptest.MakeTestSnapWithFiles(c, withComponents("name: some-snap\nversion: 1", []string{"comp"}), nil) + digest, size, err := asserts.SnapFileSHA3_384(snapPath) + c.Assert(err, check.IsNil) + + content, err := os.ReadFile(snapPath) + c.Assert(err, check.IsNil) + snapFormData[filepath.Base(snapPath)] = string(content) + + rev := mockStoreAssertion(c, signer, signer.AuthorityID, account.AccountID(), asserts.SnapRevisionType, map[string]interface{}{ + "snap-id": snapID, + "snap-sha3-384": digest, + "developer-id": account.AccountID(), + "snap-size": strconv.Itoa(int(size)), + "snap-revision": "10", + }) + + decl := mockStoreAssertion(c, signer, signer.AuthorityID, account.AccountID(), asserts.SnapDeclarationType, map[string]interface{}{ + "series": "16", + "snap-id": snapID, + "snap-name": snapName, + "publisher-id": account.AccountID(), + }) + + compPath, resRev, resPair := makeStandardComponent(c, signer, signer.AuthorityID, account.AccountID(), snapName, "comp") + content, err = os.ReadFile(compPath) + c.Assert(err, check.IsNil) + snapFormData[filepath.Base(compPath)] = string(content) + + // we handle components that are not associated with any of the snaps that + // are being uploaded a little differently, this part of the test helps + // cover that case + extraSnapDecl := mockStoreAssertion(c, signer, signer.AuthorityID, account.AccountID(), asserts.SnapDeclarationType, map[string]interface{}{ + "series": "16", + "snap-id": snaptest.AssertedSnapID("other-snap"), + "snap-name": "other-snap", + "publisher-id": account.AccountID(), + }) + extraCompPath, extraResRev, extraResPair := makeStandardComponent(c, signer, signer.AuthorityID, account.AccountID(), "other-snap", "comp") + content, err = os.ReadFile(extraCompPath) + c.Assert(err, check.IsNil) + snapFormData[filepath.Base(extraCompPath)] = string(content) + + snapstate.Set(st, "other-snap", &snapstate.SnapState{ + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{{ + RealName: "other-snap", + Revision: snap.R(10), + SnapID: snaptest.AssertedSnapID("other-snap"), + }}), + Current: snap.R(10), + Active: true, + }) + + var assertions strings.Builder + encoder := asserts.NewEncoder(&assertions) + + for _, a := range []asserts.Assertion{rev, decl, resRev, resPair, extraSnapDecl, extraResRev, extraResPair, account} { + err := encoder.Encode(a) + c.Assert(err, check.IsNil) + } + + fields := map[string][]string{ + "new-model": {string(asserts.Encode(newModel))}, + "assertion": {assertions.String()}, + } + form, boundary := createFormData(c, fields, snapFormData) + + defer daemon.MockDevicestateRemodel(func(st *state.State, nm *asserts.Model, opts devicestate.RemodelOptions) (*state.Change, error) { + c.Check(opts.Offline, check.Equals, true) + c.Assert(len(opts.LocalSnaps), check.Equals, 1) + c.Check(opts.LocalSnaps[0].SideInfo.RealName, check.Equals, snapName) + c.Check(opts.LocalSnaps[0].SideInfo.Revision, check.Equals, snap.Revision{N: 10}) + + snapPath := filepath.Join(dirs.SnapBlobDir, "some-snap_10.snap") + c.Check(strings.HasSuffix(opts.LocalSnaps[0].Path, snapPath), check.Equals, true) + + c.Assert(len(opts.LocalComponents), check.Equals, 2) + compPath := filepath.Join(dirs.SnapBlobDir, "some-snap+comp_20.comp") + c.Check(strings.HasSuffix(opts.LocalComponents[0].Path, compPath), check.Equals, true) + + extraCompPath := filepath.Join(dirs.SnapBlobDir, "other-snap+comp_20.comp") + c.Check(strings.HasSuffix(opts.LocalComponents[1].Path, extraCompPath), check.Equals, true) + + c.Check(nm, check.DeepEquals, newModel) + + chg := st.NewChange("remodel", "...") + return chg, nil + })() + + req, err := http.NewRequest("POST", "/v2/model", &form) + c.Assert(err, check.IsNil) + req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + req.Header.Set("Content-Length", strconv.Itoa(form.Len())) + + st.Unlock() + rsp := s.asyncReq(c, req, nil) + st.Lock() + + c.Assert(rsp.Status, check.Equals, 202) + c.Check(rsp.Change, check.DeepEquals, "1") + + chg := st.Change(rsp.Change) + c.Assert(chg, check.NotNil) + + c.Assert(st.Changes(), check.HasLen, 1) + chg1 := st.Changes()[0] + c.Assert(chg, check.DeepEquals, chg1) + c.Assert(chg.Kind(), check.Equals, "remodel") + c.Assert(chg.Err(), check.IsNil) +} diff --git a/daemon/api_systems_test.go b/daemon/api_systems_test.go index 88654e68dbf..264bf20f7a8 100644 --- a/daemon/api_systems_test.go +++ b/daemon/api_systems_test.go @@ -1386,10 +1386,20 @@ func (s *systemsCreateSuite) mockDevAssertion(c *check.C, t *asserts.AssertionTy } func (s *systemsCreateSuite) mockStoreAssertion(c *check.C, t *asserts.AssertionType, extras map[string]interface{}) asserts.Assertion { + return mockStoreAssertion(c, s.storeSigning, s.storeSigning.AuthorityID, s.dev1acct.AccountID(), t, extras) +} + +func mockStoreAssertion( + c *check.C, + signer assertstest.SignerDB, + authorityID, accountID string, + t *asserts.AssertionType, + extras map[string]interface{}, +) asserts.Assertion { headers := map[string]interface{}{ "type": t.Name, - "authority-id": s.storeSigning.AuthorityID, - "account-id": s.dev1acct.AccountID(), + "authority-id": authorityID, + "account-id": accountID, "series": "16", "revision": "5", "timestamp": "2030-11-06T09:16:26Z", @@ -1399,7 +1409,7 @@ func (s *systemsCreateSuite) mockStoreAssertion(c *check.C, t *asserts.Assertion headers[k] = v } - vs, err := s.storeSigning.Sign(t, headers, nil, "") + vs, err := signer.Sign(t, headers, nil, "") c.Assert(err, check.IsNil) return vs } @@ -2053,15 +2063,26 @@ func (s *systemsCreateSuite) TestCreateSystemActionWithComponentsOffline(c *chec } func (s *systemsCreateSuite) makeStandardComponent(c *check.C, snapName string, compName string) (compPath string, resourceRev, resourcePair asserts.Assertion) { + return makeStandardComponent(c, s.storeSigning, s.storeSigning.AuthorityID, s.dev1acct.AccountID(), snapName, compName) +} + +func makeStandardComponent( + c *check.C, + signer assertstest.SignerDB, + authorityID string, + accountID string, + snapName string, + compName string, +) (compPath string, resourceRev, resourcePair asserts.Assertion) { yaml := fmt.Sprintf("component: %s+%s\nversion: 1\ntype: standard", snapName, compName) compPath = snaptest.MakeTestComponent(c, yaml) digest, size, err := asserts.SnapFileSHA3_384(compPath) c.Assert(err, check.IsNil) - resRev := s.mockStoreAssertion(c, asserts.SnapResourceRevisionType, map[string]interface{}{ + resRev := mockStoreAssertion(c, signer, authorityID, accountID, asserts.SnapResourceRevisionType, map[string]interface{}{ "snap-id": snaptest.AssertedSnapID(snapName), - "developer-id": s.dev1acct.AccountID(), + "developer-id": accountID, "resource-name": compName, "resource-sha3-384": digest, "resource-revision": "20", @@ -2069,9 +2090,9 @@ func (s *systemsCreateSuite) makeStandardComponent(c *check.C, snapName string, "timestamp": time.Now().Format(time.RFC3339), }) - resPair := s.mockStoreAssertion(c, asserts.SnapResourcePairType, map[string]interface{}{ + resPair := mockStoreAssertion(c, signer, authorityID, accountID, asserts.SnapResourcePairType, map[string]interface{}{ "snap-id": snaptest.AssertedSnapID(snapName), - "developer-id": s.dev1acct.AccountID(), + "developer-id": accountID, "resource-name": compName, "resource-revision": "20", "snap-revision": "10", diff --git a/tests/nested/manual/remodel-with-components-offline/task.yaml b/tests/nested/manual/remodel-with-components-offline/task.yaml new file mode 100644 index 00000000000..e6527f57742 --- /dev/null +++ b/tests/nested/manual/remodel-with-components-offline/task.yaml @@ -0,0 +1,123 @@ +summary: remodel to a model that contains components + +details: | + This test remodels to a model that contains components. Specifically, this + tests updating the kernel snap to a version that supports the newly required + component and installing that component. + + validates that the newly created system can be rebooted into. + +systems: [-ubuntu-1*, -ubuntu-20*, -ubuntu-22*] + +environment: + INITIAL_MODEL_JSON: $TESTSLIB/assertions/test-snapd-component-remodel-initial-pc-24.json + NEW_MODEL_JSON: $TESTSLIB/assertions/test-snapd-component-remodel-new-pc-24.json + NESTED_ENABLE_TPM: true + NESTED_ENABLE_SECURE_BOOT: true + NESTED_BUILD_SNAPD_FROM_CURRENT: true + NESTED_REPACK_GADGET_SNAP: true + NESTED_REPACK_KERNEL_SNAP: true + NESTED_REPACK_BASE_SNAP: true + NESTED_REPACK_FOR_FAKESTORE: true + NESTED_FAKESTORE_BLOB_DIR: $(pwd)/fake-store-blobdir + NESTED_SIGN_SNAPS_FAKESTORE: true + NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL: http://localhost:11028 + +prepare: | + if [ "${TRUST_TEST_KEYS}" = "false" ]; then + echo "This test needs test keys to be trusted" + exit + fi + + # although nested_start_core_vm_unit usually installs this, the fake store + # will already have been set up, so we need to install it here + snap install test-snapd-swtpm --edge + + "${TESTSTOOLS}/store-state" setup-fake-store "${NESTED_FAKESTORE_BLOB_DIR}" + + gendeveloper1 sign-model < "${INITIAL_MODEL_JSON}" > initial-model.assert + + cp "${TESTSLIB}/assertions/testrootorg-store.account-key" "${NESTED_FAKESTORE_BLOB_DIR}/asserts" + cp "${TESTSLIB}/assertions/developer1.account" "${NESTED_FAKESTORE_BLOB_DIR}/asserts" + cp "${TESTSLIB}/assertions/developer1.account-key" "${NESTED_FAKESTORE_BLOB_DIR}/asserts" + cp initial-model.assert "${NESTED_FAKESTORE_BLOB_DIR}/asserts" + + tests.nested prepare-essential-snaps + + export SNAPPY_FORCE_API_URL="${NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL}" + ubuntu-image snap --image-size 10G ./initial-model.assert + + image_dir=$(tests.nested get images-path) + image_name=$(tests.nested get image-name core) + cp ./pc.img "${image_dir}/${image_name}" + tests.nested configure-default-user + + # run the fake device service too, so that the device can be initialised + systemd-run --collect --unit fakedevicesvc fakedevicesvc localhost:11029 + + tests.nested build-image core + tests.nested create-vm core + + #shellcheck source=tests/lib/core-config.sh + . "$TESTSLIB"/core-config.sh + wait_for_first_boot_change + + remote.exec 'sudo systemctl stop snapd snapd.socket' + + remote.exec 'sudo cat /var/lib/snapd/state.json' | gojq '.data.auth.device."session-macaroon"="fake-session"' > state.json + remote.push state.json + remote.exec 'sudo mv state.json /var/lib/snapd/state.json' + remote.exec 'sudo systemctl start snapd snapd.socket' + +restore: | + systemctl stop fakedevicesvc + "${TESTSTOOLS}/store-state" teardown-fake-store "${NESTED_FAKESTORE_BLOB_DIR}" + +execute: | + unsquashfs "${NESTED_FAKESTORE_BLOB_DIR}/pc-kernel.snap" + sed -i -e '/^version/ s/$/-with-comps/' squashfs-root/meta/snap.yaml + snap pack --filename=pc-kernel-with-comps.snap ./squashfs-root + "${TESTSTOOLS}"/build_kernel_with_comps.sh mac80211_hwsim wifi-comp pc-kernel-with-comps.snap + + kernel_id='pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza' + + # bump the available kernel version in the fake store + "${TESTSTOOLS}"/store-state make-snap-installable --noack \ + --revision 2 \ + "${NESTED_FAKESTORE_BLOB_DIR}" \ + ./pc-kernel-with-comps.snap \ + "${kernel_id}" + + "${TESTSTOOLS}"/store-state make-component-installable --noack \ + --snap-revision 2 \ + --component-revision 1 \ + --snap-id "${kernel_id}" \ + "${NESTED_FAKESTORE_BLOB_DIR}" \ + ./pc-kernel+wifi-comp.comp + + gendeveloper1 sign-model < "${NEW_MODEL_JSON}" > new-model.assert + remote.push new-model.assert + + boot_id="$(tests.nested boot-id)" + remote.exec 'SNAPPY_FORCE_API_URL=http://10.0.2.2:11028 snap download pc-kernel+wifi-comp --basename=pc-kernel' + change_id="$(remote.exec "sudo snap remodel --no-wait --offline \ + --snap=./pc-kernel.snap \ + --snap=./pc-kernel+wifi-comp.comp \ + --assertion=./pc-kernel.assert \ + new-model.assert")" + remote.wait-for reboot "${boot_id}" + + # this remodel expects two reboots, once for testing the recovery system + # and once for rebooting into the new kernel + boot_id="$(tests.nested boot-id)" + remote.wait-for reboot "${boot_id}" + + remote.exec "snap watch ${change_id}" + remote.exec 'snap list pc-kernel' | awk '$NR != 1 { print $3 }' | MATCH '2' + remote.exec 'snap components pc-kernel' | sed 1d | MATCH 'pc-kernel\+wifi-comp\s+installed' + + # make sure that the kernel module got installed and is loaded from our + # component + remote.exec sudo modprobe mac80211_hwsim + remote.exec ip link show wlan0 + remote.exec modinfo --filename mac80211_hwsim | MATCH '/lib/modules/.*/updates/wifi-comp'