o/snapstate, o/devicestate: remodel with components from the store

overlord/devicestate/devicestate_systems_test.go

@@ -3599,6 +3599,7 @@ func (s *deviceMgrSystemsCreateSuite) testDeviceManagerCreateRecoverySystemValid
 
 		tValidate.WaitFor(tDownload)
 		ts := state.NewTaskSet(tDownload, tValidate)
+		ts.MarkEdge(tDownload, snapstate.SnapSetupEdge)
 		ts.MarkEdge(tValidate, snapstate.LastBeforeLocalModificationsEdge)
 		return ts, info, nil
 	})
@@ -4175,6 +4176,7 @@ func (s *deviceMgrSystemsCreateSuite) testDeviceManagerCreateRecoverySystemValid
 		}
 
 		snapsupTask.Set("component-setup-tasks", compsupTaskIDs)
+		ts.MarkEdge(snapsupTask, snapstate.SnapSetupEdge)
 		ts.MarkEdge(prev, snapstate.LastBeforeLocalModificationsEdge)
 
 		return ts, nil
@@ -4234,6 +4236,7 @@ func (s *deviceMgrSystemsCreateSuite) testDeviceManagerCreateRecoverySystemValid
 		}
 
 		download.Set("component-setup-tasks", compsupTaskIDs)
+		ts.MarkEdge(download, snapstate.SnapSetupEdge)
 		ts.MarkEdge(prev, snapstate.LastBeforeLocalModificationsEdge)
 
 		_, info := snaptest.MakeTestSnapInfoWithFiles(c, withComponents(snapYamls[name], compsToTypes(name)), snapFiles[name], si)
@@ -5826,6 +5829,7 @@ plugs:
 
 		tValidate.WaitFor(tDownload)
 		ts := state.NewTaskSet(tDownload, tValidate)
+		ts.MarkEdge(tDownload, snapstate.SnapSetupEdge)
 		ts.MarkEdge(tValidate, snapstate.LastBeforeLocalModificationsEdge)
 		return ts, info, nil
 	})

overlord/devicestate/export_test.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/snapcore/snapd/asserts"
+	"github.com/snapcore/snapd/asserts/snapasserts"
 	"github.com/snapcore/snapd/boot"
 	"github.com/snapcore/snapd/gadget"
 	"github.com/snapcore/snapd/gadget/install"
@@ -185,6 +186,14 @@ func MockSnapstatePathInstallGoal(mock func(snapstate.PathSnap) snapstate.Instal
 	return testutil.Mock(&snapstatePathInstallGoal, mock)
 }
 
+func MockSnapstateInstallComponents(mock func(ctx context.Context, st *state.State, names []string, info *snap.Info, vsets *snapasserts.ValidationSets, opts snapstate.Options) ([]*state.TaskSet, error)) (restore func()) {
+	return testutil.Mock(&snapstateInstallComponents, mock)
+}
+
+func MockSnapstateInstallComponentPath(mock func(st *state.State, csi *snap.ComponentSideInfo, info *snap.Info, path string, opts snapstate.Options) (*state.TaskSet, error)) (restore func()) {
+	return testutil.Mock(&snapstateInstallComponentPath, mock)
+}
+
 func MockSnapstateDownload(f func(ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error)) (restore func()) {
 	r := testutil.Backup(&snapstateDownload)
 	snapstateDownload = f

overlord/snapstate/snapstate.go

@@ -1684,6 +1684,7 @@ func downloadTasks(
 			t.Set("snap-setup", snapsup)
 			snapsupTask = t
 			ts.MarkEdge(t, BeginEdge)
+			ts.MarkEdge(t, SnapSetupEdge)
 		} else {
 			t.WaitFor(prev)
 			t.Set("snap-setup-task", snapsupTask.ID())

overlord/snapstate/snapstate_test.go

@@ -10087,6 +10087,7 @@ func (s *snapmgrTestSuite) TestDownload(c *C) {
 	downloadSnap := ts.MaybeEdge(snapstate.BeginEdge)
 	c.Assert(downloadSnap, NotNil)
 	c.Check(downloadSnap.Kind(), Equals, "download-snap")
+	c.Check(downloadSnap, DeepEquals, ts.MaybeEdge(snapstate.SnapSetupEdge))
 
 	var snapsup snapstate.SnapSetup
 	err = downloadSnap.Get("snap-setup", &snapsup)
@@ -10184,6 +10185,7 @@ func (s *snapmgrTestSuite) TestDownloadWithComponents(c *C) {
 	begin := ts.MaybeEdge(snapstate.BeginEdge)
 	c.Assert(begin, NotNil)
 	c.Check(begin.Kind(), Equals, "download-snap")
+	c.Check(begin, DeepEquals, ts.MaybeEdge(snapstate.SnapSetupEdge))
 
 	const componentExclusive = false
 	verifySnapAndComponentSetupsForDownload(c, begin, ts, downloadDir, componentExclusive)
@@ -10473,6 +10475,7 @@ func (s *snapmgrTestSuite) TestDownloadComponents(c *C) {
 	begin := ts.MaybeEdge(snapstate.BeginEdge)
 	c.Assert(begin, NotNil)
 	c.Check(begin.Kind(), Equals, "download-component")
+	c.Check(begin, DeepEquals, ts.MaybeEdge(snapstate.SnapSetupEdge))
 
 	const componentExclusive = true
 	verifySnapAndComponentSetupsForDownload(c, begin, ts, downloadDir, componentExclusive)

tests/lib/assertions/test-snapd-component-remodel-initial-pc-24.json

@@ -0,0 +1,39 @@
+{
+    "type": "model",
+    "authority-id": "developer1",
+    "series": "16",
+    "brand-id": "developer1",
+    "model": "my-model",
+    "revision": "1",
+    "architecture": "amd64",
+    "timestamp": "2024-04-24T00:00:00+00:00",
+    "grade": "dangerous",
+    "base": "core24",
+    "serial-authority": ["generic"],
+    "snaps": [
+        {
+            "default-channel": "24/edge",
+            "id": "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH",
+            "name": "pc",
+            "type": "gadget"
+        },
+        {
+            "default-channel": "24/edge",
+            "id": "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza",
+            "name": "pc-kernel",
+            "type": "kernel"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "dwTAh7MZZ01zyriOZErqd1JynQLiOGvM",
+            "name": "core24",
+            "type": "base"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4",
+            "name": "snapd",
+            "type": "snapd"
+        }
+    ]
+}

tests/lib/assertions/test-snapd-component-remodel-new-pc-24.json

@@ -0,0 +1,42 @@
+{
+    "type": "model",
+    "authority-id": "developer1",
+    "series": "16",
+    "brand-id": "developer1",
+    "model": "my-model",
+    "revision": "2",
+    "architecture": "amd64",
+    "timestamp": "2024-04-24T00:00:00+00:00",
+    "grade": "dangerous",
+    "base": "core24",
+    "serial-authority": ["generic"],
+    "snaps": [
+        {
+            "default-channel": "24/edge",
+            "id": "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH",
+            "name": "pc",
+            "type": "gadget"
+        },
+        {
+            "default-channel": "24/edge",
+            "id": "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza",
+            "name": "pc-kernel",
+            "type": "kernel",
+            "components": {
+              "wifi-comp": "required"
+            }
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "dwTAh7MZZ01zyriOZErqd1JynQLiOGvM",
+            "name": "core24",
+            "type": "base"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4",
+            "name": "snapd",
+            "type": "snapd"
+        }
+    ]
+}

tests/nested/manual/remodel-with-components/task.yaml

@@ -0,0 +1,131 @@
+summary: remodel to a model that contains components
+
+details: |
+  This test remodels to a model that contains components. Specifically, this
+  tests updating the kernel snap to a version that supports the newly required
+  component and installing that component.
+
+  validates that the newly created system can be rebooted into.
+
+systems: [-ubuntu-1*, -ubuntu-20*, -ubuntu-22*]
+
+environment:
+  INITIAL_MODEL_JSON: $TESTSLIB/assertions/test-snapd-component-remodel-initial-pc-24.json
+  NEW_MODEL_JSON: $TESTSLIB/assertions/test-snapd-component-remodel-new-pc-24.json
+  NESTED_ENABLE_TPM: true
+  NESTED_ENABLE_SECURE_BOOT: true
+  NESTED_BUILD_SNAPD_FROM_CURRENT: true
+  NESTED_REPACK_GADGET_SNAP: true
+  NESTED_REPACK_KERNEL_SNAP: true
+  NESTED_REPACK_BASE_SNAP: true
+  NESTED_REPACK_FOR_FAKESTORE: true
+  NESTED_FAKESTORE_BLOB_DIR: $(pwd)/fake-store-blobdir
+  NESTED_SIGN_SNAPS_FAKESTORE: true
+  NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL: http://localhost:11028
+
+prepare: |
+    if [ "${TRUST_TEST_KEYS}" = "false" ]; then
+        echo "This test needs test keys to be trusted"
+        exit
+    fi
+
+    # although nested_start_core_vm_unit usually installs this, the fake store
+    # will already have been set up, so we need to install it here
+    snap install test-snapd-swtpm --edge
+
+    "${TESTSTOOLS}/store-state" setup-fake-store "${NESTED_FAKESTORE_BLOB_DIR}"
+
+    gendeveloper1 sign-model < "${INITIAL_MODEL_JSON}" > initial-model.assert
+
+    cp "${TESTSLIB}/assertions/testrootorg-store.account-key" "${NESTED_FAKESTORE_BLOB_DIR}/asserts"
+    cp "${TESTSLIB}/assertions/developer1.account" "${NESTED_FAKESTORE_BLOB_DIR}/asserts"
+    cp "${TESTSLIB}/assertions/developer1.account-key" "${NESTED_FAKESTORE_BLOB_DIR}/asserts"
+    cp initial-model.assert "${NESTED_FAKESTORE_BLOB_DIR}/asserts"
+
+    tests.nested prepare-essential-snaps
+
+    export SNAPPY_FORCE_API_URL="${NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL}"
+    ubuntu-image snap --image-size 10G ./initial-model.assert
+
+    image_dir=$(tests.nested get images-path)
+    image_name=$(tests.nested get image-name core)
+    cp ./pc.img "${image_dir}/${image_name}"
+    tests.nested configure-default-user
+
+    # run the fake device service too, so that the device can be initialised
+    systemd-run --collect --unit fakedevicesvc fakedevicesvc localhost:11029
+
+    tests.nested build-image core
+    tests.nested create-vm core
+
+    #shellcheck source=tests/lib/core-config.sh
+    . "$TESTSLIB"/core-config.sh
+    wait_for_first_boot_change
+
+    remote.exec 'sudo systemctl stop snapd snapd.socket'
+
+    remote.exec 'sudo cat /var/lib/snapd/state.json' | gojq '.data.auth.device."session-macaroon"="fake-session"' > state.json
+    remote.push state.json
+    remote.exec 'sudo mv state.json /var/lib/snapd/state.json'
+    remote.exec 'sudo systemctl start snapd snapd.socket'
+
+restore: |
+    systemctl stop fakedevicesvc
+    "${TESTSTOOLS}/store-state" teardown-fake-store "${NESTED_FAKESTORE_BLOB_DIR}"
+
+execute: |
+  function post_json_data() {
+    route=$1
+    template=$2
+    shift 2
+
+    # shellcheck disable=SC2059
+    response=$(printf "${template}" "$@" | remote.exec "sudo snap debug api -X POST -H 'Content-Type: application/json' ${route}")
+    if ! gojq -e .change <<< "${response}"; then
+      echo "could not get change id from response: ${response}"
+      false
+    fi
+  }
+
+  unsquashfs "${NESTED_FAKESTORE_BLOB_DIR}/pc-kernel.snap"
+  sed -i -e '/^version/ s/$/-with-comps/' squashfs-root/meta/snap.yaml
+  snap pack --filename=pc-kernel-with-comps.snap ./squashfs-root
+  "${TESTSTOOLS}"/build_kernel_with_comps.sh mac80211_hwsim wifi-comp pc-kernel-with-comps.snap
+
+  kernel_id='pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza'
+
+  # bump the available kernel version in the fake store
+  "${TESTSTOOLS}"/store-state make-snap-installable --noack \
+    --revision 2 \
+    "${NESTED_FAKESTORE_BLOB_DIR}" \
+    ./pc-kernel-with-comps.snap \
+    "${kernel_id}"
+
+  "${TESTSTOOLS}"/store-state make-component-installable --noack \
+    --snap-revision 2 \
+    --component-revision 1 \
+    --snap-id "${kernel_id}" \
+    "${NESTED_FAKESTORE_BLOB_DIR}" \
+    ./pc-kernel+wifi-comp.comp
+
+  gendeveloper1 sign-model < "${NEW_MODEL_JSON}" > new-model.assert
+  remote.push new-model.assert
+
+  boot_id="$(tests.nested boot-id)"
+  change_id="$(remote.exec 'sudo snap remodel --no-wait new-model.assert')"
+  remote.wait-for reboot "${boot_id}"
+
+  # this remodel expects two reboots, once for testing the recovery system
+  # and once for rebooting into the new kernel
+  boot_id="$(tests.nested boot-id)"
+  remote.wait-for reboot "${boot_id}"
+
+  remote.exec "snap watch ${change_id}"
+  remote.exec 'snap list pc-kernel' | awk '$NR != 1 { print $3 }' | MATCH '2'
+  remote.exec 'snap components pc-kernel' | sed 1d | MATCH 'pc-kernel\+wifi-comp\s+installed'
+
+  # make sure that the kernel module got installed and is loaded from our
+  # component
+  remote.exec sudo modprobe mac80211_hwsim
+  remote.exec ip link show wlan0
+  remote.exec modinfo --filename mac80211_hwsim | MATCH '/lib/modules/.*/updates/wifi-comp'