o/snapstate, overlord, o/devicestate: support downloading components …

…in snapstate.Download (#14835)

* o/snapstate, o/assertstate: support downloading components to arbitrary directories

* o/snapstate, overlord, o/devicestate: support downloading components in snapstate.Download

* o/snapstate: leverage opts.ExpectOneSnap

* o/snapstate: combine functions for validating a snap and components against validation sets

* o/snapstate, o/devicestate, o/assertstate: rename snapstate.{Snap,Component}Setup.MountFile to BlobPath

overlord/assertstate/assertmgr.go

@@ -30,7 +30,6 @@ import (
 	"github.com/snapcore/snapd/asserts/sysdb"
 	"github.com/snapcore/snapd/overlord/snapstate"
 	"github.com/snapcore/snapd/overlord/state"
-	"github.com/snapcore/snapd/snap"
 )
 
 // AssertManager is responsible for the enforcement of assertions in
@@ -205,12 +204,7 @@ func fetchAssertsAndValidateComponent(st *state.State, compsup *snapstate.Compon
 	// run this task, since we may need to download a new snap-resource-pair.
 	compPath := compsup.CompPath
 	if compPath == "" {
-		cpi := snap.MinimalComponentContainerPlaceInfo(
-			compsup.ComponentName(),
-			compsup.Revision(),
-			snapsup.InstanceName(),
-		)
-		compPath = cpi.MountFile()
+		compPath = compsup.BlobPath(snapsup.InstanceName())
 	}
 
 	sha3_384, compSize, err := asserts.SnapFileSHA3_384(compPath)

overlord/devicestate/devicestate.go

@@ -1764,15 +1764,14 @@ func CreateRecoverySystem(st *state.State, label string, opts CreateRecoverySyst
 			continue
 		}
 
-		const userID = 0
 		// TODO: this respects the passed in validation sets, but does not
 		// currently respect refresh-control style of constraining snap
 		// revisions.
-		ts, info, err := snapstateDownload(context.TODO(), st, sn.Name, dirs.SnapBlobDir, &snapstate.RevisionOptions{
+		ts, info, err := snapstateDownload(context.TODO(), st, sn.Name, nil, dirs.SnapBlobDir, snapstate.RevisionOptions{
 			Channel:        sn.DefaultChannel,
 			Revision:       rev,
 			ValidationSets: valsets,
-		}, userID, snapstate.Flags{}, nil)
+		}, snapstate.Options{})
 		if err != nil {
 			return nil, err
 		}

overlord/devicestate/devicestate_systems_test.go

@@ -1655,10 +1655,10 @@ func (s *deviceMgrSystemsCreateSuite) TestDeviceManagerCreateRecoverySystemRemod
 	s.setupSnapDeclForNameAndID(c, "bar", s.ss.AssertedSnapID("bar"), "canonical")
 	s.setupSnapRevisionForFileAndID(c, barSnap, s.ss.AssertedSnapID("bar"), "canonical", snap.R(100))
 	// when download completes, the files will be at /var/lib/snapd/snap
-	c.Assert(os.MkdirAll(filepath.Dir(snapsupFoo.MountFile()), 0755), IsNil)
-	c.Assert(os.Rename(fooSnap, snapsupFoo.MountFile()), IsNil)
-	c.Assert(os.MkdirAll(filepath.Dir(snapsupBar.MountFile()), 0755), IsNil)
-	c.Assert(os.Rename(barSnap, snapsupBar.MountFile()), IsNil)
+	c.Assert(os.MkdirAll(filepath.Dir(snapsupFoo.BlobPath()), 0755), IsNil)
+	c.Assert(os.Rename(fooSnap, snapsupFoo.BlobPath()), IsNil)
+	c.Assert(os.MkdirAll(filepath.Dir(snapsupBar.BlobPath()), 0755), IsNil)
+	c.Assert(os.Rename(barSnap, snapsupBar.BlobPath()), IsNil)
 	tSnapsup1.Set("snap-setup", snapsupFoo)
 	tSnapsup2.Set("snap-setup", snapsupBar)
 
@@ -3417,7 +3417,7 @@ func (s *deviceMgrSystemsCreateSuite) testDeviceManagerCreateRecoverySystemValid
 
 		s.setupSnapDeclForNameAndID(c, snapsup.SideInfo.RealName, snapsup.SideInfo.SnapID, "canonical")
 		s.setupSnapRevisionForFileAndID(
-			c, snapsup.MountFile(), snapsup.SideInfo.SnapID, "canonical", snapRevisions[snapsup.SideInfo.RealName],
+			c, snapsup.BlobPath(), snapsup.SideInfo.SnapID, "canonical", snapRevisions[snapsup.SideInfo.RealName],
 		)
 
 		return nil
@@ -3465,20 +3465,20 @@ func (s *deviceMgrSystemsCreateSuite) testDeviceManagerCreateRecoverySystemValid
 	}, nil)
 
 	devicestate.MockSnapstateDownload(func(
-		_ context.Context, _ *state.State, name string, _ string, opts *snapstate.RevisionOptions, _ int, _ snapstate.Flags, _ snapstate.DeviceContext) (*state.TaskSet, *snap.Info, error,
+		ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error,
 	) {
 		expectedRev, ok := snapRevisions[name]
 		if !ok {
 			return nil, nil, fmt.Errorf("unexpected snap name %q", name)
 		}
 
-		c.Check(expectedRev, Equals, opts.Revision)
+		c.Check(expectedRev, Equals, revOpts.Revision)
 
-		tDownload := s.state.NewTask("mock-download", fmt.Sprintf("Download %s to track %s", name, opts.Channel))
+		tDownload := s.state.NewTask("mock-download", fmt.Sprintf("Download %s to track %s", name, revOpts.Channel))
 
 		si := &snap.SideInfo{
 			RealName: name,
-			Revision: opts.Revision,
+			Revision: revOpts.Revision,
 			SnapID:   fakeSnapID(name),
 		}
 		tDownload.Set("snap-setup", &snapstate.SnapSetup{
@@ -3654,7 +3654,7 @@ func (s *deviceMgrSystemsCreateSuite) TestDeviceManagerCreateRecoverySystemOffli
 	defer s.state.Unlock()
 
 	devicestate.MockSnapstateDownload(func(
-		_ context.Context, _ *state.State, name string, _ string, opts *snapstate.RevisionOptions, _ int, _ snapstate.Flags, _ snapstate.DeviceContext) (*state.TaskSet, *snap.Info, error,
+		ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error,
 	) {
 		c.Errorf("snapstate.Download called unexpectedly")
 		return nil, nil, nil
@@ -3825,7 +3825,7 @@ func (s *deviceMgrSystemsCreateSuite) TestDeviceManagerCreateRecoverySystemValid
 	assertstatetest.AddMany(s.state, vsetAssert)
 
 	devicestate.MockSnapstateDownload(func(
-		_ context.Context, _ *state.State, name string, _ string, opts *snapstate.RevisionOptions, _ int, _ snapstate.Flags, _ snapstate.DeviceContext) (*state.TaskSet, *snap.Info, error,
+		ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error,
 	) {
 		c.Errorf("snapstate.Download called unexpectedly")
 		return nil, nil, nil
@@ -4310,19 +4310,19 @@ func (s *deviceMgrSystemsCreateSuite) TestDeviceManagerCreateRecoverySystemValid
 	vset := vsetAssert.(*asserts.ValidationSet)
 
 	devicestate.MockSnapstateDownload(func(
-		_ context.Context, _ *state.State, name string, _ string, opts *snapstate.RevisionOptions, _ int, _ snapstate.Flags, _ snapstate.DeviceContext) (*state.TaskSet, *snap.Info, error,
+		ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error,
 	) {
 		expectedRev, ok := snapRevisions[name]
 		if !ok {
 			return nil, nil, fmt.Errorf("unexpected snap name %q", name)
 		}
 
-		c.Check(expectedRev, Equals, opts.Revision)
+		c.Check(expectedRev, Equals, revOpts.Revision)
 
-		tDownload := s.state.NewTask("fake-download", fmt.Sprintf("Download %s to track %s", name, opts.Channel))
+		tDownload := s.state.NewTask("fake-download", fmt.Sprintf("Download %s to track %s", name, revOpts.Channel))
 		si := &snap.SideInfo{
 			RealName: name,
-			Revision: opts.Revision,
+			Revision: revOpts.Revision,
 			SnapID:   fakeSnapID(name),
 		}
 

overlord/devicestate/export_test.go

@@ -185,7 +185,7 @@ func MockSnapstateUpdatePathWithDeviceContext(f func(st *state.State, si *snap.S
 	return r
 }
 
-func MockSnapstateDownload(f func(ctx context.Context, st *state.State, name string, blobDirectory string, opts *snapstate.RevisionOptions, userID int, flags snapstate.Flags, deviceCtx snapstate.DeviceContext) (*state.TaskSet, *snap.Info, error)) (restore func()) {
+func MockSnapstateDownload(f func(ctx context.Context, st *state.State, name string, components []string, blobDirectory string, revOpts snapstate.RevisionOptions, opts snapstate.Options) (*state.TaskSet, *snap.Info, error)) (restore func()) {
 	r := testutil.Backup(&snapstateDownload)
 	snapstateDownload = f
 	return r

overlord/devicestate/systems.go

@@ -280,7 +280,7 @@ func (ig *setupInfoGetter) SnapInfo(st *state.State, name string) (info *snap.In
 		}
 		// by the time this task runs, the file has already been
 		// downloaded and validated
-		snapFile, err := snapfile.Open(snapsup.MountFile())
+		snapFile, err := snapfile.Open(snapsup.BlobPath())
 		if err != nil {
 			return nil, "", false, err
 		}

overlord/managers_test.go

@@ -13693,7 +13693,7 @@ func (s *mgrsSuite) testDownload(c *C, downloadDir string) {
 	st.Lock()
 	defer st.Unlock()
 
-	ts, info, err := snapstate.Download(context.TODO(), st, "foo", downloadDir, nil, 0, snapstate.Flags{}, nil)
+	ts, info, err := snapstate.Download(context.TODO(), st, "foo", nil, downloadDir, snapstate.RevisionOptions{}, snapstate.Options{})
 	c.Assert(err, IsNil)
 	chg := st.NewChange("download-snap", "...")
 	chg.AddAll(ts)
@@ -13753,9 +13753,9 @@ func (s *mgrsSuite) TestDownloadSpecificRevision(c *C) {
 	st.Lock()
 	defer st.Unlock()
 
-	ts, info, err := snapstate.Download(context.TODO(), st, "foo", "", &snapstate.RevisionOptions{
+	ts, info, err := snapstate.Download(context.TODO(), st, "foo", nil, "", snapstate.RevisionOptions{
 		Revision: snap.R(snapOldRev),
-	}, 0, snapstate.Flags{}, nil)
+	}, snapstate.Options{})
 	c.Assert(err, IsNil)
 	chg := st.NewChange("download-snap", "...")
 	chg.AddAll(ts)

overlord/snapstate/handlers.go

@@ -725,7 +725,7 @@ func (m *SnapManager) doDownloadSnap(t *state.Task, tomb *tomb.Tomb) error {
 	}
 
 	meter := NewTaskProgressAdapterUnlocked(t)
-	targetFn := snapsup.MountFile()
+	targetFn := snapsup.BlobPath()
 
 	dlOpts := &store.DownloadOptions{
 		Scheduled: snapsup.IsAutoRefresh,
@@ -821,7 +821,7 @@ func (m *SnapManager) doPreDownloadSnap(t *state.Task, tomb *tomb.Tomb) error {
 		return err
 	}
 
-	targetFn := snapsup.MountFile()
+	targetFn := snapsup.BlobPath()
 	dlOpts := &store.DownloadOptions{
 		// pre-downloads are only triggered in auto-refreshes
 		Scheduled: true,

overlord/snapstate/handlers_components.go

@@ -191,15 +191,7 @@ func (m *SnapManager) doDownloadComponent(t *state.Task, tomb *tomb.Tomb) error
 		rate = autoRefreshRateLimited(st)
 	}
 
-	cpi := snap.MinimalComponentContainerPlaceInfo(
-		compsup.ComponentName(), compsup.CompSideInfo.Revision,
-		snapsup.InstanceName(),
-	)
-
-	// TODO:COMPS: to be consistent with snaps, this should be able to point
-	// somewhere else, based on a path that is in the compsup. this would be
-	// used for creating new recovery systems, like it is now for snaps
-	target := cpi.MountFile()
+	target := compsup.BlobPath(snapsup.InstanceName())
 
 	sto := Store(st, deviceCtx)
 	meter := NewTaskProgressAdapterUnlocked(t)

overlord/snapstate/handlers_components_download_test.go

@@ -47,16 +47,27 @@ func (s *downloadComponentSuite) SetUpTest(c *C) {
 }
 
 func (s *downloadComponentSuite) TestDoDownloadComponent(c *C) {
-	const autoRefresh = false
-	s.testDoDownloadComponent(c, autoRefresh)
+	s.testDoDownloadComponent(c, testDoDownloadComponentOpts{})
 }
 
 func (s *downloadComponentSuite) TestDoDownloadComponentAutoRefresh(c *C) {
-	const autoRefresh = true
-	s.testDoDownloadComponent(c, autoRefresh)
+	s.testDoDownloadComponent(c, testDoDownloadComponentOpts{
+		autoRefresh: true,
+	})
 }
 
-func (s *downloadComponentSuite) testDoDownloadComponent(c *C, autoRefresh bool) {
+func (s *downloadComponentSuite) TestDoDownloadComponentCustomBlobDir(c *C) {
+	s.testDoDownloadComponent(c, testDoDownloadComponentOpts{
+		blobDir: c.MkDir(),
+	})
+}
+
+type testDoDownloadComponentOpts struct {
+	autoRefresh bool
+	blobDir     string
+}
+
+func (s *downloadComponentSuite) testDoDownloadComponent(c *C, opts testDoDownloadComponentOpts) {
 	s.state.Lock()
 	defer s.state.Unlock()
 
@@ -75,9 +86,10 @@ func (s *downloadComponentSuite) testDoDownloadComponent(c *C, autoRefresh bool)
 	t := s.state.NewTask("download-component", "...")
 
 	t.Set("snap-setup", &snapstate.SnapSetup{
-		SideInfo:    si,
-		InstanceKey: "key",
-		Flags:       snapstate.Flags{IsAutoRefresh: autoRefresh},
+		SideInfo:        si,
+		InstanceKey:     "key",
+		Flags:           snapstate.Flags{IsAutoRefresh: opts.autoRefresh},
+		DownloadBlobDir: opts.blobDir,
 	})
 
 	t.Set("component-setup", &snapstate.ComponentSetup{
@@ -89,6 +101,7 @@ func (s *downloadComponentSuite) testDoDownloadComponent(c *C, autoRefresh bool)
 		DownloadInfo: &snap.DownloadInfo{
 			DownloadURL: "http://some-url.com/comp",
 		},
+		DownloadBlobDir: opts.blobDir,
 	})
 
 	chg := s.state.NewChange("download", "...")
@@ -111,17 +124,21 @@ func (s *downloadComponentSuite) testDoDownloadComponent(c *C, autoRefresh bool)
 		},
 	})
 
-	expectedPath := filepath.Join(dirs.SnapBlobDir, "snap_key+comp_11.comp")
+	blobDir := opts.blobDir
+	if blobDir == "" {
+		blobDir = dirs.SnapBlobDir
+	}
+	expectedPath := filepath.Join(blobDir, "snap_key+comp_11.comp")
 
 	var compsup snapstate.ComponentSetup
 	err := t.Get("component-setup", &compsup)
 	c.Assert(err, IsNil)
 	c.Check(compsup.CompPath, Equals, expectedPath)
 	c.Check(t.Status(), Equals, state.DoneStatus)
 
-	var opts *store.DownloadOptions
-	if autoRefresh {
-		opts = &store.DownloadOptions{
+	var downloadOpts *store.DownloadOptions
+	if opts.autoRefresh {
+		downloadOpts = &store.DownloadOptions{
 			RateLimit: 1234,
 			Scheduled: true,
 		}
@@ -131,7 +148,7 @@ func (s *downloadComponentSuite) testDoDownloadComponent(c *C, autoRefresh bool)
 		{
 			name:   "snap+comp",
 			target: expectedPath,
-			opts:   opts,
+			opts:   downloadOpts,
 		},
 	})
 }

overlord/snapstate/snapmgr.go

@@ -197,8 +197,10 @@ func (snapsup *SnapSetup) MountDir() string {
 	return snap.MountDir(snapsup.InstanceName(), snapsup.Revision())
 }
 
-// MountFile returns the path to the snap/squashfs file that is used to mount the snap.
-func (snapsup *SnapSetup) MountFile() string {
+// BlobPath returns the path to the snap/squashfs file that backs the snap that
+// is being setup. Unless the snap was downloaded to a custom location, this
+// will be under dirs.SnapBlobDir.
+func (snapsup *SnapSetup) BlobPath() string {
 	blobDir := snapsup.DownloadBlobDir
 	if blobDir == "" {
 		blobDir = dirs.SnapBlobDir
@@ -223,6 +225,9 @@ type ComponentSetup struct {
 	// SkipAssertionsDownload indicates that all assertions needed to install
 	// the component should already be present on the system.
 	SkipAssertionsDownload bool `json:"skip-assertions-download,omitempty"`
+	// DownloadBlobDir is the directory where the component file is downloaded to. If
+	// empty, then the components are downloaded to the default download directory.
+	DownloadBlobDir string `json:"download-blob-dir,omitempty"`
 	// ComponentInstallFlags is a set of flags that control the behavior of the
 	// component's installation/update.
 	ComponentInstallFlags
@@ -245,6 +250,29 @@ func (compsu *ComponentSetup) Revision() snap.Revision {
 	return compsu.CompSideInfo.Revision
 }
 
+// BlobPath returns the path to the component/squashfs file that backs the
+// component that is being setup. Unless the component was downloaded to a
+// custom location, this will be under dirs.SnapBlobDir.
+func (compsu *ComponentSetup) BlobPath(instanceName string) string {
+	if instanceName == "" {
+		instanceName = compsu.CompSideInfo.Component.SnapName
+	}
+
+	blobDir := compsu.DownloadBlobDir
+	if blobDir == "" {
+		blobDir = dirs.SnapBlobDir
+	}
+
+	cpi := snap.MinimalComponentContainerPlaceInfo(
+		compsu.CompSideInfo.Component.ComponentName,
+		compsu.CompSideInfo.Revision,
+		instanceName,
+	)
+
+	return filepath.Join(blobDir,
+		fmt.Sprintf("%s_%s.comp", cpi.ContainerName(), compsu.CompSideInfo.Revision))
+}
+
 // ComponentSetupFromSnapSetup returns a list of ComponentSetup structs for the
 // given task. Since the task could originate from one of a few different
 // scenarios, we inspect the task for various keys to determine how to find the