Merge branch 'main' into filesystem_customization

Schutzfile

@@ -46,14 +46,14 @@
           {
             "title": "updates",
             "name": "updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-x86_64-updates-released-20231101"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-x86_64-updates-released-20231201"
           }
         ],
         "aarch64": [
           {
             "title": "updates",
             "name": "updates",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-aarch64-updates-released-20231101"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-aarch64-updates-released-20231201"
           }
         ]
       },
@@ -63,17 +63,17 @@
           {
             "title": "updates-modular",
             "name": "updates-modular",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-x86_64-updates-released-modular-20231101"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-x86_64-updates-released-modular-20231201"
           }
         ],
         "aarch64": [
           {
             "title": "updates-modular",
             "name": "updates-modular",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-aarch64-updates-released-modular-20231101"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/public/f38/f38-aarch64-updates-released-modular-20231201"
           }
         ]
       }
     ]
   }
-}
+}

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
 	github.com/BurntSushi/toml v1.3.2
-	github.com/aws/aws-sdk-go v1.48.10
+	github.com/aws/aws-sdk-go v1.48.11
 	github.com/containers/common v0.57.0
 	github.com/containers/image/v5 v5.29.0
 	github.com/gobwas/glob v0.2.3

go.sum

@@ -60,8 +60,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat6
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
-github.com/aws/aws-sdk-go v1.48.10 h1:0LIFG3wp2Dt6PsxKWCg1Y1xRrn2vZnW5/gWdgaBalKg=
-github.com/aws/aws-sdk-go v1.48.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.48.11 h1:9YbiSbaF/jWi+qLRl+J5dEhr2mcbDYHmKg2V7RBcD5M=
+github.com/aws/aws-sdk-go v1.48.11/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=

pkg/disk/disk_test.go

@@ -536,9 +536,52 @@ func TestCreatePartitionTableLVMOnly(t *testing.T) {
 			}
 
 			// root should always be on a LVM
-			parent := rootPath[1]
-			_, ok := parent.(*LVMLogicalVolume)
-			assert.True(ok, "PT %q BP %q: root's parent (%q) is not an LVM logical volume", ptName, bpName, parent)
+			rootParent := rootPath[1]
+			{
+				_, ok := rootParent.(*LVMLogicalVolume)
+				assert.True(ok, "PT %q BP %q: root's parent (%+v) is not an LVM logical volume", ptName, bpName, rootParent)
+			}
+
+			// check logical volume sizes against blueprint
+			var lvsum uint64
+			for _, mnt := range tbp {
+				if mnt.Mountpoint == "/boot" {
+					// not on LVM; skipping
+					continue
+				}
+				mntPath := entityPath(mpt, mnt.Mountpoint)
+				mntParent := mntPath[1]
+				mntLV, ok := mntParent.(*LVMLogicalVolume) // the partition's parent should be the logical volume
+				assert.True(ok, "PT %q BP %q: %s's parent (%+v) is not an LVM logical volume", ptName, bpName, mnt.Mountpoint, mntParent)
+				assert.GreaterOrEqualf(mntLV.Size, mnt.MinSize, "PT %q BP %q: %s's size (%d) is smaller than the requested minsize (%d)", ptName, bpName, mnt.Mountpoint, mntLV.Size, mnt.MinSize)
+				lvsum += mntLV.Size
+			}
+
+			// root LV's parent should be the VG
+			lvParent := rootPath[2]
+			{
+				_, ok := lvParent.(*LVMVolumeGroup)
+				assert.True(ok, "PT %q BP %q: root LV's parent (%+v) is not an LVM volume group", ptName, bpName, lvParent)
+			}
+
+			// the root partition is the second to last entity in the path (the last is the partition table itself)
+			rootTop := rootPath[len(rootPath)-2]
+			vgPart, ok := rootTop.(*Partition)
+
+			// if the VG is in a LUKS container, check that there's enough space for the header too
+			{
+				vgParent := rootPath[3]
+				luksContainer, ok := vgParent.(*LUKSContainer)
+				if ok {
+					// this isn't the lvsum anymore, but the lvsum + the luks
+					// header, which should regardless be equal to or smaller
+					// than the partition
+					lvsum += luksContainer.MetadataSize()
+				}
+			}
+
+			assert.True(ok, "PT %q BP %q: root VG top level entity (%+v) is not a partition", ptName, bpName, rootTop)
+			assert.GreaterOrEqualf(vgPart.Size, lvsum, "PT %q BP %q: VG partition's size (%d) is smaller than the sum of logical volumes (%d)", ptName, bpName, vgPart.Size, lvsum)
 		}
 	}
 }
@@ -975,7 +1018,7 @@ func TestEnsureDirectorySizes(t *testing.T) {
 			rootVG := rootLUKS.Payload.(*LVMVolumeGroup)
 			rootLV := rootVG.LogicalVolumes[0]
 			homeLV := rootVG.LogicalVolumes[1]
-			assert.Equal(uint64(17*GiB)+rootVG.MetadataSize(), rootPart.Size)
+			assert.Equal(uint64(17*GiB)+rootVG.MetadataSize()+rootLUKS.MetadataSize(), rootPart.Size)
 			assert.Equal(uint64(7*GiB), rootLV.Size)
 			assert.Equal(uint64(10*GiB), homeLV.Size)
 

pkg/disk/partition_table.go

@@ -385,7 +385,7 @@ func (pt *PartitionTable) applyCustomization(mountpoints []blueprint.FilesystemC
 
 // Dynamically calculate and update the start point for each of the existing
 // partitions. Adjusts the overall size of image to either the supplied
-// value in `size` or to the sum of all partitions if that is lager.
+// value in `size` or to the sum of all partitions if that is larger.
 // Will grow the root partition if there is any empty space.
 // Returns the updated start point.
 func (pt *PartitionTable) relayout(size uint64) uint64 {
@@ -406,6 +406,8 @@ func (pt *PartitionTable) relayout(size uint64) uint64 {
 	for idx := range pt.Partitions {
 		partition := &pt.Partitions[idx]
 		if len(entityPath(partition, "/")) != 0 {
+			// keep the root partition index to handle after all the other
+			// partitions have been moved and resized
 			rootIdx = idx
 			continue
 		}
@@ -586,6 +588,13 @@ func resizeEntityBranch(path []Entity, size uint64) {
 				break
 			}
 		}
+		// If containerSize is 0, it means it doesn't have any direct sizeable
+		// children (e.g., a LUKS container with a VG child).  In that case,
+		// set the containerSize to the desired size for the branch before
+		// adding any metadata.
+		if containerSize == 0 {
+			containerSize = size
+		}
 		if vc, ok := element.(VolumeContainer); ok {
 			containerSize += vc.MetadataSize()
 		}
@@ -642,14 +651,17 @@ func (pt *PartitionTable) ensureLVM() error {
 			Description: "created via lvm2 and osbuild",
 		}
 
+		// create root logical volume on the new volume group with the same
+		// size and filesystem as the previous root partition
 		_, err := vg.CreateLogicalVolume("root", part.Size, filesystem)
 		if err != nil {
 			panic(fmt.Sprintf("Could not create LV: %v", err))
 		}
 
+		// replace the top-level partition payload with the new volume group
 		part.Payload = vg
 
-		// reset it so it will be grown later
+		// reset the vg partition size - it will be grown later
 		part.Size = 0
 
 		if pt.Type == "gpt" {

pkg/distro/rhel8/qcow2.go

@@ -155,14 +155,18 @@ func openstackCommonPackageSet(t *imageType) rpmmd.PackageSet {
 	return rpmmd.PackageSet{
 		Include: []string{
 			// Defaults
-			"@Core", "langpacks-en",
+			"@Core",
+			"langpacks-en",
 
 			// From the lorax kickstart
-			"selinux-policy-targeted", "cloud-init", "qemu-guest-agent",
+			"selinux-policy-targeted",
+			"cloud-init",
+			"qemu-guest-agent",
 			"spice-vdagent",
 		},
 		Exclude: []string{
-			"dracut-config-rescue", "rng-tools",
+			"dracut-config-rescue",
+			"rng-tools",
 		},
 	}
 }

pkg/distro/rhel9/ami.go

@@ -300,6 +300,7 @@ func ec2BuildPackageSet(t *imageType) rpmmd.PackageSet {
 func ec2CommonPackageSet(t *imageType) rpmmd.PackageSet {
 	return rpmmd.PackageSet{
 		Include: []string{
+			"@core",
 			"authselect-compat",
 			"chrony",
 			"cloud-init",
@@ -314,23 +315,43 @@ func ec2CommonPackageSet(t *imageType) rpmmd.PackageSet {
 			"redhat-release",
 			"redhat-release-eula",
 			"rsync",
+			"tuned",
 			"tar",
 		},
 		Exclude: []string{
 			"aic94xx-firmware",
 			"alsa-firmware",
 			"alsa-tools-firmware",
 			"biosdevname",
+			"firewalld",
 			"iprutils",
 			"ivtv-firmware",
+			"iwl1000-firmware",
+			"iwl100-firmware",
+			"iwl105-firmware",
+			"iwl135-firmware",
+			"iwl2000-firmware",
+			"iwl2030-firmware",
+			"iwl3160-firmware",
+			"iwl3945-firmware",
+			"iwl4965-firmware",
+			"iwl5000-firmware",
+			"iwl5150-firmware",
+			"iwl6000-firmware",
+			"iwl6000g2a-firmware",
+			"iwl6000g2b-firmware",
+			"iwl6050-firmware",
+			"iwl7260-firmware",
+			"libertas-sd8686-firmware",
 			"libertas-sd8787-firmware",
+			"libertas-usb8388-firmware",
 			"plymouth",
 			// RHBZ#2064087
 			"dracut-config-rescue",
 			// RHBZ#2075815
 			"qemu-guest-agent",
 		},
-	}.Append(coreOsCommonPackageSet(t)).Append(distroSpecificPackageSet(t))
+	}.Append(distroSpecificPackageSet(t))
 }
 
 // common rhel ec2 RHUI image package set

pkg/distro/rhel9/azure.go

@@ -149,14 +149,6 @@ func azureCommonPackageSet(t *imageType) rpmmd.PackageSet {
 		},
 	}.Append(distroSpecificPackageSet(t))
 
-	if t.arch.distro.isRHEL() {
-		ps.Append(rpmmd.PackageSet{
-			Include: []string{
-				"rhc",
-			},
-		})
-	}
-
 	return ps
 }
 

pkg/distro/rhel9/bare_metal.go

@@ -47,6 +47,7 @@ var (
 func bareMetalPackageSet(t *imageType) rpmmd.PackageSet {
 	ps := rpmmd.PackageSet{
 		Include: []string{
+			"@core",
 			"authselect-compat",
 			"chrony",
 			"cockpit-system",
@@ -82,8 +83,12 @@ func bareMetalPackageSet(t *imageType) rpmmd.PackageSet {
 			"rsync",
 			"tar",
 			"tcpdump",
+			"tuned",
 		},
-	}.Append(coreOsCommonPackageSet(t)).Append(distroBuildPackageSet(t))
+		Exclude: []string{
+			"dracut-config-rescue",
+		},
+	}.Append(distroBuildPackageSet(t))
 
 	// Ensure to not pull in subscription-manager on non-RHEL distro
 	if t.arch.distro.isRHEL() {

pkg/distro/rhel9/gce.go

@@ -214,6 +214,7 @@ func defaultGceRhuiImageConfig(rhsm bool) *distro.ImageConfig {
 func gceCommonPackageSet(t *imageType) rpmmd.PackageSet {
 	ps := rpmmd.PackageSet{
 		Include: []string{
+			"@core",
 			"langpacks-en", // not in Google's KS
 			"acpid",
 			"dhcp-client",
@@ -237,12 +238,14 @@ func gceCommonPackageSet(t *imageType) rpmmd.PackageSet {
 			// EFI
 			"grub2-tools",
 			"grub2-tools-minimal",
-			"firewalld", // not pulled in any more as on RHEL-8
+			// Performance tuning
+			"tuned",
 		},
 		Exclude: []string{
 			"alsa-utils",
 			"b43-fwcutter",
 			"dmraid",
+			"dracut-config-rescue",
 			"eject",
 			"gpm",
 			"irqbalance",
@@ -256,14 +259,20 @@ func gceCommonPackageSet(t *imageType) rpmmd.PackageSet {
 			"ipw2200-firmware",
 			"ivtv-firmware",
 			"iwl100-firmware",
+			"iwl105-firmware",
+			"iwl135-firmware",
 			"iwl1000-firmware",
+			"iwl2000-firmware",
+			"iwl2030-firmware",
+			"iwl3160-firmware",
 			"iwl3945-firmware",
 			"iwl4965-firmware",
 			"iwl5000-firmware",
 			"iwl5150-firmware",
 			"iwl6000-firmware",
 			"iwl6000g2a-firmware",
 			"iwl6050-firmware",
+			"iwl7260-firmware",
 			"kernel-firmware",
 			"libertas-usb8388-firmware",
 			"ql2100-firmware",
@@ -278,12 +287,9 @@ func gceCommonPackageSet(t *imageType) rpmmd.PackageSet {
 			// RHBZ#2075815
 			"qemu-guest-agent",
 		},
-	}.Append(coreOsCommonPackageSet(t)).Append(distroSpecificPackageSet(t))
+	}.Append(distroSpecificPackageSet(t))
 
-	// Some excluded packages are part of the @core group package set returned
-	// by coreOsCommonPackageSet(). Ensure that the conflicting packages are
-	// returned from the list of `Include` packages.
-	return ps.ResolveConflictsExclude()
+	return ps
 }
 
 // GCE BYOS image