phase1: remove kmods in target packages if archive is enabled

OPKG gets confused if kmod packages are present in both, target packages as well as kernel version specific folder. Remove them from target packages to make opkg pick the kmods from kmod archive folder only. We reorder the step by: 1) Move the kmods to dedicated directory 2) Index packages 3) Index kmods 4) Make checksums 5) Extract kmods sums from remote sums 6) Merge the local sums with the remote kmods-only sums This permits to produce a better sha256sums that also reflect the hash of the entire directory. Co-developed-by: Christian Marangi <[email protected]> Signed-off-by: Daniel Golle <[email protected]> [ rework implementation, sha256sumsclean + --remove-source-files ] Signed-off-by: Christian Marangi <[email protected]>
openwrt · Nov 13, 2024 · ad0db5f · ad0db5f
1 parent d014e42
commit ad0db5f
Showing 1 changed file with 107 additions and 61 deletions.
diff --git a/phase1/master.cfg b/phase1/master.cfg
@@ -1218,23 +1218,6 @@ def prepareFactory(target):
         )
     )
 
-    factory.addStep(
-        ShellCommand(
-            name="pkgindex",
-            description="Indexing packages",
-            descriptionDone="Packages indexed",
-            command=[
-                "make",
-                Interpolate("-j%(prop:nproc:-1)s"),
-                "package/index",
-                "V=s",
-                "CONFIG_SIGNED_PACKAGES=",
-            ],
-            env=MakeEnv(),
-            haltOnFailure=True,
-        )
-    )
-
     factory.addStep(
         ShellCommand(
             name="images",
@@ -1271,17 +1254,6 @@ def prepareFactory(target):
         )
     )
 
-    factory.addStep(
-        ShellCommand(
-            name="checksums",
-            description="Calculating checksums",
-            descriptionDone="Checksums calculated",
-            command=["make", "-j1", "checksum", "V=s"],
-            env=MakeEnv(),
-            haltOnFailure=True,
-        )
-    )
-
     factory.addStep(
         ShellCommand(
             name="kmoddir",
@@ -1303,10 +1275,11 @@ def prepareFactory(target):
     factory.addStep(
         ShellCommand(
             name="kmodprepare",
-            description="Preparing kmod archive",
-            descriptionDone="Kmod archive prepared",
+            description="Moving kmod to archive",
+            descriptionDone="Kmod moved",
             command=[
                 "rsync",
+                "--remove-source-files",
                 "--include=/kmod-*.ipk",
                 "--include=/kmod-*.apk",
                 "--exclude=*",
@@ -1327,6 +1300,23 @@ def prepareFactory(target):
         )
     )
 
+    factory.addStep(
+        ShellCommand(
+            name="pkgindex",
+            description="Indexing packages",
+            descriptionDone="Packages indexed",
+            command=[
+                "make",
+                Interpolate("-j%(prop:nproc:-1)s"),
+                "package/index",
+                "V=s",
+                "CONFIG_SIGNED_PACKAGES=",
+            ],
+            env=MakeEnv(),
+            haltOnFailure=True,
+        )
+    )
+
     factory.addStep(
         ShellCommand(
             name="kmodindex",
@@ -1350,6 +1340,93 @@ def prepareFactory(target):
         )
     )
 
+    factory.addStep(
+        ShellCommand(
+            name="checksums",
+            description="Calculating checksums",
+            descriptionDone="Checksums calculated",
+            command=["make", "-j1", "checksum", "V=s"],
+            env=MakeEnv(),
+            haltOnFailure=True,
+        )
+    )
+
+    # download remote sha256sums to 'target-sha256sums'
+    factory.addStep(
+        ShellCommand(
+            name="target-sha256sums",
+            description="Fetching remote sha256sums for target",
+            descriptionDone="Remote sha256sums for target fetched",
+            command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
+            + rsync_defopts
+            + [
+                Interpolate(
+                    "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
+                    url=GetRsyncParams.withArgs("bin", "url"),
+                    target=target,
+                    subtarget=subtarget,
+                    prefix=GetVersionPrefix,
+                ),
+                "target-sha256sums",
+            ],
+            env={
+                "RSYNC_PASSWORD": Interpolate(
+                    "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
+                )
+            },
+            logEnviron=False,
+            haltOnFailure=False,
+            flunkOnFailure=False,
+            warnOnFailure=False,
+            doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
+        )
+    )
+
+    factory.addStep(
+        ShellCommand(
+            name="target-sha256sums_kmodsparse",
+            description="Extract kmods from remote sha256sums",
+            descriptionDone="Kmods extracted",
+            command=[
+                "sed",
+                "/ \\*kmods\\//! d",
+                "target-sha256sums",
+                "| tee",
+                "target-sha256sums-kmods"
+            ],
+            haltOnFailure=False,
+            doStepIf=IsKmodArchiveEnabled,
+        )
+    )
+
+    factory.addStep(
+        ShellCommand(
+            name="mergesha256sum",
+            description="Merge sha256sums kmods with sha256sums",
+            descriptionDone="Sha256sums merged",
+            command=[
+                "sort",
+                "-t \"\"",
+                "-k 2",
+                "-u",
+                Interpolate(
+                    "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
+                    target=target,
+                    subtarget=subtarget,
+                ),
+                "target-sha256sums-kmods",
+                "-o",
+                Interpolate(
+                    "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
+                    target=target,
+                    subtarget=subtarget,
+                ),
+            ],
+            haltOnFailure=False,
+            doStepIf=IsKmodArchiveEnabled,
+        )
+    )
+
     # sign
     factory.addStep(
         MasterShellCommand(
@@ -1508,37 +1585,6 @@ def prepareFactory(target):
         )
     )
 
-    # download remote sha256sums to 'target-sha256sums'
-    factory.addStep(
-        ShellCommand(
-            name="target-sha256sums",
-            description="Fetching remote sha256sums for target",
-            descriptionDone="Remote sha256sums for target fetched",
-            command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
-            + rsync_defopts
-            + [
-                Interpolate(
-                    "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
-                    url=GetRsyncParams.withArgs("bin", "url"),
-                    target=target,
-                    subtarget=subtarget,
-                    prefix=GetVersionPrefix,
-                ),
-                "target-sha256sums",
-            ],
-            env={
-                "RSYNC_PASSWORD": Interpolate(
-                    "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
-                )
-            },
-            logEnviron=False,
-            haltOnFailure=False,
-            flunkOnFailure=False,
-            warnOnFailure=False,
-            doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
-        )
-    )
-
     # build list of files to upload
     factory.addStep(
         FileDownload(