Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

phase1: remove kmods in target packages if archive is enabled #37

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

phase1: remove kmods in target packages if archive is enabled #37

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d7fb14
phase1: Implement custom step ShellCommandAndSetProperty
Ansuel Nov 13, 2024
c50f1a4
phase1: remove kmods in target packages if archive is enabled
dangowrt May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 138 additions & 61 deletions phase1/master.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ from buildbot import locks
from buildbot.data import resultspec
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.process import buildstep
from buildbot.plugins import reporters
from buildbot.plugins import schedulers
from buildbot.plugins import steps
Expand Down Expand Up @@ -587,6 +588,10 @@ def IsKmodArchiveAndRsyncEnabled(step):
return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))


def IsRemoteShaSumsAvailable(step):
return step.getProperty("have_remote_shasums")


def GetBaseVersion(branch):
if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
return branch.split("-")[1]
Expand Down Expand Up @@ -758,6 +763,37 @@ c["builders"].append(
)


# CUSTOM CLASS

# Extension of ShellCommand and sets in property:
# - True: the command succeded
# - False: the command failed
class ShellCommandAndSetProperty(buildstep.ShellMixin, buildstep.BuildStep):
name = "shellandsetproperty"
renderables = ['property']

def __init__(
self,
property=None,
**kwargs,
):
kwargs = self.setupShellMixin(kwargs)

self.property = property

super().__init__(**kwargs)

@defer.inlineCallbacks
def run(self):
cmd = yield self.makeRemoteShellCommand()

yield self.runCommand(cmd)

self.setProperty(self.property, not cmd.didFail(), "ShellCommandAndSetProperty Step")

return cmd.results()


# NB the phase1 build factory assumes workers are single-build only
def prepareFactory(target):
(target, subtarget) = target.split("/")
Expand Down Expand Up @@ -1218,23 +1254,6 @@ def prepareFactory(target):
)
)

factory.addStep(
ShellCommand(
name="pkgindex",
description="Indexing packages",
descriptionDone="Packages indexed",
command=[
"make",
Interpolate("-j%(prop:nproc:-1)s"),
"package/index",
"V=s",
"CONFIG_SIGNED_PACKAGES=",
],
env=MakeEnv(),
haltOnFailure=True,
)
)

factory.addStep(
ShellCommand(
name="images",
Expand Down Expand Up @@ -1271,17 +1290,6 @@ def prepareFactory(target):
)
)

factory.addStep(
ShellCommand(
name="checksums",
description="Calculating checksums",
descriptionDone="Checksums calculated",
command=["make", "-j1", "checksum", "V=s"],
env=MakeEnv(),
haltOnFailure=True,
)
)

factory.addStep(
ShellCommand(
name="kmoddir",
Expand All @@ -1303,10 +1311,11 @@ def prepareFactory(target):
factory.addStep(
ShellCommand(
name="kmodprepare",
description="Preparing kmod archive",
descriptionDone="Kmod archive prepared",
description="Moving kmod to archive",
descriptionDone="Kmod moved",
command=[
"rsync",
"--remove-source-files",
"--include=/kmod-*.ipk",
"--include=/kmod-*.apk",
"--exclude=*",
Expand All @@ -1327,6 +1336,23 @@ def prepareFactory(target):
)
)

factory.addStep(
ShellCommand(
name="pkgindex",
description="Indexing packages",
descriptionDone="Packages indexed",
command=[
"make",
Interpolate("-j%(prop:nproc:-1)s"),
"package/index",
"V=s",
"CONFIG_SIGNED_PACKAGES=",
],
env=MakeEnv(),
haltOnFailure=True,
)
)

factory.addStep(
ShellCommand(
name="kmodindex",
Expand All @@ -1350,6 +1376,88 @@ def prepareFactory(target):
)
)

factory.addStep(
ShellCommand(
name="checksums",
description="Calculating checksums",
descriptionDone="Checksums calculated",
command=["make", "-j1", "checksum", "V=s"],
env=MakeEnv(),
haltOnFailure=True,
)
)

# download remote sha256sums to 'target-sha256sums'
factory.addStep(
ShellCommandAndSetProperty(
name="target-sha256sums",
description="Fetching remote sha256sums for target",
descriptionDone="Remote sha256sums for target fetched",
command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
+ rsync_defopts
+ [
Interpolate(
"%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
url=GetRsyncParams.withArgs("bin", "url"),
target=target,
subtarget=subtarget,
prefix=GetVersionPrefix,
),
"target-sha256sums",
],
env={
"RSYNC_PASSWORD": Interpolate(
"%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
)
},
property="have_remote_shasums",
logEnviron=False,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=False,
doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
)
)

factory.addStep(
ShellCommand(
name="target-sha256sums_kmodsparse",
description="Extract kmods from remote sha256sums",
descriptionDone="Kmods extracted",
command="sed \"/ \\*kmods\\//! d\" target-sha256sums | tee target-sha256sums-kmods",
haltOnFailure=False,
doStepIf=IsRemoteShaSumsAvailable,
)
)

factory.addStep(
ShellCommand(
name="mergesha256sum",
description="Merge sha256sums kmods with sha256sums",
descriptionDone="Sha256sums merged",
command=[
"sort",
"-t", " ",
"-k", 2,
"-u",
Interpolate(
"bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
target=target,
subtarget=subtarget,
),
"target-sha256sums-kmods",
"-o",
Interpolate(
"bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
target=target,
subtarget=subtarget,
),
],
haltOnFailure=False,
doStepIf=IsRemoteShaSumsAvailable,
)
)

# sign
factory.addStep(
MasterShellCommand(
Expand Down Expand Up @@ -1508,37 +1616,6 @@ def prepareFactory(target):
)
)

# download remote sha256sums to 'target-sha256sums'
factory.addStep(
ShellCommand(
name="target-sha256sums",
description="Fetching remote sha256sums for target",
descriptionDone="Remote sha256sums for target fetched",
command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
+ rsync_defopts
+ [
Interpolate(
"%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
url=GetRsyncParams.withArgs("bin", "url"),
target=target,
subtarget=subtarget,
prefix=GetVersionPrefix,
),
"target-sha256sums",
],
env={
"RSYNC_PASSWORD": Interpolate(
"%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
)
},
logEnviron=False,
haltOnFailure=False,
flunkOnFailure=False,
warnOnFailure=False,
doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
)
)

# build list of files to upload
factory.addStep(
FileDownload(
Expand Down