From d41c453e2061fdbd4fad4fa86dba72e30e109bbe Mon Sep 17 00:00:00 2001 From: Eric Fahlgren Date: Tue, 8 Oct 2024 10:15:23 -0700 Subject: [PATCH] misc: updates and enhancments to host-side management tools cleaner - add a tool to wipe out unused public/store/ entries update_all_targets - extract configuration data from settings - add a mechanism to optionally flush the redis database - add a filtering mechanism so users can reduce footprint and bandwidth use on their local servers Signed-off-by: Eric Fahlgren --- misc/cleaner.py | 31 +++++++++ misc/update_all_targets.py | 125 +++++++++++++++++++++++++++++++------ 2 files changed, 138 insertions(+), 18 deletions(-) create mode 100644 misc/cleaner.py diff --git a/misc/cleaner.py b/misc/cleaner.py new file mode 100644 index 00000000..42b81154 --- /dev/null +++ b/misc/cleaner.py @@ -0,0 +1,31 @@ +""" +Clean out the `public/store` directory, deleting all expired builds. + +First, collect a set of all the hash keys from the current results. +Next, scan the `public/store` directories to see if they are in the set +(i.e., they are still being referenced in the database). If not, then we +know the directory has expired, and we can delete it. + +A host cron job to run this on a regular basis looks like this, where +`asu-venv` is your local Python virtual environment directory: + + cd ~/asu/ && . ~/asu-venv/bin/activate && python misc/cleaner.py +""" + +from asu.util import get_redis_client +from asu.config import settings +from shutil import rmtree + +redis_client = get_redis_client() +active_hashes = {key.split(":")[-1] for key in redis_client.keys("rq:results:*")} +print(f"{active_hashes = }") + +store = settings.public_path / "store" +for dir in store.glob("*"): + # We check length as an added precaution against doing something stupid. + # Currently all build hashes are 32, see util.py for details. + if len(dir.name) == 32 and dir.name not in active_hashes: + print(f"Delete it {dir = !s}") + rmtree(dir) + else: + print(f"Keep it {dir = !s}") diff --git a/misc/update_all_targets.py b/misc/update_all_targets.py index d4755f57..7ea62285 100644 --- a/misc/update_all_targets.py +++ b/misc/update_all_targets.py @@ -1,39 +1,128 @@ +""" +Update the version/target database with all the specified versions, targets +and subtargets. By setting appropriate values in the `version_filter` +dictionary, you can limit the collection of data stored. This is useful when +running a local server where you are only interested in supporting a small +subset of the full OpenWrt builds available. + +By default, `version_filter` is set to `None`, resulting in all available +data being updated. +""" +import sys from requests import Session +from asu.config import settings +from asu.util import get_redis_client session = Session() -asu_url = "http://localhost:5001" +port = "8000" # Somehow extract from podman-compose.yml? +asu_url = f"http://localhost:{port}" +token = settings.update_token +upstream = settings.upstream_url + +extra_versions = ["22.03-SNAPSHOT", "23.05-SNAPSHOT"] + +# Example version filtering: set to your desired versions and targets. +# Set `version_filter = None` to reload all versions, targets and subtargets. +version_filter = { + "21.02.0": {"x86/64"}, + "21.02.1": {"x86/64"}, + "21.02.2": {"x86/64"}, + "21.02.3": {"x86/64"}, + "21.02.4": {"x86/64"}, + "21.02.5": {"x86/64"}, + "21.02.6": {"x86/64"}, + "21.02.7": {"x86/64"}, + "22.03.0": {"x86/64"}, + "22.03.1": {"x86/64"}, + "22.03.2": {"x86/64"}, + "22.03.3": {"x86/64"}, + "22.03.4": {"x86/64"}, + "22.03.5": {"x86/64"}, + "22.03.6": {"x86/64"}, + "22.03.7": {"x86/64"}, + "22.03-SNAPSHOT": {"x86/64"}, + "23.05.0": {"x86/64"}, + "23.05.2": {"x86/64"}, + "23.05.3": {"x86/64"}, + "23.05.4": {"x86/64"}, + "23.05.5": { + "ath79/generic", + "x86/64", + }, + "23.05-SNAPSHOT": {"x86/64"}, + "SNAPSHOT": { + "ath79/generic", + "bcm27xx/bcm2708", + "bcm53xx/generic", + "mediatek/mt7622", + "mvebu/cortexa72", + "tegra/generic", + "x86/64", + "x86/generic", + "x86/geode", + "x86/legacy", + }, +} +version_filter = None # Delete this line to use above filters. + + +def skip_version(version): + return version_filter and version not in version_filter + + +def targets_from_version(version): + return version_filter.get(version, None) if version_filter else None + + +def skip_target(target, filter): + return filter and target not in filter def reload_all(): - versions = session.get("https://downloads.openwrt.org/.versions.json").json()[ - "versions_list" - ] - for version in versions: + versions = session.get(f"{upstream}/.versions.json").json()["versions_list"] + versions.extend(extra_versions) + for version in sorted(set(versions)): + if skip_version(version): + continue + print(f"Reloading {version}") - targets = session.get( - f"https://downloads.openwrt.org/releases/{version}/.targets.json" - ) + targets = session.get(f"{upstream}/releases/{version}/.targets.json") if targets.status_code == 404: print(f"Targets not found for {version}") continue + targets = targets.json() + target_filter = targets_from_version(version) for target in targets: + if skip_target(target, target_filter): + continue print(f"Reloading {version}/{target}") session.get( f"{asu_url}/api/v1/update/{version}/{target}", - headers={"X-Update-Token": "changeme"}, + headers={"X-Update-Token": token}, + ) + + if not skip_version("SNAPSHOT"): + targets = session.get(f"{upstream}/snapshots/.targets.json").json() + target_filter = targets_from_version("SNAPSHOT") + for target in targets: + if skip_target(target, target_filter): + continue + print(f"Reloading SNAPSHOT/{target}") + session.get( + f"{asu_url}/api/v1/update/SNAPSHOT/{target}", + headers={"X-Update-Token": token}, ) - targets = session.get( - "https://downloads.openwrt.org/snapshots/.targets.json" - ).json() - for target in targets: - print(f"Reloading SNAPSHOT/{target}") - session.get( - f"{asu_url}/api/v1/update/SNAPSHOT/{target}", - headers={"X-Update-Token": "changeme"}, - ) +# "Flushing" wipes the whole redis database, including build status. +# Run misc/cleaner.py after if you wish to delete the unused entries +# in public/store/. +flush = "--flush" in sys.argv +redis_client = get_redis_client() +if flush: + print("Flushing:", redis_client.flushall()) reload_all() +print("Redis keys:", redis_client.keys()[:20])