From a052a03104dee6b900d828b55c32c50a8ccdb273 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 11:46:33 +0900 Subject: [PATCH 01/12] WIP Fix checking PKGBUILD, respecting "dirs_dir" The script should now open the PKGBUILD with the "editor" when an AUR package is to be updated. The script should eventually use "dirs_dir" in the config to place git-cloned directories of AUR package repositories. --- update.py | 184 +++++++++++++++++++++++++++--------------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/update.py b/update.py index 4ecaa1d..4f545b1 100755 --- a/update.py +++ b/update.py @@ -33,25 +33,24 @@ def log_print(string): def ensure_pkg_dir_exists(pkg, pkg_state): log_print('Checking that dir for "{}" exists...'.format(pkg)) - pkg_dir = os.path.join(SCRIPT_DIR, pkg) - if os.path.isdir(pkg_dir): + pkgdir = os.path.join(pkg_state['dirs'], pkg) + if os.path.isdir(pkgdir): log_print('Dir for "{}" exists.'.format(pkg)) return True - elif os.path.exists(pkg_dir): - log_print('"{}" exists but is not a dir'.format(pkg_dir)) + elif os.path.exists(pkgdir): + log_print('"{}" exists but is not a dir'.format(pkgdir)) return False elif "repo_path" not in pkg_state[pkg]: pkg_state[pkg]["repo_path"] = AUR_GIT_REPO_PATH_TEMPLATE.format(pkg) try: subprocess.run( - ["git", "clone", pkg_state[pkg]["repo_path"], pkg], + ["git", "clone", pkg_state[pkg]["repo_path"], pkgdir], check=True, - cwd=SCRIPT_DIR, ) except subprocess.CalledProcessError: log_print( 'ERROR: Failed to git clone "{}" (tried repo path "{}")'.format( - pkg_dir, pkg_state[pkg]["repo_path"] + pkgdir, pkg_state[pkg]["repo_path"] ) ) return False @@ -65,12 +64,13 @@ def ensure_pkg_dir_exists(pkg, pkg_state): def update_pkg_dir(pkg, state): log_print('Making sure pkg dir for "{}" is up to date...'.format(pkg)) + pkgdir = os.path.join(state['dirs'], pkg) # fetch all try: subprocess.run( ["git", "fetch", "-p", "--all"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, ) except subprocess.CalledProcessError: log_print( @@ -84,7 +84,7 @@ def update_pkg_dir(pkg, state): result = subprocess.run( ["git", "remote"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, capture_output=True, encoding="UTF-8", ) @@ -111,7 +111,7 @@ def update_pkg_dir(pkg, state): result = subprocess.run( ["git", "status", "-sb", "--porcelain"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, capture_output=True, encoding="UTF-8", ) @@ -143,7 +143,7 @@ def update_pkg_dir(pkg, state): result = subprocess.run( ["git", "log", "-1", "--format=format:%H"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, capture_output=True, encoding="UTF-8", ) @@ -169,7 +169,7 @@ def update_pkg_dir(pkg, state): result = subprocess.run( ["git", "log", "-1", "--format=format:%H", selected_remote], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, capture_output=True, encoding="UTF-8", ) @@ -193,19 +193,19 @@ def update_pkg_dir(pkg, state): if current_branch_hash != remote_branch_hash: try: subprocess.run( - ["git", "pull"], check=True, cwd=os.path.join(SCRIPT_DIR, pkg) + ["git", "pull"], check=True, cwd=pkgdir ) except subprocess.CalledProcessError: try: subprocess.run( ["git", "checkout", "--", "*"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, ) subprocess.run( ["git", "pull"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, ) except subprocess.CalledProcessError: log_print( @@ -219,12 +219,13 @@ def update_pkg_dir(pkg, state): return True, False -def check_pkg_build(pkg, editor): +def check_pkg_build(pkg, state, editor): """Returns "ok", "not_ok", "abort", or "force_build".""" + pkgdir = os.path.join(state['dirs'], pkg) log_print('Checking PKGBUILD for "{}"...'.format(pkg)) try: subprocess.run( - [editor, os.path.join(pkg, "PKGBUILD")], check=True, cwd=SCRIPT_DIR + [editor, "PKGBUILD"], check=True, cwd=pkgdir ) except subprocess.CalledProcessError: log_print('ERROR: Failed checking PKGBUILD for "{}"'.format(pkg)) @@ -242,7 +243,7 @@ def check_pkg_build(pkg, editor): return "not_ok" elif user_input == "c": log_print("User will check PKGBUILD again") - return check_pkg_build(pkg, editor) + return check_pkg_build(pkg, state, editor) elif user_input == "a": return "abort" elif user_input == "f": @@ -254,17 +255,17 @@ def check_pkg_build(pkg, editor): continue -def check_pkg_version(pkgdir, pkg_state, repo, force_check_srcinfo): +def check_pkg_version(pkg, pkg_state, repo, force_check_srcinfo): """Returns "fail", "install", or "done".""" status, current_epoch, current_version = get_pkg_current_version( - pkgdir, pkg_state, repo + pkg, pkg_state, repo ) if status != "fetched": return status elif current_version is None: log_print( 'ERROR: Failed to get version from package "{}".'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "fail" @@ -272,19 +273,19 @@ def check_pkg_version(pkgdir, pkg_state, repo, force_check_srcinfo): 'Got version "{}:{}" for installed pkg "{}"'.format( current_epoch if current_epoch is not None else "0", current_version, - pkg_state[pkgdir]["pkg_name"], + pkg_state[pkg]["pkg_name"], ) ) return get_srcinfo_check_result( - current_epoch, current_version, pkgdir, force_check_srcinfo + current_epoch, current_version, pkg, force_check_srcinfo, pkg_state ) -def get_srcinfo_version(pkgdir): +def get_srcinfo_version(pkg, state): """Returns (success_bool, pkgepoch, pkgver, pkgrel)""" - if not os.path.exists(os.path.join(SCRIPT_DIR, pkgdir, ".SRCINFO")): - log_print(f'ERROR: .SRCINFO does not exist for pkg "{pkgdir}"') + if not os.path.exists(os.path.join(state['dirs'], pkg, ".SRCINFO")): + log_print(f'ERROR: .SRCINFO does not exist for pkg "{pkg}"') return False, None, None, None pkgver_reprog = re.compile("^\\s*pkgver\\s*=\\s*([a-zA-Z0-9._+-]+)\\s*$") pkgrel_reprog = re.compile("^\\s*pkgrel\\s*=\\s*([0-9.]+)\\s*$") @@ -293,7 +294,7 @@ def get_srcinfo_version(pkgdir): pkgrel = "" pkgepoch = "" with open( - os.path.join(SCRIPT_DIR, pkgdir, ".SRCINFO"), encoding="UTF-8" + os.path.join(state['dirs'], pkg, ".SRCINFO"), encoding="UTF-8" ) as fo: line = fo.readline() while len(line) > 0: @@ -310,9 +311,10 @@ def get_srcinfo_version(pkgdir): return True, pkgepoch, pkgver, pkgrel -def get_pkgbuild_version(pkgdir, force_check_srcinfo): +def get_pkgbuild_version(pkg, force_check_srcinfo, state): """Returns (success, epoch, version, release)""" - log_print(f'Getting version of "{pkgdir}"...') + pkgdir = os.path.join(state['dirs'], pkg) + log_print(f'Getting version of "{pkg}"...') while True and not force_check_srcinfo: log_print("Use .SRCINFO or directly parse PKGBUILD?") user_input = input("1 for .SRCINFO, 2 for PKGBUILD > ") @@ -321,7 +323,7 @@ def get_pkgbuild_version(pkgdir, force_check_srcinfo): # TODO support split packages if force_check_srcinfo or user_input == "1": srcinfo_fetch_success, pkgepoch, pkgver, pkgrel = get_srcinfo_version( - pkgdir + pkg, state ) if not srcinfo_fetch_success: log_print("ERROR: Failed to get pkg info from .SRCINFO") @@ -334,20 +336,20 @@ def get_pkgbuild_version(pkgdir, force_check_srcinfo): subprocess.run( ["makepkg", "-c", "--nobuild", "-s", "-r"], check=True, - cwd=os.path.join(SCRIPT_DIR, pkgdir), + cwd=pkgdir, ) except subprocess.CalledProcessError: log_print( 'ERROR: Failed to run "makepkg --nobuild" in "{}".'.format( - pkgdir + pkg ) ) - if os.path.exists(os.path.join(SCRIPT_DIR, pkgdir, "src")): - shutil.rmtree(os.path.join(SCRIPT_DIR, pkgdir, "src")) + if os.path.exists(os.path.join(pkgdir, "src")): + shutil.rmtree(os.path.join(pkgdir, "src")) return False, None, None, None - if os.path.exists(os.path.join(SCRIPT_DIR, pkgdir, "src")): - shutil.rmtree(os.path.join(SCRIPT_DIR, pkgdir, "src")) + if os.path.exists(os.path.join(pkgdir, "src")): + shutil.rmtree(os.path.join(pkgdir, "src")) pkgepoch = "" pkgver = "" pkgrel = "" @@ -357,7 +359,7 @@ def get_pkgbuild_version(pkgdir, force_check_srcinfo): [ "bash", "-c", - f"source {os.path.join(SCRIPT_DIR, pkgdir, 'PKGBUILD')}; echo \"pkgver=$pkgver\"; echo \"pkgrel=$pkgrel\"; echo \"epoch=$epoch\"", + f"source {os.path.join(pkgdir, 'PKGBUILD')}; echo \"pkgver=$pkgver\"; echo \"pkgrel=$pkgrel\"; echo \"epoch=$epoch\"", ], capture_output=True, text=True, @@ -392,29 +394,29 @@ def get_pkgbuild_version(pkgdir, force_check_srcinfo): return True, pkgepoch, pkgver, pkgrel else: log_print( - 'ERROR: Failed to get PKGBUILD version of "{}".'.format(pkgdir) + 'ERROR: Failed to get PKGBUILD version of "{}".'.format(pkg) ) return False, None, None, None def get_srcinfo_check_result( - current_epoch, current_version, pkgdir, force_check_srcinfo + current_epoch, current_version, pkg, force_check_srcinfo, state ): ver_success, pkgepoch, pkgver, pkgrel = get_pkgbuild_version( - pkgdir, force_check_srcinfo + pkg, force_check_srcinfo, state ) if ver_success: if current_epoch is None and pkgepoch is not None: log_print( 'Current installed version of "{}" is out of date (missing epoch).'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "install" elif current_epoch is not None and pkgepoch is None: log_print( 'Current installed version of "{}" is up to date (has epoch).'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "done" @@ -425,7 +427,7 @@ def get_srcinfo_check_result( ): log_print( 'Current installed version of "{}" is out of date (older epoch).'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "install" @@ -437,31 +439,31 @@ def get_srcinfo_check_result( ): log_print( 'Current installed version of "{}" is out of date (older version).'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "install" else: log_print( 'Current installed version of "{}" is up to date.'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "done" else: log_print( 'ERROR: Failed to get pkg_version of "{}"'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "fail" -def get_pkg_current_version(pkgdir, pkg_state, repo): +def get_pkg_current_version(pkg, pkg_state, repo): """Returns (status, epoch, version)""" log_print( 'Checking version of installed pkg "{}"...'.format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) current_epoch = None @@ -469,7 +471,7 @@ def get_pkg_current_version(pkgdir, pkg_state, repo): try: result = subprocess.run( "tar -tf {} | grep '{}.*/$'".format( - repo, pkg_state[pkgdir]["pkg_name"] + repo, pkg_state[pkg]["pkg_name"] ), check=True, capture_output=True, @@ -478,7 +480,7 @@ def get_pkg_current_version(pkgdir, pkg_state, repo): ) reprog = re.compile( "^{}-(?P[0-9]+:)?(?P[^-/: ]*-[0-9]+)/$".format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ), flags=re.MULTILINE, ) @@ -492,7 +494,7 @@ def get_pkg_current_version(pkgdir, pkg_state, repo): else: log_print( "ERROR: Failed to get current version from repo for package {}".format( - pkg_state[pkgdir]["pkg_name"] + pkg_state[pkg]["pkg_name"] ) ) return "fail", None, None @@ -584,7 +586,6 @@ def setup_ccache(chroot): f"{chroot}/root/etc/makepkg.conf", ], check=True, - cwd=SCRIPT_DIR, ) except subprocess.CalledProcessError: log_print("ERROR: Failed to enable ccache in makepkg.conf") @@ -603,7 +604,6 @@ def cleanup_ccache(chroot): f"{chroot}/root/etc/makepkg.conf", ], check=True, - cwd=SCRIPT_DIR, ) except subprocess.CalledProcessError: log_print("ERROR: Failed to disable ccache in makepkg.conf") @@ -651,7 +651,6 @@ def cleanup_sccache(chroot): f"{chroot}/root/usr/local/bin/rustc", ], check=False, - cwd=SCRIPT_DIR, ) except BaseException: log_print("WARNING: Failed to cleanup sccache files") @@ -749,7 +748,7 @@ def update_pkg_list( subprocess.run( command_list + post_command_list, check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=pkgdir, stdout=log_stdout, stderr=log_stderr, ) @@ -1014,34 +1013,35 @@ if __name__ == "__main__": sys.exit(1) pkg_state = {} + other_state = {} if args.pkg and not args.config: for pkg in args.pkg: pkg_state[pkg] = {} pkg_state[pkg]["aur_deps"] = [] - args_chroot = args.chroot - args_pkg_dir = args.pkg_dir - args_repo = args.repo - args_gpg_home = args.gpg_dir - args_logs_dir = args.logs_dir + other_state['chroot'] = args.chroot + other_state['pkgdir'] = args.pkg_dir + other_state['repo'] = args.repo + other_state['gpg_home'] = args.gpg_dir + other_state['logs_dir'] = args.logs_dir if args_logs_dir is not None: GLOBAL_LOG_FILE = args_logs_dir + "/update.py_logs" log_print( f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M %Z')}" ) log_print(f"Set GLOBAL_LOG_FILE to {GLOBAL_LOG_FILE}") - args_signing_gpg_dir = args.signing_gpg_dir - args_signing_gpg_key_fp = args.signing_gpg_key_fp + other_state['signing_gpg_dir'] = args.signing_gpg_dir + other_state['signing_gpg_key_fp'] = args.signing_gpg_key_fp if args_signing_gpg_key_fp is None: log_print( 'ERROR: Signing key fingerprint "signing_gpg_key_fp" not present in config' ) sys.exit(1) if args_signing_gpg_dir is not None and not args.no_store: - args_signing_gpg_pass = getpass.getpass("gpg signing key pass: ") + other_state['signing_gpg_pass'] = getpass.getpass("gpg signing key pass: ") if not test_gpg_passphrase( - args_signing_gpg_dir, - args_signing_gpg_key_fp, - args_signing_gpg_pass, + other_state['signing_gpg_dir'], + other_state['signing_gpg_key_fp'], + other_state['signing_gpg_pass'], ): sys.exit(1) elif args.config: @@ -1078,11 +1078,12 @@ if __name__ == "__main__": pkg_state[entry["name"]]["skip_branch_up_to_date"] = True else: pkg_state[entry["name"]]["skip_branch_up_to_date"] = False - args_chroot = d["chroot"] - args_pkg_dir = d["pkg_dir"] - args_repo = d["repo"] - args_gpg_home = d["gpg_dir"] - args_logs_dir = d["logs_dir"] + other_state['chroot'] = d["chroot"] + other_state['pkgdir'] = d["pkg_dir"] + other_state['repo'] = d["repo"] + other_state['gpg_home'] = d["gpg_dir"] + other_state['logs_dir'] = d["logs_dir"] + ohter_state['dirs'] = d["dirs_dir"] if args_logs_dir is not None: GLOBAL_LOG_FILE = args_logs_dir + "/update.py_logs" log_print( @@ -1168,27 +1169,26 @@ if __name__ == "__main__": pkg_state[pkg_list[i]]["state"] = "up to date" i += 1 continue - else: - check_pkg_build_result = check_pkg_build(pkg_list[i], editor) - if check_pkg_build_result == "ok": - pass - elif check_pkg_build_result == "not_ok": - pkg_state[pkg_list[i]]["state"] = "skip" - i += 1 - continue - elif check_pkg_build_result == "force_build": - pkg_state[pkg_list[i]]["state"] = "install" - i += 1 - continue - elif check_pkg_build_result == "invalid": - continue - elif check_pkg_build_result == "back": - if i > 0: - i -= 1 - continue - else: # check_pkg_build_result == "abort": - print_state_info_and_get_update_list(pkg_state) - sys.exit(1) + check_pkg_build_result = check_pkg_build(pkg_list[i], pkg_state, editor) + if check_pkg_build_result == "ok": + pass + elif check_pkg_build_result == "not_ok": + pkg_state[pkg_list[i]]["state"] = "skip" + i += 1 + continue + elif check_pkg_build_result == "force_build": + pkg_state[pkg_list[i]]["state"] = "install" + i += 1 + continue + elif check_pkg_build_result == "invalid": + continue + elif check_pkg_build_result == "back": + if i > 0: + i -= 1 + continue + else: # check_pkg_build_result == "abort": + print_state_info_and_get_update_list(pkg_state) + sys.exit(1) while True: if skip_on_same_ver and check_pkg_version_result is not None: state_result = check_pkg_version_result From 6e36ed0efc7a9884d2609950a2dd346c1f718e0d Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:14:37 +0900 Subject: [PATCH 02/12] WIP Impl no hardcoding of clone dir Also modified example_config.toml due to changed variable names and added a few comments. --- example_config.toml | 6 ++- update.py | 119 +++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 64 deletions(-) diff --git a/example_config.toml b/example_config.toml index aabeeec..14a047b 100644 --- a/example_config.toml +++ b/example_config.toml @@ -1,8 +1,10 @@ ########## MANDATORY VARIABLES chroot = "/home/stephen/Downloads/aur/chroot" -pkg_dir = "/home/custompkgs" +# Location to place built packages +pkg_out_dir = "/home/custompkgs" repo = "/home/custompkgs/custom.db.tar" -dirs_dir = "/home/stephen/Downloads/aur" +# Location to clone packages from AUR +clones_dir = "/home/stephen/Downloads/aur" gpg_dir = "/home/stephen/Downloads/aur/checkingGPG" logs_dir = "/home/stephen/Downloads/aur/logs" signing_gpg_dir = "/home/stephen/Downloads/aur/signingGPG" diff --git a/update.py b/update.py index 4f545b1..dfe7f13 100755 --- a/update.py +++ b/update.py @@ -31,9 +31,9 @@ def log_print(string): print(string, file=lf) -def ensure_pkg_dir_exists(pkg, pkg_state): +def ensure_pkg_dir_exists(pkg, pkg_state, other_state): log_print('Checking that dir for "{}" exists...'.format(pkg)) - pkgdir = os.path.join(pkg_state['dirs'], pkg) + pkgdir = os.path.join(other_state['clones_dir'], pkg) if os.path.isdir(pkgdir): log_print('Dir for "{}" exists.'.format(pkg)) return True @@ -61,10 +61,10 @@ def ensure_pkg_dir_exists(pkg, pkg_state): return False -def update_pkg_dir(pkg, state): +def update_pkg_dir(pkg, pkg_state, other_state): log_print('Making sure pkg dir for "{}" is up to date...'.format(pkg)) - pkgdir = os.path.join(state['dirs'], pkg) + pkgdir = os.path.join(other_state['clones_dir'], pkg) # fetch all try: subprocess.run( @@ -212,16 +212,16 @@ def update_pkg_dir(pkg, state): 'ERROR: Failed to update pkg dir of "{}".'.format(pkg) ) return False, False - elif state[pkg]["skip_branch_up_to_date"]: + elif pkg_state[pkg]["skip_branch_up_to_date"]: log_print(f'"{pkg}" is up to date') return True, True log_print('Updated pkg dir for "{}"'.format(pkg)) return True, False -def check_pkg_build(pkg, state, editor): +def check_pkg_build(pkg, pkg_state, other_state, editor): """Returns "ok", "not_ok", "abort", or "force_build".""" - pkgdir = os.path.join(state['dirs'], pkg) + pkgdir = os.path.join(other_state['clones_dir'], pkg) log_print('Checking PKGBUILD for "{}"...'.format(pkg)) try: subprocess.run( @@ -243,7 +243,7 @@ def check_pkg_build(pkg, state, editor): return "not_ok" elif user_input == "c": log_print("User will check PKGBUILD again") - return check_pkg_build(pkg, state, editor) + return check_pkg_build(pkg, pkg_state, other_state, editor) elif user_input == "a": return "abort" elif user_input == "f": @@ -255,7 +255,7 @@ def check_pkg_build(pkg, state, editor): continue -def check_pkg_version(pkg, pkg_state, repo, force_check_srcinfo): +def check_pkg_version(pkg, pkg_state, repo, force_check_srcinfo, other_state): """Returns "fail", "install", or "done".""" status, current_epoch, current_version = get_pkg_current_version( pkg, pkg_state, repo @@ -278,13 +278,13 @@ def check_pkg_version(pkg, pkg_state, repo, force_check_srcinfo): ) return get_srcinfo_check_result( - current_epoch, current_version, pkg, force_check_srcinfo, pkg_state + current_epoch, current_version, pkg, force_check_srcinfo, other_state ) -def get_srcinfo_version(pkg, state): +def get_srcinfo_version(pkg, other_state): """Returns (success_bool, pkgepoch, pkgver, pkgrel)""" - if not os.path.exists(os.path.join(state['dirs'], pkg, ".SRCINFO")): + if not os.path.exists(os.path.join(other_state['clones_dir'], pkg, ".SRCINFO")): log_print(f'ERROR: .SRCINFO does not exist for pkg "{pkg}"') return False, None, None, None pkgver_reprog = re.compile("^\\s*pkgver\\s*=\\s*([a-zA-Z0-9._+-]+)\\s*$") @@ -294,7 +294,7 @@ def get_srcinfo_version(pkg, state): pkgrel = "" pkgepoch = "" with open( - os.path.join(state['dirs'], pkg, ".SRCINFO"), encoding="UTF-8" + os.path.join(other_state['clones_dir'], pkg, ".SRCINFO"), encoding="UTF-8" ) as fo: line = fo.readline() while len(line) > 0: @@ -311,9 +311,9 @@ def get_srcinfo_version(pkg, state): return True, pkgepoch, pkgver, pkgrel -def get_pkgbuild_version(pkg, force_check_srcinfo, state): +def get_pkgbuild_version(pkg, force_check_srcinfo, other_state): """Returns (success, epoch, version, release)""" - pkgdir = os.path.join(state['dirs'], pkg) + pkgdir = os.path.join(other_state['clones_dir'], pkg) log_print(f'Getting version of "{pkg}"...') while True and not force_check_srcinfo: log_print("Use .SRCINFO or directly parse PKGBUILD?") @@ -323,7 +323,7 @@ def get_pkgbuild_version(pkg, force_check_srcinfo, state): # TODO support split packages if force_check_srcinfo or user_input == "1": srcinfo_fetch_success, pkgepoch, pkgver, pkgrel = get_srcinfo_version( - pkg, state + pkg, other_state ) if not srcinfo_fetch_success: log_print("ERROR: Failed to get pkg info from .SRCINFO") @@ -400,10 +400,10 @@ def get_pkgbuild_version(pkg, force_check_srcinfo, state): def get_srcinfo_check_result( - current_epoch, current_version, pkg, force_check_srcinfo, state + current_epoch, current_version, pkg, force_check_srcinfo, other_state ): ver_success, pkgepoch, pkgver, pkgrel = get_pkgbuild_version( - pkg, force_check_srcinfo, state + pkg, force_check_srcinfo, other_state ) if ver_success: if current_epoch is None and pkgepoch is not None: @@ -659,10 +659,7 @@ def cleanup_sccache(chroot): def update_pkg_list( pkgs, pkg_state, - chroot, - pkg_out_dir, - repo, - logs_dir, + other_state, no_update, signing_gpg_dir, signing_gpg_key_fp, @@ -676,17 +673,18 @@ def update_pkg_list( log_print("Updating the chroot...") try: subprocess.run( - ["arch-nspawn", "{}/root".format(chroot), "pacman", "-Syu"], + ["arch-nspawn", "{}/root".format(other_state['chroot']), "pacman", "-Syu"], check=True, ) except subprocess.CalledProcessError: log_print("ERROR: Failed to update the chroot") sys.exit(1) for pkg in pkgs: + pkgdir = os.path.join(other_state['clones_dir'], pkg) log_print(f'Building "{pkg}"...') if "ccache_dir" in pkg_state[pkg]: - cleanup_sccache(chroot) - setup_ccache(chroot) + cleanup_sccache(other_state['chroot']) + setup_ccache(other_state['chroot']) else: cleanup_ccache(chroot) if "sccache_dir" in pkg_state[pkg]: @@ -698,7 +696,7 @@ def update_pkg_list( "makechrootpkg", "-c", "-r", - chroot, + other_state['chroot'], ] post_command_list = [ "--", @@ -715,7 +713,7 @@ def update_pkg_list( command_list.insert(1, "-I") command_list.insert(2, dep_fullpath) for aur_dep in pkg_state[pkg]["aur_deps"]: - aur_dep_fullpath = get_latest_pkg(aur_dep, pkg_out_dir) + aur_dep_fullpath = get_latest_pkg(aur_dep,other_state['pkg_out_dir']) if not aur_dep_fullpath: log_print('ERROR: Failed to get aur_dep "{}"'.format(aur_dep)) sys.exit(1) @@ -738,10 +736,10 @@ def update_pkg_list( ) # log_print(f"Using command list: {command_list + post_command_list}") # DEBUG with open( - os.path.join(logs_dir, "{}_stdout_{}".format(pkg, nowstring)), + os.path.join(other_state['logs_dir'], "{}_stdout_{}".format(pkg, nowstring)), "w", ) as log_stdout, open( - os.path.join(logs_dir, "{}_stderr_{}".format(pkg, nowstring)), + os.path.join(other_state['logs_dir'], "{}_stderr_{}".format(pkg, nowstring)), "w", ) as log_stderr: try: @@ -793,7 +791,7 @@ def update_pkg_list( log_print("Adding built pkgs to repo...") try: - command_list = ["repo-add", repo] + command_list = ["repo-add", other_state['repo']] for gpkg in pkg_list: command_list.append(gpkg) subprocess.run(command_list, check=True) @@ -804,13 +802,13 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "add_fail" continue - log_print(f'Signing "{repo}"...') + log_print(f'Signing "{other_state["repo"]}"...') try: subprocess.run( [ "/usr/bin/rm", "-f", - str(os.path.join(pkg_out_dir, f"{repo}.sig")), + str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}.sig")), ] ) subprocess.run( @@ -824,14 +822,14 @@ def update_pkg_list( "--default-key", signing_gpg_key_fp, "--detach-sign", - str(os.path.join(pkg_out_dir, f"{repo}")), + str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}")), ], check=True, input=signing_gpg_pass, text=True, env={"GNUPGHOME": signing_gpg_dir}, ) - repo_sig_name = f"{repo}.sig" + repo_sig_name = f"{other_state['repo']}.sig" if repo_sig_name.rfind("/") != -1: repo_sig_name = repo_sig_name.rsplit(sep="/", maxsplit=1)[1] subprocess.run( @@ -839,27 +837,27 @@ def update_pkg_list( "/usr/bin/ln", "-sf", repo_sig_name, - str(os.path.join(pkg_out_dir, f"{repo}")).removesuffix( + str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}")).removesuffix( ".tar" ) + ".sig", ] ) except subprocess.CalledProcessError: - log_print(f'WARNING: Failed to sign "{repo}"') + log_print(f'WARNING: Failed to sign "{other_state["repo"]}"') pkg_state[pkg]["build_status"] = "success" log_print("Moving pkg to pkgs directory...") for f in pkg_list: log_print(f'Moving "{f}"...') - os.rename(f, os.path.join(pkg_out_dir, os.path.basename(f))) + os.rename(f, os.path.join(other_state['pkg_out_dir'], os.path.basename(f))) sig_name = f + ".sig" if os.path.exists(sig_name): log_print(f'Moving "{sig_name}"...') os.rename( sig_name, - os.path.join(pkg_out_dir, os.path.basename(sig_name)), + os.path.join(other_state['pkg_out_dir'], os.path.basename(sig_name)), ) for pkg in pkgs: @@ -1019,7 +1017,7 @@ if __name__ == "__main__": pkg_state[pkg] = {} pkg_state[pkg]["aur_deps"] = [] other_state['chroot'] = args.chroot - other_state['pkgdir'] = args.pkg_dir + other_state['pkg_out_dir'] = args.pkg_dir other_state['repo'] = args.repo other_state['gpg_home'] = args.gpg_dir other_state['logs_dir'] = args.logs_dir @@ -1079,11 +1077,11 @@ if __name__ == "__main__": else: pkg_state[entry["name"]]["skip_branch_up_to_date"] = False other_state['chroot'] = d["chroot"] - other_state['pkgdir'] = d["pkg_dir"] + other_state['pkg_out_dir'] = d["pkg_out_dir"] other_state['repo'] = d["repo"] other_state['gpg_home'] = d["gpg_dir"] other_state['logs_dir'] = d["logs_dir"] - ohter_state['dirs'] = d["dirs_dir"] + other_state['clones_dir'] = d["clones_dir"] if args_logs_dir is not None: GLOBAL_LOG_FILE = args_logs_dir + "/update.py_logs" log_print( @@ -1102,13 +1100,13 @@ if __name__ == "__main__": del pkg_state[to_remove] if "signing_gpg_dir" in d and not args.no_store: - args_signing_gpg_dir = d["signing_gpg_dir"] - args_signing_gpg_key_fp = d["signing_gpg_key_fp"] - args_signing_gpg_pass = getpass.getpass("gpg signing key pass: ") + other_state['signing_gpg_dir'] = d["signing_gpg_dir"] + other_state['signing_gpg_key_fp'] = d["signing_gpg_key_fp"] + other_state['signing_gpg_pass'] = getpass.getpass("gpg signing key pass: ") if not test_gpg_passphrase( - args_signing_gpg_dir, - args_signing_gpg_key_fp, - args_signing_gpg_pass, + other_state['signing_gpg_dir'], + other_state['signing_gpg_key_fp'], + other_state['signing_gpg_pass'], ): sys.exit(1) if "editor" in d: @@ -1123,20 +1121,20 @@ if __name__ == "__main__": if editor is None: editor = DEFAULT_EDITOR - os.putenv("CHROOT", os.path.realpath(args_chroot)) - os.putenv("GNUPGHOME", os.path.realpath(args_gpg_home)) - if not os.path.exists(args_logs_dir): - os.makedirs(args_logs_dir) - elif not os.path.isdir(args_logs_dir): + os.putenv("CHROOT", os.path.realpath(other_state['chroot'])) + os.putenv("GNUPGHOME", os.path.realpath(other_state['gpg_home'])) + if not os.path.exists(other_state['logs_dir']): + os.makedirs(other_state['logs_dir']) + elif not os.path.isdir(other_state['logs_dir']): log_print( - 'ERROR: logs_dir "{}" must be a directory'.format(args_logs_dir) + 'ERROR: logs_dir "{}" must be a directory'.format(other_state['logs_dir']) ) sys.exit(1) pkg_list = [temp_pkg_name for temp_pkg_name in pkg_state.keys()] i = 0 while i < len(pkg_list): going_back = False - if not ensure_pkg_dir_exists(pkg_list[i], pkg_state): + if not ensure_pkg_dir_exists(pkg_list[i], pkg_state, other_state): print_state_info_and_get_update_list(pkg_state) sys.exit(1) skip = False @@ -1148,7 +1146,7 @@ if __name__ == "__main__": update_pkg_dir_success = False while update_pkg_dir_count < 5: (success, skip_on_same_ver) = update_pkg_dir( - pkg_list[i], pkg_state + pkg_list[i], pkg_state, other_state ) if success: update_pkg_dir_success = True @@ -1162,14 +1160,14 @@ if __name__ == "__main__": sys.exit(1) if skip_on_same_ver: check_pkg_version_result = check_pkg_version( - pkg_list[i], pkg_state, args_repo, True + pkg_list[i], pkg_state, args_repo, True, other_state ) if check_pkg_version_result != "install": log_print(f"Pkg {pkg_list[i]} is up to date, skipping...") pkg_state[pkg_list[i]]["state"] = "up to date" i += 1 continue - check_pkg_build_result = check_pkg_build(pkg_list[i], pkg_state, editor) + check_pkg_build_result = check_pkg_build(pkg_list[i], pkg_state, other_state, editor) if check_pkg_build_result == "ok": pass elif check_pkg_build_result == "not_ok": @@ -1194,7 +1192,7 @@ if __name__ == "__main__": state_result = check_pkg_version_result else: state_result = check_pkg_version( - pkg_list[i], pkg_state, args_repo, False + pkg_list[i], pkg_state, args_repo, False, other_state ) confirm_result_result = confirm_result(pkg_list[i], state_result) if confirm_result_result == "continue": @@ -1235,10 +1233,7 @@ if __name__ == "__main__": update_pkg_list( pkgs_to_update, pkg_state, - os.path.realpath(args_chroot), - os.path.realpath(args_pkg_dir), - os.path.realpath(args_repo), - os.path.realpath(args_logs_dir), + other_state, args.no_update, "" if args.no_store else args_signing_gpg_dir, "" if args.no_store else args_signing_gpg_key_fp, From 8d022beb8a9ae7c25a2f49990605888d9fa61e76 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:15:41 +0900 Subject: [PATCH 03/12] Fixes to README.md and update.py --- README.md | 2 +- update.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6ba4063..00a82e9 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ script to run. Use `/usr/bin/mkarchroot` to create your CHROOT in a directory. - mkarchroot $HOME/mychroot base base-devel ccache sccache + mkarchroot $HOME/mychroot/root base base-devel ccache sccache You must refer to the CHROOT as `$HOME/mychroot` if you used the same name as in the previous example. diff --git a/update.py b/update.py index dfe7f13..55f37a5 100755 --- a/update.py +++ b/update.py @@ -1012,6 +1012,7 @@ if __name__ == "__main__": pkg_state = {} other_state = {} + other_state['logs_dir'] = None if args.pkg and not args.config: for pkg in args.pkg: pkg_state[pkg] = {} @@ -1082,8 +1083,8 @@ if __name__ == "__main__": other_state['gpg_home'] = d["gpg_dir"] other_state['logs_dir'] = d["logs_dir"] other_state['clones_dir'] = d["clones_dir"] - if args_logs_dir is not None: - GLOBAL_LOG_FILE = args_logs_dir + "/update.py_logs" + if other_state['logs_dir'] is not None: + GLOBAL_LOG_FILE = other_state['logs_dir'] + "/update.py_logs" log_print( f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M %Z')}" ) @@ -1160,7 +1161,7 @@ if __name__ == "__main__": sys.exit(1) if skip_on_same_ver: check_pkg_version_result = check_pkg_version( - pkg_list[i], pkg_state, args_repo, True, other_state + pkg_list[i], pkg_state, other_state['repo'], True, other_state ) if check_pkg_version_result != "install": log_print(f"Pkg {pkg_list[i]} is up to date, skipping...") @@ -1192,7 +1193,7 @@ if __name__ == "__main__": state_result = check_pkg_version_result else: state_result = check_pkg_version( - pkg_list[i], pkg_state, args_repo, False, other_state + pkg_list[i], pkg_state, other_state['repo'], False, other_state ) confirm_result_result = confirm_result(pkg_list[i], state_result) if confirm_result_result == "continue": From 3ea3fbfb1c787998a4c76ff6799d1c1db59a4a4a Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:23:07 +0900 Subject: [PATCH 04/12] Fix invalid variable names --- update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/update.py b/update.py index 55f37a5..87492ff 100755 --- a/update.py +++ b/update.py @@ -1236,9 +1236,9 @@ if __name__ == "__main__": pkg_state, other_state, args.no_update, - "" if args.no_store else args_signing_gpg_dir, - "" if args.no_store else args_signing_gpg_key_fp, - "" if args.no_store else args_signing_gpg_pass, + "" if args.no_store else other_state['signing_gpg_dir'], + "" if args.no_store else other_state['signing_gpg_key_fp'], + "" if args.no_store else other_state['signing_gpg_pass'], args.no_store, ) else: From 69e9751be9546c014fc0a4a8b516e3c7c903726b Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:26:27 +0900 Subject: [PATCH 05/12] Fix invalid variable names when building --- update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/update.py b/update.py index 87492ff..c2fa5cd 100755 --- a/update.py +++ b/update.py @@ -686,11 +686,11 @@ def update_pkg_list( cleanup_sccache(other_state['chroot']) setup_ccache(other_state['chroot']) else: - cleanup_ccache(chroot) + cleanup_ccache(other_state['chroot']) if "sccache_dir" in pkg_state[pkg]: - setup_sccache(chroot) + setup_sccache(other_state['chroot']) else: - cleanup_sccache(chroot) + cleanup_sccache(other_state['chroot']) command_list = [ "makechrootpkg", From 616f82e46920e7b3cc63ebdab68d8e8c6a447868 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:51:20 +0900 Subject: [PATCH 06/12] Fix remnants of old usage hardcoding to SCRIPT_DIR --- update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/update.py b/update.py index c2fa5cd..47f4f2e 100755 --- a/update.py +++ b/update.py @@ -16,7 +16,7 @@ import shutil import getpass import tempfile -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +#SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SUDO_PROC = False AUR_GIT_REPO_PATH = "https://aur.archlinux.org" AUR_GIT_REPO_PATH_TEMPLATE = AUR_GIT_REPO_PATH + "/{}.git" @@ -761,7 +761,7 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "success" continue - pkg_list = glob.glob(os.path.join(SCRIPT_DIR, pkg, "*.pkg.tar*")) + pkg_list = glob.glob(os.path.join(other_state['clonesdir'], pkg, "*.pkg.tar*")) log_print("Signing package...") for gpkg in pkg_list: @@ -781,7 +781,7 @@ def update_pkg_list( subprocess.run( command_list, check=True, - cwd=os.path.join(SCRIPT_DIR, pkg), + cwd=os.path.join(other_state['clonesdir'], pkg), input=signing_gpg_pass, text=True, env={"GNUPGHOME": signing_gpg_dir}, From 86216d4553914a10b284b69f8da7731923deccbe Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 16:58:18 +0900 Subject: [PATCH 07/12] Ensure output pkgs dir exists on build --- update.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/update.py b/update.py index 47f4f2e..de908fa 100755 --- a/update.py +++ b/update.py @@ -15,6 +15,7 @@ import time import shutil import getpass import tempfile +from pathlib import Path #SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SUDO_PROC = False @@ -848,6 +849,10 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "success" + log_print("Ensuring pkgs directory exists...") + if not os.path.exists(other_state['pkg_out_dir']): + pkg_out_dir_path = Path(other_state['pkg_out_dir']) + pkg_out_dir_path.mkdir(parents=True) log_print("Moving pkg to pkgs directory...") for f in pkg_list: log_print(f'Moving "{f}"...') From 9574b51ac040a06a1448ad7a437cb43dff8a5938 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 17:01:47 +0900 Subject: [PATCH 08/12] Fix typo --- update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update.py b/update.py index de908fa..6a9a170 100755 --- a/update.py +++ b/update.py @@ -762,7 +762,7 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "success" continue - pkg_list = glob.glob(os.path.join(other_state['clonesdir'], pkg, "*.pkg.tar*")) + pkg_list = glob.glob(os.path.join(other_state['clones_dir'], pkg, "*.pkg.tar*")) log_print("Signing package...") for gpkg in pkg_list: @@ -782,7 +782,7 @@ def update_pkg_list( subprocess.run( command_list, check=True, - cwd=os.path.join(other_state['clonesdir'], pkg), + cwd=os.path.join(other_state['clones_dir'], pkg), input=signing_gpg_pass, text=True, env={"GNUPGHOME": signing_gpg_dir}, From 848d7a970c617ac8487f470b38da0e644cdbcf75 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 17:06:27 +0900 Subject: [PATCH 09/12] Some fixes to example_config.toml --- example_config.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/example_config.toml b/example_config.toml index 14a047b..d56220e 100644 --- a/example_config.toml +++ b/example_config.toml @@ -1,9 +1,11 @@ ########## MANDATORY VARIABLES chroot = "/home/stephen/Downloads/aur/chroot" -# Location to place built packages +# Location to place built packages. pkg_out_dir = "/home/custompkgs" +# It is recommended to put the repo file in the "pkg_out_dir". +# If the tar file doesn't already exist, it will be automatically created. repo = "/home/custompkgs/custom.db.tar" -# Location to clone packages from AUR +# Location to clone packages from AUR. clones_dir = "/home/stephen/Downloads/aur" gpg_dir = "/home/stephen/Downloads/aur/checkingGPG" logs_dir = "/home/stephen/Downloads/aur/logs" From 0938cf934aa94c20d60e905e3f8ae0b134c857c2 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 17:15:52 +0900 Subject: [PATCH 10/12] Add some path validation/dir-creation --- update.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/update.py b/update.py index 6a9a170..9797246 100755 --- a/update.py +++ b/update.py @@ -849,10 +849,6 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "success" - log_print("Ensuring pkgs directory exists...") - if not os.path.exists(other_state['pkg_out_dir']): - pkg_out_dir_path = Path(other_state['pkg_out_dir']) - pkg_out_dir_path.mkdir(parents=True) log_print("Moving pkg to pkgs directory...") for f in pkg_list: log_print(f'Moving "{f}"...') @@ -958,6 +954,30 @@ def test_gpg_passphrase(signing_gpg_dir, signing_key_fp, passphrase): return True +def validate_and_verify_paths(other_state): + if not os.path.exists(other_state['chroot']): + log_print(f"ERROR: chroot at "{other_state['chroot']}" does not exist") + sys.exit(1) + log_print("Ensuring pkgs directory exists...") + if not os.path.exists(other_state['pkg_out_dir']): + pkg_out_dir_path = Path(other_state['pkg_out_dir']) + pkg_out_dir_path.mkdir(parents=True) + if not os.path.exists(other_state['gpg_home']): + log_print(f"ERROR: checkingGPG at "{other_state['gpg_home']}" does not exist") + sys.exit(1) + log_print("Ensuring logs directory exists...") + if other_state['logs_dir'] is None: + log_print('ERROR: "logs_dir" was not specified!') + sys.exit(1) + if not os.path.exists(other_state['logs_dir']): + logs_dir_path = Path(other_state['logs_dir']) + logs_dir_path.mkdir(parents=True) + log_print("Ensuring clones directory exists...") + if not os.path.exists(other_state['clones_dir']): + clones_dir_path = Path(other_state['clones_dir']) + clones_dir_path.mkdir(parents=True) + + if __name__ == "__main__": editor = None parser = argparse.ArgumentParser(description="Update AUR pkgs") @@ -1121,6 +1141,8 @@ if __name__ == "__main__": log_print('ERROR: At least "--config" or "--pkg" must be specified') sys.exit(1) + validate_and_verify_paths(other_state) + if args.editor is not None: editor = args.editor From 3ff09fc7a9da976263bf13d66f9b3364432be5f5 Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 17:17:15 +0900 Subject: [PATCH 11/12] Add path validation for "signingGPG" directory --- update.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/update.py b/update.py index 9797246..067a2ff 100755 --- a/update.py +++ b/update.py @@ -965,6 +965,9 @@ def validate_and_verify_paths(other_state): if not os.path.exists(other_state['gpg_home']): log_print(f"ERROR: checkingGPG at "{other_state['gpg_home']}" does not exist") sys.exit(1) + if 'signing_gpg_dir' in other_state and not os.path.exists(other_state['signing_gpg_dir']): + log_print(f"ERROR: signingGPG at "{other_state['signing_gpg_dir']}" does not exist") + sys.exit(1) log_print("Ensuring logs directory exists...") if other_state['logs_dir'] is None: log_print('ERROR: "logs_dir" was not specified!') From 759d8d1f9dd9b705f762dca6c8c03c3a92db2d8a Mon Sep 17 00:00:00 2001 From: Stephen Seo Date: Sat, 4 Jun 2022 17:22:29 +0900 Subject: [PATCH 12/12] Formatting with python-black --- update.py | 225 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 134 insertions(+), 91 deletions(-) diff --git a/update.py b/update.py index 067a2ff..dcc74b4 100755 --- a/update.py +++ b/update.py @@ -17,7 +17,7 @@ import getpass import tempfile from pathlib import Path -#SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +# SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SUDO_PROC = False AUR_GIT_REPO_PATH = "https://aur.archlinux.org" AUR_GIT_REPO_PATH_TEMPLATE = AUR_GIT_REPO_PATH + "/{}.git" @@ -34,7 +34,7 @@ def log_print(string): def ensure_pkg_dir_exists(pkg, pkg_state, other_state): log_print('Checking that dir for "{}" exists...'.format(pkg)) - pkgdir = os.path.join(other_state['clones_dir'], pkg) + pkgdir = os.path.join(other_state["clones_dir"], pkg) if os.path.isdir(pkgdir): log_print('Dir for "{}" exists.'.format(pkg)) return True @@ -65,7 +65,7 @@ def ensure_pkg_dir_exists(pkg, pkg_state, other_state): def update_pkg_dir(pkg, pkg_state, other_state): log_print('Making sure pkg dir for "{}" is up to date...'.format(pkg)) - pkgdir = os.path.join(other_state['clones_dir'], pkg) + pkgdir = os.path.join(other_state["clones_dir"], pkg) # fetch all try: subprocess.run( @@ -193,9 +193,7 @@ def update_pkg_dir(pkg, pkg_state, other_state): # update current branch if not same commit if current_branch_hash != remote_branch_hash: try: - subprocess.run( - ["git", "pull"], check=True, cwd=pkgdir - ) + subprocess.run(["git", "pull"], check=True, cwd=pkgdir) except subprocess.CalledProcessError: try: subprocess.run( @@ -222,12 +220,10 @@ def update_pkg_dir(pkg, pkg_state, other_state): def check_pkg_build(pkg, pkg_state, other_state, editor): """Returns "ok", "not_ok", "abort", or "force_build".""" - pkgdir = os.path.join(other_state['clones_dir'], pkg) + pkgdir = os.path.join(other_state["clones_dir"], pkg) log_print('Checking PKGBUILD for "{}"...'.format(pkg)) try: - subprocess.run( - [editor, "PKGBUILD"], check=True, cwd=pkgdir - ) + subprocess.run([editor, "PKGBUILD"], check=True, cwd=pkgdir) except subprocess.CalledProcessError: log_print('ERROR: Failed checking PKGBUILD for "{}"'.format(pkg)) return "abort" @@ -285,7 +281,9 @@ def check_pkg_version(pkg, pkg_state, repo, force_check_srcinfo, other_state): def get_srcinfo_version(pkg, other_state): """Returns (success_bool, pkgepoch, pkgver, pkgrel)""" - if not os.path.exists(os.path.join(other_state['clones_dir'], pkg, ".SRCINFO")): + if not os.path.exists( + os.path.join(other_state["clones_dir"], pkg, ".SRCINFO") + ): log_print(f'ERROR: .SRCINFO does not exist for pkg "{pkg}"') return False, None, None, None pkgver_reprog = re.compile("^\\s*pkgver\\s*=\\s*([a-zA-Z0-9._+-]+)\\s*$") @@ -295,7 +293,8 @@ def get_srcinfo_version(pkg, other_state): pkgrel = "" pkgepoch = "" with open( - os.path.join(other_state['clones_dir'], pkg, ".SRCINFO"), encoding="UTF-8" + os.path.join(other_state["clones_dir"], pkg, ".SRCINFO"), + encoding="UTF-8", ) as fo: line = fo.readline() while len(line) > 0: @@ -314,7 +313,7 @@ def get_srcinfo_version(pkg, other_state): def get_pkgbuild_version(pkg, force_check_srcinfo, other_state): """Returns (success, epoch, version, release)""" - pkgdir = os.path.join(other_state['clones_dir'], pkg) + pkgdir = os.path.join(other_state["clones_dir"], pkg) log_print(f'Getting version of "{pkg}"...') while True and not force_check_srcinfo: log_print("Use .SRCINFO or directly parse PKGBUILD?") @@ -341,9 +340,7 @@ def get_pkgbuild_version(pkg, force_check_srcinfo, other_state): ) except subprocess.CalledProcessError: log_print( - 'ERROR: Failed to run "makepkg --nobuild" in "{}".'.format( - pkg - ) + 'ERROR: Failed to run "makepkg --nobuild" in "{}".'.format(pkg) ) if os.path.exists(os.path.join(pkgdir, "src")): shutil.rmtree(os.path.join(pkgdir, "src")) @@ -394,9 +391,7 @@ def get_pkgbuild_version(pkg, force_check_srcinfo, other_state): if pkgver is not None and pkgrel is not None: return True, pkgepoch, pkgver, pkgrel else: - log_print( - 'ERROR: Failed to get PKGBUILD version of "{}".'.format(pkg) - ) + log_print('ERROR: Failed to get PKGBUILD version of "{}".'.format(pkg)) return False, None, None, None @@ -674,30 +669,35 @@ def update_pkg_list( log_print("Updating the chroot...") try: subprocess.run( - ["arch-nspawn", "{}/root".format(other_state['chroot']), "pacman", "-Syu"], + [ + "arch-nspawn", + "{}/root".format(other_state["chroot"]), + "pacman", + "-Syu", + ], check=True, ) except subprocess.CalledProcessError: log_print("ERROR: Failed to update the chroot") sys.exit(1) for pkg in pkgs: - pkgdir = os.path.join(other_state['clones_dir'], pkg) + pkgdir = os.path.join(other_state["clones_dir"], pkg) log_print(f'Building "{pkg}"...') if "ccache_dir" in pkg_state[pkg]: - cleanup_sccache(other_state['chroot']) - setup_ccache(other_state['chroot']) + cleanup_sccache(other_state["chroot"]) + setup_ccache(other_state["chroot"]) else: - cleanup_ccache(other_state['chroot']) + cleanup_ccache(other_state["chroot"]) if "sccache_dir" in pkg_state[pkg]: - setup_sccache(other_state['chroot']) + setup_sccache(other_state["chroot"]) else: - cleanup_sccache(other_state['chroot']) + cleanup_sccache(other_state["chroot"]) command_list = [ "makechrootpkg", "-c", "-r", - other_state['chroot'], + other_state["chroot"], ] post_command_list = [ "--", @@ -714,7 +714,9 @@ def update_pkg_list( command_list.insert(1, "-I") command_list.insert(2, dep_fullpath) for aur_dep in pkg_state[pkg]["aur_deps"]: - aur_dep_fullpath = get_latest_pkg(aur_dep,other_state['pkg_out_dir']) + aur_dep_fullpath = get_latest_pkg( + aur_dep, other_state["pkg_out_dir"] + ) if not aur_dep_fullpath: log_print('ERROR: Failed to get aur_dep "{}"'.format(aur_dep)) sys.exit(1) @@ -737,10 +739,14 @@ def update_pkg_list( ) # log_print(f"Using command list: {command_list + post_command_list}") # DEBUG with open( - os.path.join(other_state['logs_dir'], "{}_stdout_{}".format(pkg, nowstring)), + os.path.join( + other_state["logs_dir"], "{}_stdout_{}".format(pkg, nowstring) + ), "w", ) as log_stdout, open( - os.path.join(other_state['logs_dir'], "{}_stderr_{}".format(pkg, nowstring)), + os.path.join( + other_state["logs_dir"], "{}_stderr_{}".format(pkg, nowstring) + ), "w", ) as log_stderr: try: @@ -762,7 +768,9 @@ def update_pkg_list( pkg_state[pkg]["build_status"] = "success" continue - pkg_list = glob.glob(os.path.join(other_state['clones_dir'], pkg, "*.pkg.tar*")) + pkg_list = glob.glob( + os.path.join(other_state["clones_dir"], pkg, "*.pkg.tar*") + ) log_print("Signing package...") for gpkg in pkg_list: @@ -782,7 +790,7 @@ def update_pkg_list( subprocess.run( command_list, check=True, - cwd=os.path.join(other_state['clones_dir'], pkg), + cwd=os.path.join(other_state["clones_dir"], pkg), input=signing_gpg_pass, text=True, env={"GNUPGHOME": signing_gpg_dir}, @@ -792,7 +800,7 @@ def update_pkg_list( log_print("Adding built pkgs to repo...") try: - command_list = ["repo-add", other_state['repo']] + command_list = ["repo-add", other_state["repo"]] for gpkg in pkg_list: command_list.append(gpkg) subprocess.run(command_list, check=True) @@ -809,7 +817,12 @@ def update_pkg_list( [ "/usr/bin/rm", "-f", - str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}.sig")), + str( + os.path.join( + other_state["pkg_out_dir"], + f"{other_state['repo']}.sig", + ) + ), ] ) subprocess.run( @@ -823,7 +836,11 @@ def update_pkg_list( "--default-key", signing_gpg_key_fp, "--detach-sign", - str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}")), + str( + os.path.join( + other_state["pkg_out_dir"], f"{other_state['repo']}" + ) + ), ], check=True, input=signing_gpg_pass, @@ -838,9 +855,11 @@ def update_pkg_list( "/usr/bin/ln", "-sf", repo_sig_name, - str(os.path.join(other_state['pkg_out_dir'], f"{other_state['repo']}")).removesuffix( - ".tar" - ) + str( + os.path.join( + other_state["pkg_out_dir"], f"{other_state['repo']}" + ) + ).removesuffix(".tar") + ".sig", ] ) @@ -852,13 +871,17 @@ def update_pkg_list( log_print("Moving pkg to pkgs directory...") for f in pkg_list: log_print(f'Moving "{f}"...') - os.rename(f, os.path.join(other_state['pkg_out_dir'], os.path.basename(f))) + os.rename( + f, os.path.join(other_state["pkg_out_dir"], os.path.basename(f)) + ) sig_name = f + ".sig" if os.path.exists(sig_name): log_print(f'Moving "{sig_name}"...') os.rename( sig_name, - os.path.join(other_state['pkg_out_dir'], os.path.basename(sig_name)), + os.path.join( + other_state["pkg_out_dir"], os.path.basename(sig_name) + ), ) for pkg in pkgs: @@ -955,29 +978,37 @@ def test_gpg_passphrase(signing_gpg_dir, signing_key_fp, passphrase): def validate_and_verify_paths(other_state): - if not os.path.exists(other_state['chroot']): - log_print(f"ERROR: chroot at "{other_state['chroot']}" does not exist") + if not os.path.exists(other_state["chroot"]): + log_print( + f"ERROR: chroot at \"{other_state['chroot']}\" does not exist" + ) sys.exit(1) log_print("Ensuring pkgs directory exists...") - if not os.path.exists(other_state['pkg_out_dir']): - pkg_out_dir_path = Path(other_state['pkg_out_dir']) + if not os.path.exists(other_state["pkg_out_dir"]): + pkg_out_dir_path = Path(other_state["pkg_out_dir"]) pkg_out_dir_path.mkdir(parents=True) - if not os.path.exists(other_state['gpg_home']): - log_print(f"ERROR: checkingGPG at "{other_state['gpg_home']}" does not exist") + if not os.path.exists(other_state["gpg_home"]): + log_print( + f"ERROR: checkingGPG at \"{other_state['gpg_home']}\" does not exist" + ) sys.exit(1) - if 'signing_gpg_dir' in other_state and not os.path.exists(other_state['signing_gpg_dir']): - log_print(f"ERROR: signingGPG at "{other_state['signing_gpg_dir']}" does not exist") + if "signing_gpg_dir" in other_state and not os.path.exists( + other_state["signing_gpg_dir"] + ): + log_print( + f"ERROR: signingGPG at \"{other_state['signing_gpg_dir']}\" does not exist" + ) sys.exit(1) log_print("Ensuring logs directory exists...") - if other_state['logs_dir'] is None: + if other_state["logs_dir"] is None: log_print('ERROR: "logs_dir" was not specified!') sys.exit(1) - if not os.path.exists(other_state['logs_dir']): - logs_dir_path = Path(other_state['logs_dir']) + if not os.path.exists(other_state["logs_dir"]): + logs_dir_path = Path(other_state["logs_dir"]) logs_dir_path.mkdir(parents=True) log_print("Ensuring clones directory exists...") - if not os.path.exists(other_state['clones_dir']): - clones_dir_path = Path(other_state['clones_dir']) + if not os.path.exists(other_state["clones_dir"]): + clones_dir_path = Path(other_state["clones_dir"]) clones_dir_path.mkdir(parents=True) @@ -1040,35 +1071,37 @@ if __name__ == "__main__": pkg_state = {} other_state = {} - other_state['logs_dir'] = None + other_state["logs_dir"] = None if args.pkg and not args.config: for pkg in args.pkg: pkg_state[pkg] = {} pkg_state[pkg]["aur_deps"] = [] - other_state['chroot'] = args.chroot - other_state['pkg_out_dir'] = args.pkg_dir - other_state['repo'] = args.repo - other_state['gpg_home'] = args.gpg_dir - other_state['logs_dir'] = args.logs_dir + other_state["chroot"] = args.chroot + other_state["pkg_out_dir"] = args.pkg_dir + other_state["repo"] = args.repo + other_state["gpg_home"] = args.gpg_dir + other_state["logs_dir"] = args.logs_dir if args_logs_dir is not None: GLOBAL_LOG_FILE = args_logs_dir + "/update.py_logs" log_print( f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M %Z')}" ) log_print(f"Set GLOBAL_LOG_FILE to {GLOBAL_LOG_FILE}") - other_state['signing_gpg_dir'] = args.signing_gpg_dir - other_state['signing_gpg_key_fp'] = args.signing_gpg_key_fp + other_state["signing_gpg_dir"] = args.signing_gpg_dir + other_state["signing_gpg_key_fp"] = args.signing_gpg_key_fp if args_signing_gpg_key_fp is None: log_print( 'ERROR: Signing key fingerprint "signing_gpg_key_fp" not present in config' ) sys.exit(1) if args_signing_gpg_dir is not None and not args.no_store: - other_state['signing_gpg_pass'] = getpass.getpass("gpg signing key pass: ") + other_state["signing_gpg_pass"] = getpass.getpass( + "gpg signing key pass: " + ) if not test_gpg_passphrase( - other_state['signing_gpg_dir'], - other_state['signing_gpg_key_fp'], - other_state['signing_gpg_pass'], + other_state["signing_gpg_dir"], + other_state["signing_gpg_key_fp"], + other_state["signing_gpg_pass"], ): sys.exit(1) elif args.config: @@ -1105,14 +1138,14 @@ if __name__ == "__main__": pkg_state[entry["name"]]["skip_branch_up_to_date"] = True else: pkg_state[entry["name"]]["skip_branch_up_to_date"] = False - other_state['chroot'] = d["chroot"] - other_state['pkg_out_dir'] = d["pkg_out_dir"] - other_state['repo'] = d["repo"] - other_state['gpg_home'] = d["gpg_dir"] - other_state['logs_dir'] = d["logs_dir"] - other_state['clones_dir'] = d["clones_dir"] - if other_state['logs_dir'] is not None: - GLOBAL_LOG_FILE = other_state['logs_dir'] + "/update.py_logs" + other_state["chroot"] = d["chroot"] + other_state["pkg_out_dir"] = d["pkg_out_dir"] + other_state["repo"] = d["repo"] + other_state["gpg_home"] = d["gpg_dir"] + other_state["logs_dir"] = d["logs_dir"] + other_state["clones_dir"] = d["clones_dir"] + if other_state["logs_dir"] is not None: + GLOBAL_LOG_FILE = other_state["logs_dir"] + "/update.py_logs" log_print( f"{datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M %Z')}" ) @@ -1129,13 +1162,15 @@ if __name__ == "__main__": del pkg_state[to_remove] if "signing_gpg_dir" in d and not args.no_store: - other_state['signing_gpg_dir'] = d["signing_gpg_dir"] - other_state['signing_gpg_key_fp'] = d["signing_gpg_key_fp"] - other_state['signing_gpg_pass'] = getpass.getpass("gpg signing key pass: ") + other_state["signing_gpg_dir"] = d["signing_gpg_dir"] + other_state["signing_gpg_key_fp"] = d["signing_gpg_key_fp"] + other_state["signing_gpg_pass"] = getpass.getpass( + "gpg signing key pass: " + ) if not test_gpg_passphrase( - other_state['signing_gpg_dir'], - other_state['signing_gpg_key_fp'], - other_state['signing_gpg_pass'], + other_state["signing_gpg_dir"], + other_state["signing_gpg_key_fp"], + other_state["signing_gpg_pass"], ): sys.exit(1) if "editor" in d: @@ -1152,13 +1187,15 @@ if __name__ == "__main__": if editor is None: editor = DEFAULT_EDITOR - os.putenv("CHROOT", os.path.realpath(other_state['chroot'])) - os.putenv("GNUPGHOME", os.path.realpath(other_state['gpg_home'])) - if not os.path.exists(other_state['logs_dir']): - os.makedirs(other_state['logs_dir']) - elif not os.path.isdir(other_state['logs_dir']): + os.putenv("CHROOT", os.path.realpath(other_state["chroot"])) + os.putenv("GNUPGHOME", os.path.realpath(other_state["gpg_home"])) + if not os.path.exists(other_state["logs_dir"]): + os.makedirs(other_state["logs_dir"]) + elif not os.path.isdir(other_state["logs_dir"]): log_print( - 'ERROR: logs_dir "{}" must be a directory'.format(other_state['logs_dir']) + 'ERROR: logs_dir "{}" must be a directory'.format( + other_state["logs_dir"] + ) ) sys.exit(1) pkg_list = [temp_pkg_name for temp_pkg_name in pkg_state.keys()] @@ -1191,14 +1228,16 @@ if __name__ == "__main__": sys.exit(1) if skip_on_same_ver: check_pkg_version_result = check_pkg_version( - pkg_list[i], pkg_state, other_state['repo'], True, other_state + pkg_list[i], pkg_state, other_state["repo"], True, other_state ) if check_pkg_version_result != "install": log_print(f"Pkg {pkg_list[i]} is up to date, skipping...") pkg_state[pkg_list[i]]["state"] = "up to date" i += 1 continue - check_pkg_build_result = check_pkg_build(pkg_list[i], pkg_state, other_state, editor) + check_pkg_build_result = check_pkg_build( + pkg_list[i], pkg_state, other_state, editor + ) if check_pkg_build_result == "ok": pass elif check_pkg_build_result == "not_ok": @@ -1223,7 +1262,11 @@ if __name__ == "__main__": state_result = check_pkg_version_result else: state_result = check_pkg_version( - pkg_list[i], pkg_state, other_state['repo'], False, other_state + pkg_list[i], + pkg_state, + other_state["repo"], + False, + other_state, ) confirm_result_result = confirm_result(pkg_list[i], state_result) if confirm_result_result == "continue": @@ -1266,9 +1309,9 @@ if __name__ == "__main__": pkg_state, other_state, args.no_update, - "" if args.no_store else other_state['signing_gpg_dir'], - "" if args.no_store else other_state['signing_gpg_key_fp'], - "" if args.no_store else other_state['signing_gpg_pass'], + "" if args.no_store else other_state["signing_gpg_dir"], + "" if args.no_store else other_state["signing_gpg_key_fp"], + "" if args.no_store else other_state["signing_gpg_pass"], args.no_store, ) else: