Enforce a "log_limit" to limit stdout/stderr logs

Default "log_limit" of 1 GiB for stdout/stderr from building packages
output.

diff --git a/README.md b/README.md
index f9b633a2f304185eaa2f324064ec7295329abe42..9d400b20043124f787c4c6dccdbc38ce6a633ce4 100644 (file)
--- a/README.md
+++ b/README.md
@@ -30,6 +30,16 @@ the TOML config file in a "other\_deps" array for the package like so:
         "mesa"
     ]
 
+## Package stdout/stderr size limit
+
+The possible issue of output logs filling up disk space is addressed with a
+"log\_limit" config option. By default, if the output log file reaches the
+limit, the compilation output is no longer logged to file in the logs dir.
+
+Change "log\_limit" in the config to a value in bytes if the default of 1 GiB
+is too little for your use case (if the size of your output logs extend past 1
+GiB somehow).
+
 # Setting up the AUR Helper
 
 The AUR Helper requires several things:

diff --git a/example_config.toml b/example_config.toml
index 439f997ee1f6fb7bb009395c12f18040a0e41ee5..2246289de75ddbef025631e3d6d053ba48b115d7 100644 (file)
--- a/example_config.toml
+++ b/example_config.toml
@@ -16,6 +16,8 @@ editor = "/usr/bin/vim"
 is_timed = true
 # if true, all output build logs are prepended with current time in UTC
 is_log_timed = true
+# Default log_limit is 1 GiB
+log_limit = 1073741824
 ########## END OF MANDATORY VARIABLES
 
 # Each [[entry]] needs a "name".

diff --git a/update.py b/update.py
index c84af13766ddbc0d7ae5674ca012cc80ebfaa8e8..6c1be0f4146f881dbd3de8dabbec6196e6d7529c 100755 (executable)
--- a/update.py
+++ b/update.py
@@ -1083,7 +1083,25 @@ def cleanup_sccache(chroot: str):
         )
 
 
-def prepend_timestamp_stream(handle, output_file):
+def limited_stream(handle, output_file, log_limit: int):
+    log_count = 0
+    while True:
+        line = handle.readline()
+        if len(line) == 0:
+            break
+        log_count += len(line)
+        if log_count > log_limit:
+            output_file.write(
+                "\nWARNING: Reached log_limit! No longer logging to file!\n"
+            )
+            output_file.flush()
+            break
+        output_file.write(line)
+        output_file.flush()
+
+
+def prepend_timestamp_stream(handle, output_file, log_limit: int):
+    log_count = 0
     while True:
         line = handle.readline()
         if len(line) == 0:
@@ -1091,6 +1109,13 @@ def prepend_timestamp_stream(handle, output_file):
         nowstring = datetime.datetime.now(datetime.timezone.utc).strftime(
             "%Y-%m-%d_%H-%M-%S_%Z "
         )
+        log_count += len(nowstring) + len(line)
+        if log_count > log_limit:
+            output_file.write(
+                "\nWARNING: Reached log_limit! No longer logging to file!\n"
+            )
+            output_file.flush()
+            break
         output_file.write(nowstring + line)
         output_file.flush()
 
@@ -1223,54 +1248,44 @@ def update_pkg_list(
             encoding="utf-8",
         ) as log_stderr:
             try:
-                if other_state["is_log_timed"]:
-                    p1 = subprocess.Popen(
-                        command_list + post_command_list,
-                        cwd=pkgdir,
-                        text=True,
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                    )
-                    tout = threading.Thread(
-                        target=prepend_timestamp_stream,
-                        args=[p1.stdout, log_stdout],
-                    )
-                    terr = threading.Thread(
-                        target=prepend_timestamp_stream,
-                        args=[p1.stderr, log_stderr],
-                    )
+                p1 = subprocess.Popen(
+                    command_list + post_command_list,
+                    cwd=pkgdir,
+                    text=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+                tout = threading.Thread(
+                    target=prepend_timestamp_stream
+                    if other_state["is_log_timed"]
+                    else limited_stream,
+                    args=[p1.stdout, log_stdout, other_state["log_limit"]],
+                )
+                terr = threading.Thread(
+                    target=prepend_timestamp_stream
+                    if other_state["is_log_timed"]
+                    else limited_stream,
+                    args=[p1.stderr, log_stderr, other_state["log_limit"]],
+                )
 
-                    tout.start()
-                    terr.start()
+                tout.start()
+                terr.start()
 
-                    p1.wait()
-                    tout.join()
-                    terr.join()
+                p1.wait()
+                tout.join()
+                terr.join()
 
-                    if (
-                        p1.returncode is None
-                        or type(p1.returncode) is not int
-                        or p1.returncode != 0
-                    ):
-                        raise RuntimeError("pOpen process failed")
-                else:
-                    subprocess.run(
-                        command_list + post_command_list,
-                        check=True,
-                        cwd=pkgdir,
-                        stdout=log_stdout,
-                        stderr=log_stderr,
+                if p1.returncode is None:
+                    raise RuntimeError("pOpen process didn't finish")
+                elif type(p1.returncode) is not int:
+                    raise RuntimeError("pOpen process non-integer returncode")
+                elif p1.returncode != 0:
+                    raise RuntimeError(
+                        f"pOpen process non-zero return code {p1.returncode}"
                     )
-            except subprocess.CalledProcessError:
-                log_print(
-                    'ERROR: Failed to build pkg "{}" in chroot'.format(pkg),
-                    other_state=other_state,
-                )
-                pkg_state[pkg]["build_status"] = "fail"
-                continue
             except BaseException as e:
                 log_print(
-                    'ERROR: Failed to build pkg "{}" in chroot (unknown Exception): {}'.format(
+                    'ERROR: Failed to build pkg "{}" in chroot: {}'.format(
                         pkg, e
                     ),
                     other_state=other_state,
@@ -1638,6 +1653,7 @@ if __name__ == "__main__":
     pkg_state = {}
     other_state = {}
     other_state["logs_dir"] = None
+    other_state["log_limit"] = 1024 * 1024 * 1024
     if args.pkg and not args.config:
         for pkg in args.pkg:
             pkg_state[pkg] = {}
@@ -1773,6 +1789,21 @@ if __name__ == "__main__":
             other_state["is_log_timed"] = True
         else:
             other_state["is_log_timed"] = False
+        if (
+            "log_limit" in d
+            and type(d["log_limit"]) is int
+            and d["log_limit"] > 0
+        ):
+            other_state["log_limit"] = d["log_limit"]
+            log_print('Set "log_limit" to {}'.format(d["log_limit"]))
+        else:
+            log_print(
+                'Using default "log_limit" of {}'.format(
+                    other_state["log_limit"]
+                )
+            )
+        log_print("  {} KiB".format(other_state["log_limit"] / 1024))
+        log_print("  {} MiB".format(other_state["log_limit"] / 1024 / 1024))
     else:
         log_print(
             'ERROR: At least "--config" or "--pkg" must be specified',