From 6b640dde67767ddcf71308fa98d326779d1f45ea Mon Sep 17 00:00:00 2001 From: sagar Date: Fri, 13 Mar 2026 23:43:06 +0530 Subject: [PATCH 1/4] feat: add progress indicator to repository check loop Repository.check() can run for a long time on large repositories without any visible progress. Add a simple progress indicator to show ongoing activity while objects are being verified. Avoid computing totals with store.list(), since that could be slow for large or remote repositories. fix: #9443 --- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/progress.py | 34 +++++++++++++++++++++++++++++++++- src/borg/repository.py | 7 +++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 12db71b2ac..2759bccafb 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -50,7 +50,7 @@ from .process import daemonize, daemonizing, ThreadRunner from .process import signal_handler, raising_signal_handler, sig_int, ignore_sigint, SigHup, SigTerm from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process -from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage +from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage, ProgressIndicatorObjectCounter from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH diff --git a/src/borg/helpers/progress.py b/src/borg/helpers/progress.py index 600208c6b0..96c26393cb 100644 --- a/src/borg/helpers/progress.py +++ b/src/borg/helpers/progress.py @@ -9,7 +9,7 @@ class ProgressIndicatorBase: LOGGER = "borg.output.progress" - JSON_TYPE: str = None + JSON_TYPE: str = None # type: ignore[assignment] operation_id_counter = 0 @@ -41,6 +41,38 @@ def output(self, msg): self.logger.info(j) +class ProgressIndicatorObjectCounter(ProgressIndicatorBase): + JSON_TYPE = "progress_message" + + def __init__(self, step=1000, msg="%d objects", msgid=None): + """ + Activity-based progress indicator, simply tracking a changing count. + + :param step: step size + :param msg: output message; must contain one %d placeholder for the count. + """ + self.step = step + self.msg = msg + self.trigger_at = step + super().__init__(msgid=msgid) + + def show(self, current=None): + """ + Show and output the progress message if the step condition is met. + + :param current: Set the current counter value. + """ + if current is not None and current >= self.trigger_at: + # adjust trigger_at to the next step threshold + while self.trigger_at <= current: + self.trigger_at += self.step + return self.output(self.msg % current) + + def output(self, message): + j = self.make_json(message=message) + self.logger.info(j) + + class ProgressIndicatorPercent(ProgressIndicatorBase): JSON_TYPE = "progress_percent" diff --git a/src/borg/repository.py b/src/borg/repository.py index a3f8aa8cc2..13a338a453 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -14,7 +14,7 @@ from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Error, ErrorWithTraceback, IntegrityError from .helpers import Location -from .helpers import bin_to_hex, hex_to_bin +from .helpers import bin_to_hex, hex_to_bin, ProgressIndicatorObjectCounter from .storelocking import Lock from .logger import create_logger from .manifest import NoManifestError @@ -317,7 +317,7 @@ def check_object(obj): else: log_error("too small.") - # TODO: progress indicator, ... + pi = ProgressIndicatorObjectCounter(step=1000, msg="Checking objects: %d", msgid="repository.check") partial = bool(max_duration) assert not (repair and partial) mode = "partial" if partial else "full" @@ -364,6 +364,7 @@ def check_object(obj): obj_corrupted = False check_object(obj) objs_checked += 1 + pi.show(objs_checked) if obj_corrupted: objs_errors += 1 if repair: @@ -397,6 +398,7 @@ def check_object(obj): self.store.store(LAST_KEY_CHECKED, key.encode()) break else: + pi.finish() logger.info("Finished repository check.") try: self.store.delete(LAST_KEY_CHECKED) @@ -411,6 +413,7 @@ def check_object(obj): ) except StoreObjectNotFound: # it can be that there is no "data/" at all, then it crashes when iterating infos. + pi.finish() pass logger.info(f"Checked {objs_checked} repository objects, {objs_errors} errors.") if objs_errors == 0: From 52453a7b1cecbf5d27bad6492e1cecd4a235aaf8 Mon Sep 17 00:00:00 2001 From: sagar Date: Sun, 15 Mar 2026 02:34:00 +0530 Subject: [PATCH 2/4] check: refine repository progress indicator implementation. - rename ProgressIndicatorCounter and JSON_TYPE - update message to "Checked objects: %d" - ensure pi.finish() before early break - enable progress output only with --progress --- src/borg/archiver/check_cmd.py | 2 +- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/progress.py | 4 ++-- src/borg/remote.py | 8 ++++++-- src/borg/repository.py | 17 +++++++++++------ 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/borg/archiver/check_cmd.py b/src/borg/archiver/check_cmd.py index 83d1f6e294..fdb8d381fd 100644 --- a/src/borg/archiver/check_cmd.py +++ b/src/borg/archiver/check_cmd.py @@ -52,7 +52,7 @@ def do_check(self, args, repository): except IntegrityError: pass # will try to make key later again if not args.archives_only: - if not repository.check(repair=args.repair, max_duration=args.max_duration): + if not repository.check(repair=args.repair, max_duration=args.max_duration, progress=args.progress): set_ec(EXIT_WARNING) if not args.repo_only and not archive_checker.check( repository, diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 2759bccafb..c3400e38e6 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -50,7 +50,7 @@ from .process import daemonize, daemonizing, ThreadRunner from .process import signal_handler, raising_signal_handler, sig_int, ignore_sigint, SigHup, SigTerm from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process -from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage, ProgressIndicatorObjectCounter +from .progress import ProgressIndicatorPercent, ProgressIndicatorMessage, ProgressIndicatorCounter from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS from .time import format_time, format_timedelta, OutputTimestamp, archive_ts_now from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH diff --git a/src/borg/helpers/progress.py b/src/borg/helpers/progress.py index 96c26393cb..13976917e8 100644 --- a/src/borg/helpers/progress.py +++ b/src/borg/helpers/progress.py @@ -41,8 +41,8 @@ def output(self, msg): self.logger.info(j) -class ProgressIndicatorObjectCounter(ProgressIndicatorBase): - JSON_TYPE = "progress_message" +class ProgressIndicatorCounter(ProgressIndicatorBase): + JSON_TYPE = "progress_counter" def __init__(self, step=1000, msg="%d objects", msgid=None): """ diff --git a/src/borg/remote.py b/src/borg/remote.py index 5fdafb4223..aa0f86c4e9 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -954,8 +954,12 @@ def open(self, path, create=False, lock_wait=None, lock=True, exclusive=False, v def info(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0"), max_duration={"since": parse_version("1.2.0a4"), "previously": 0}) - def check(self, repair=False, max_duration=0): + @api( + since=parse_version("1.0.0"), + max_duration={"since": parse_version("1.2.0a4"), "previously": 0}, + progress={"since": parse_version("never"), "dontcare": True}, + ) + def check(self, repair=False, max_duration=0, progress=False): """actual remoting is done via self.call in the @api decorator""" @api( diff --git a/src/borg/repository.py b/src/borg/repository.py index 13a338a453..3e19784ac2 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -14,7 +14,7 @@ from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Error, ErrorWithTraceback, IntegrityError from .helpers import Location -from .helpers import bin_to_hex, hex_to_bin, ProgressIndicatorObjectCounter +from .helpers import bin_to_hex, hex_to_bin, ProgressIndicatorCounter from .storelocking import Lock from .logger import create_logger from .manifest import NoManifestError @@ -290,7 +290,7 @@ def info(self): info = dict(id=self.id, version=self.version) return info - def check(self, repair=False, max_duration=0): + def check(self, repair=False, max_duration=0, progress=False): """Check repository consistency""" def log_error(msg): @@ -317,7 +317,7 @@ def check_object(obj): else: log_error("too small.") - pi = ProgressIndicatorObjectCounter(step=1000, msg="Checking objects: %d", msgid="repository.check") + pi = ProgressIndicatorCounter(step=1000, msg="Checked objects: %d", msgid="repository.check") if progress else None partial = bool(max_duration) assert not (repair and partial) mode = "partial" if partial else "full" @@ -364,7 +364,8 @@ def check_object(obj): obj_corrupted = False check_object(obj) objs_checked += 1 - pi.show(objs_checked) + if pi: + pi.show(objs_checked) if obj_corrupted: objs_errors += 1 if repair: @@ -394,11 +395,14 @@ def check_object(obj): logger.info(f"Checkpointing at key {key}.") self.store.store(LAST_KEY_CHECKED, key.encode()) if partial and now > t_start + max_duration: + if pi: + pi.finish() logger.info(f"Finished partial repository check, last key checked is {key}.") self.store.store(LAST_KEY_CHECKED, key.encode()) break else: - pi.finish() + if pi: + pi.finish() logger.info("Finished repository check.") try: self.store.delete(LAST_KEY_CHECKED) @@ -413,7 +417,8 @@ def check_object(obj): ) except StoreObjectNotFound: # it can be that there is no "data/" at all, then it crashes when iterating infos. - pi.finish() + if pi: + pi.finish() pass logger.info(f"Checked {objs_checked} repository objects, {objs_errors} errors.") if objs_errors == 0: From b9c31f5e58a560bcd6bd7894a7b97c23c2fa011e Mon Sep 17 00:00:00 2001 From: sagar Date: Thu, 19 Mar 2026 18:57:11 +0530 Subject: [PATCH 3/4] fix ci --- src/borg/repository.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 3e19784ac2..2ffb49d3b4 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -317,7 +317,15 @@ def check_object(obj): else: log_error("too small.") - pi = ProgressIndicatorCounter(step=1000, msg="Checked objects: %d", msgid="repository.check") if progress else None + pi = ( + ProgressIndicatorCounter( + step=1000, + msg="Checked objects: %d", + msgid="repository.check", + ) + if progress + else None + ) partial = bool(max_duration) assert not (repair and partial) mode = "partial" if partial else "full" From 81e11b01e892a583d0b7ab1513e221f333b77316 Mon Sep 17 00:00:00 2001 From: sagar Date: Fri, 20 Mar 2026 18:45:53 +0530 Subject: [PATCH 4/4] fix:resolve PR CI failures by fixing import loop and remote API versioning --- src/borg/remote.py | 3 ++- src/borg/repository.py | 17 +++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index aa0f86c4e9..d15a4a1877 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -957,7 +957,8 @@ def info(self): @api( since=parse_version("1.0.0"), max_duration={"since": parse_version("1.2.0a4"), "previously": 0}, - progress={"since": parse_version("never"), "dontcare": True}, + # NOTE: update version when next beta is released + progress={"since": parse_version("2.0.0b21"), "dontcare": True}, ) def check(self, repair=False, max_duration=0, progress=False): """actual remoting is done via self.call in the @api decorator""" diff --git a/src/borg/repository.py b/src/borg/repository.py index 2ffb49d3b4..337d8fb204 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -14,7 +14,7 @@ from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Error, ErrorWithTraceback, IntegrityError from .helpers import Location -from .helpers import bin_to_hex, hex_to_bin, ProgressIndicatorCounter +from .helpers import bin_to_hex, hex_to_bin from .storelocking import Lock from .logger import create_logger from .manifest import NoManifestError @@ -317,15 +317,12 @@ def check_object(obj): else: log_error("too small.") - pi = ( - ProgressIndicatorCounter( - step=1000, - msg="Checked objects: %d", - msgid="repository.check", - ) - if progress - else None - ) + if progress: + from .helpers.progress import ProgressIndicatorCounter + + pi = ProgressIndicatorCounter(step=1000, msg="Checked objects: %d", msgid="repository.check") + else: + pi = None partial = bool(max_duration) assert not (repair and partial) mode = "partial" if partial else "full"