Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions database/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -1243,7 +1243,7 @@ def _schedule_background_reconnect(
kwargs: Dict[str, Any],
mongo_url: Optional[str],
database_name: Optional[str],
delay: float = 30.0,
delay: float = 5.0,
max_bg_attempts: int = 10,
_attempt: int = 1,
) -> None:
Expand Down Expand Up @@ -1293,13 +1293,24 @@ def _try_reconnect():
client.close()
except Exception:
pass
if _attempt < max_bg_attempts:
next_delay = min(delay * 1.5, 300.0)
self._schedule_background_reconnect(
kwargs, mongo_url, database_name,
delay=next_delay, max_bg_attempts=max_bg_attempts,
_attempt=_attempt + 1,
# Never give up: main.py now does a passive wait instead of SystemExit,
# so if we stop scheduling here the app would spin forever without any
# active reconnect attempt. Exponential backoff is capped at 300s, so
# load stays bounded. We emit a one-time escalation event when the
# historical max is crossed, for observability.
if _attempt == max_bg_attempts:
emit_event(
"db_background_reconnect_escalating",
severity="error",
attempt=_attempt,
note="continuing with capped backoff",
)
next_delay = min(delay * 1.5, 300.0)
self._schedule_background_reconnect(
kwargs, mongo_url, database_name,
delay=next_delay, max_bg_attempts=max_bg_attempts,
_attempt=_attempt + 1,
)

timer = threading.Timer(delay, _try_reconnect)
timer.daemon = True
Expand Down
12 changes: 12 additions & 0 deletions docs/environment-variables.rst
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,18 @@
- ``1000``
- ``1500``
- Bot/WebApp
* - ``DB_RECONNECT_WAIT_BEFORE_POLL``
- זמן המתנה ראשוני (שניות) בעלייה של הבוט עד שה-DB חוזר, לפני שעוברים ל-poll פסיבי. הבוט לא יוצא (SystemExit) אחרי החלון — הוא ממשיך להמתין.
- לא
- ``120``
- ``120``
- Bot
* - ``DB_RECONNECT_POLL_INTERVAL``
- מרווח (שניות) בין בדיקות חיבור ב-poll פסיבי לאחר שחלון ההמתנה הראשוני פג. ה-reconnect ברקע ממשיך לנסות עם backoff מעריכי (מקסימום 300 שניות).
- לא
- ``30``
- ``30``
- Bot
* - ``DB_HEALTH_POOL_REFRESH_SEC``
- תדירות רענון מומלצת (שניות) לסטטוס ה-pool בדשבורד. (משתנה תיעודי/קונפיגורציה כללית)
- לא
Expand Down
17 changes: 14 additions & 3 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4917,9 +4917,20 @@ def main() -> None:
time.sleep(5)
_db_waited += 5
if not getattr(db, 'is_connected', True):
logger.critical("DB still unreachable after %ds; exiting to prevent unsafe operation.", _db_wait_max)
raise SystemExit(1)
logger.info("DB reconnected after %ds wait; proceeding with lock acquisition.", _db_waited)
# Passive wait instead of SystemExit: keep the process alive and let the
# background reconnect keep trying. Avoids CrashLoop + Sentry storms when
# the DB has a transient outage.
_poll_interval = int(os.getenv("DB_RECONNECT_POLL_INTERVAL", "30"))
Comment thread
amirbiron marked this conversation as resolved.
logger.warning(
"DB still unreachable after %ds; entering passive wait (polling every %ds).",
_db_wait_max, _poll_interval,
)
while not getattr(db, 'is_connected', True):
time.sleep(_poll_interval)
_db_waited += _poll_interval
Comment thread
cursor[bot] marked this conversation as resolved.
logger.info("DB reconnected after %ds total wait; proceeding with lock acquisition.", _db_waited)
else:
logger.info("DB reconnected after %ds wait; proceeding with lock acquisition.", _db_waited)

# MongoDB connection and lock management
if not manage_mongo_lock():
Expand Down
12 changes: 12 additions & 0 deletions services/config_inspector_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,18 @@ class ConfigService:
category="database",
sensitive=True,
),
"DB_RECONNECT_WAIT_BEFORE_POLL": ConfigDefinition(
key="DB_RECONNECT_WAIT_BEFORE_POLL",
default="120",
description="זמן המתנה ראשוני (שניות) להתחברות מחדש ל-DB בעלייה לפני מעבר ל-poll פסיבי",
category="database",
),
"DB_RECONNECT_POLL_INTERVAL": ConfigDefinition(
key="DB_RECONNECT_POLL_INTERVAL",
default="30",
description="מרווח (שניות) בין בדיקות חיבור ב-poll פסיבי לאחר שחלון ההמתנה הראשוני פג",
category="database",
),
"DB_HEALTH_SLOW_THRESHOLD_MS": ConfigDefinition(
key="DB_HEALTH_SLOW_THRESHOLD_MS",
default="1000",
Expand Down
Loading