Merge r8.0 into v2: retry budgets, security audit, and redesign follow-ups

.github/dependabot.yml

@@ -13,7 +13,7 @@ updates:
       timezone: "Asia/Amman"
     cooldown:
       default-days: 3
-      semver-patch-days: 1
+      semver-patch-days: 5
       semver-minor-days: 3
       semver-major-days: 7
     open-pull-requests-limit: 10
@@ -36,7 +36,7 @@ updates:
       timezone: "Asia/Amman"
     cooldown:
       default-days: 3
-      semver-patch-days: 1
+      semver-patch-days: 5
       semver-minor-days: 3
       semver-major-days: 7
     open-pull-requests-limit: 5
@@ -54,7 +54,7 @@ updates:
       timezone: "Asia/Amman"
     cooldown:
       default-days: 3
-      semver-patch-days: 1
+      semver-patch-days: 5
       semver-minor-days: 3
       semver-major-days: 7
     open-pull-requests-limit: 5

SW.Bitween.Api/Controllers/GatewayController.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Net.Mime;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using SW.Bitween.Domain;
@@ -56,6 +57,17 @@ private async Task<IActionResult> ProcessAsync([FromRoute] string gatewayApiName
         if (apiGatewayPartner == null)
             return Unauthorized();
 
+        // After authorisation on purpose: whether a gateway exists and is switched off is
+        // something only an attached partner should learn — checking it earlier would
+        // answer that for anyone who guessed the url.
+        //
+        // 503 rather than 404 because the url is right and the partner should keep it: a
+        // 404 reads as "wrong address" and sends someone hunting for a new one, where this
+        // is a gateway somebody switched off and will switch back on.
+        if (apiGateway.Inactive)
+            return StatusCode(StatusCodes.Status503ServiceUnavailable,
+                $"The '{apiGateway.Name}' gateway is currently deactivated.");
+
         var subscription = await cache.SubscriptionByIdAsync(apiGatewayPartner.SubscriptionId);
 
         if (subscription == null)

SW.Bitween.Api/Data/BitweenDbContext.cs

@@ -229,6 +229,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.Id).ValueGeneratedOnAdd();
                 b.Property(p => p.Name).IsRequired().HasMaxLength(200);
                 b.Property(p => p.Groups).StoreAsJson();
+                b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false);
+                b.Property(p => p.AlertHandlerProperties).StoreAsJson();
             });
 
             modelBuilder.Entity<DelayedRetry>(b =>
@@ -237,10 +239,36 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.HasKey(p => p.Id);
                 b.Property(p => p.Id).IsUnicode(false).HasMaxLength(50);
                 b.Property(p => p.On);
-                b.Property(p => p.GroupAttemptCounts).StoreAsJson();
                 b.HasIndex(p => p.On);
             });
 
+            modelBuilder.Entity<ReceiveAttempt>(b =>
+            {
+                b.ToTable("ReceiveAttempts");
+                b.Property(p => p.Id).ValueGeneratedOnAdd();
+                b.Property(p => p.ErrorMessage).HasMaxLength(4000);
+                b.Property(p => p.ExchangeIds).IsSeparatorDelimited();
+                b.HasIndex(p => new { p.SubscriptionId, p.StartedOn });
+            });
+
+            modelBuilder.Entity<RetryGroupUsage>(b =>
+            {
+                b.ToTable("RetryGroupUsages");
+                b.HasKey(p => new { p.SubscriptionId, p.GroupId });
+                b.Property(p => p.AttemptsUsed);
+                b.Property(p => p.LastAttemptOn);
+                b.Property(p => p.ExhaustedNotifiedOn);
+            });
+
+            modelBuilder.Entity<RetryAlertOverride>(b =>
+            {
+                b.ToTable("RetryAlertOverrides");
+                b.HasKey(p => new { p.SubscriptionId, p.GroupId });
+                b.Property(p => p.AlertMode).HasConversion<byte>();
+                b.Property(p => p.AlertHandlerId).HasMaxLength(200).IsUnicode(false);
+                b.Property(p => p.AlertHandlerProperties).StoreAsJson();
+            });
+
             modelBuilder.Entity<Xchange>(b =>
             {
                 b.ToTable("Xchanges");
@@ -253,7 +281,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.HandlerId).HasMaxLength(200).IsUnicode(false);
                 b.Property(p => p.HandlerProperties).StoreAsJson();
                 b.Property(p => p.MapperProperties).StoreAsJson();
-                b.Property(p => p.GroupAttemptCounts).StoreAsJson();
                 b.Property(p => p.InputContentType).IsUnicode(false).HasMaxLength(200);
                 b.Property(p => p.ResponseMessageTypeName).IsUnicode(false).HasMaxLength(500);
 
@@ -284,6 +311,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.ResponseName).HasMaxLength(200);
                 b.Property(p => p.ResponseContentType).IsUnicode(false).HasMaxLength(200);
                 b.Property(p => p.OutputContentType).IsUnicode(false).HasMaxLength(200);
+                b.Property(p => p.RetryBlockedReason).HasMaxLength(500);
+                b.Property(p => p.RetryGroupId);
+                b.Property(p => p.AttemptNumber);
+                b.HasIndex(p => p.RetryGroupId);
 
 
                 b.HasOne<Xchange>().WithOne().HasForeignKey<XchangeResult>(p => p.Id).OnDelete(DeleteBehavior.Cascade);
@@ -368,7 +399,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                         Disabled = false,
                         Password = defaultPasswordHash,
                         Deleted = false,
-                        Role = AccountRole.Admin
+                        Role = AccountRole.Admin,
+                        FailedLoginCount = 0
                     });
             });
 

SW.Bitween.Api/Domain/Accounts/Account.cs

@@ -28,6 +28,24 @@ public Account(string displayName, string email, string password, AccountRole ro
 
         public string Password { get; set; }
 
+        public int FailedLoginCount { get; private set; }
+        public DateTime? LockoutEnd { get; private set; }
+
+        public bool IsLockedOut(DateTime nowUtc) => LockoutEnd.HasValue && LockoutEnd.Value > nowUtc;
+
+        public void RegisterSuccessfulLogin()
+        {
+            FailedLoginCount = 0;
+            LockoutEnd = null;
+        }
+
+        // Admin action: clear a lockout before it expires.
+        public void Unlock()
+        {
+            FailedLoginCount = 0;
+            LockoutEnd = null;
+        }
+
 
         public bool AddEmailLoginMethod(string email, string password)
         {

SW.Bitween.Api/Domain/DelayedRetry.cs

@@ -1,11 +1,9 @@
 using System;
-using System.Collections.Generic;
 using SW.PrimitiveTypes;
 
 namespace SW.Bitween.Domain;
 // Id should be the same for xchangeId when retry happens the record is deleted
 public class DelayedRetry : BaseEntity<string>
 {
     public DateTime On { get; set; }
-    public Dictionary<string, int> GroupAttemptCounts { get; set; } = new();
 }

SW.Bitween.Api/Domain/Gateway/ApiGateway.cs

@@ -8,6 +8,13 @@ public class ApiGateway : BaseEntity,IAudited
 {
     public string Name { get; set; }
     public string UrlName { get; set; }
+
+    /// <summary>
+    /// Turns the gateway off without deleting it. Deleting is the only alternative today,
+    /// and it takes the partner attachments with it — so a gateway that needs stopping for
+    /// an afternoon gets rebuilt by hand afterwards, or left running.
+    /// </summary>
+    public bool Inactive { get; set; }
     public ICollection<ApiGatewayPartner> Partners { get; set; }
     public DateTime CreatedOn { get; set; }
     public string CreatedBy { get; set; }

SW.Bitween.Api/Domain/Gateway/BusGateway.cs

@@ -8,6 +8,12 @@ public class BusGateway : BaseEntity, IAudited
 {
     public string Name { get; set; }
     public int DocumentId { get; set; }
+
+    /// <summary>
+    /// Turns the gateway off without deleting it — its routes stop being offered the
+    /// message. See <see cref="ApiGateway.Inactive"/>.
+    /// </summary>
+    public bool Inactive { get; set; }
     public ICollection<BusGatewayRoute> Routes { get; set; }
     public DateTime CreatedOn { get; set; }
     public string CreatedBy { get; set; }

SW.Bitween.Api/Domain/ReceiveAttempt.cs

@@ -0,0 +1,20 @@
+using System;
+using SW.Bitween.Model;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// One execution of a Receiving subscription's receive step, written by <c>ReceivingJob</c>
+/// itself right where it already catches the receiver's own failures — kept independent of
+/// Quartz's own run history and unaffected by how Quartz treats a thrown exception.
+/// </summary>
+public class ReceiveAttempt : BaseEntity
+{
+    public int SubscriptionId { get; set; }
+    public DateTime StartedOn { get; set; }
+    public DateTime FinishedOn { get; set; }
+    public ReceiveOutcome Outcome { get; set; }
+    public string ErrorMessage { get; set; }
+    public string[] ExchangeIds { get; set; } = Array.Empty<string>();
+}

SW.Bitween.Api/Domain/RetryAlertOverride.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using SW.Bitween.Model;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// The most specific level of the retry-alert hierarchy: where one subscription's failures in one
+/// retry group should be alerted, overriding whatever the group or the policy says.
+/// </summary>
+/// <remarks>
+/// Deliberately its own table rather than columns on <see cref="RetryGroupUsage"/>. Usage rows are
+/// deleted by <c>RetryPolicies/resetusage</c>, so config stored there would be silently discarded
+/// every time someone cleared a spent budget.
+/// </remarks>
+public class RetryAlertOverride
+{
+    /// <summary>The subscription this override applies to.</summary>
+    public int SubscriptionId { get; set; }
+
+    /// <summary><c>RetryGroup.Id</c>, which survives policy edits, so the override does too.</summary>
+    public Guid GroupId { get; set; }
+
+    /// <summary>
+    /// Whether this level sends, stays silent, or defers upward. A row whose mode is
+    /// <see cref="RetryAlertMode.Inherit"/> is equivalent to having no row at all.
+    /// </summary>
+    public RetryAlertMode AlertMode { get; set; }
+
+    /// <summary>Adapter that delivers the alert. Required when <see cref="AlertMode"/> is Send.</summary>
+    public string AlertHandlerId { get; set; }
+
+    /// <summary>That adapter's own settings — api key, recipients, subject.</summary>
+    public IReadOnlyDictionary<string, string> AlertHandlerProperties { get; set; }
+}

SW.Bitween.Api/Domain/RetryBudgetExhaustedEvent.cs

@@ -0,0 +1,42 @@
+using System;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// Raised the one time a retry group's <c>MaxAttemptsTotal</c> runs out for a subscription, so the
+/// configured alert handler can be told that failures matching that group have stopped being retried.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Deliberately not an <c>IHasWorkGroup</c> event: it publishes under its own type name and is picked
+/// up by a dedicated <c>IConsume&lt;RetryBudgetExhaustedEvent&gt;</c> consumer with its own queue. A
+/// slow or broken alert handler therefore cannot delay or fail the ordinary notifier path, which
+/// shares the work group's result queue.
+/// </para>
+/// <para>
+/// Carried on <see cref="XchangeResult"/> rather than published directly, so it only reaches the bus
+/// once the failure it describes has actually been committed.
+/// </para>
+/// </remarks>
+public class RetryBudgetExhaustedEvent : BaseDomainEvent
+{
+    /// <summary>The failure that found the budget empty.</summary>
+    public string XchangeId { get; set; }
+
+    public int SubscriptionId { get; set; }
+
+    public Guid GroupId { get; set; }
+
+    /// <summary>The group's name as it was when the budget ran out, in case it is later renamed.</summary>
+    public string GroupName { get; set; }
+
+    /// <summary>
+    /// The ceiling that was reached. Not paired with an "used" count, because at exhaustion the two
+    /// are the same number — except when the ceiling was lowered below what had already been spent,
+    /// where the ceiling is still the meaningful figure.
+    /// </summary>
+    public int MaxAttemptsTotal { get; set; }
+
+    public DateTime OccurredOn { get; set; }
+}

SW.Bitween.Api/Domain/RetryGroupUsage.cs

@@ -0,0 +1,41 @@
+using System;
+
+namespace SW.Bitween.Domain;
+
+/// <summary>
+/// Running total of the retries one retry group has spent for one integration, backing
+/// <c>RetryBudget.MaxAttemptsTotal</c>.  That cap is shared by every message hitting the
+/// group, so it cannot be tracked on an individual xchange.
+/// </summary>
+/// <remarks>
+/// Once <see cref="AttemptsUsed"/> reaches the group's <c>MaxAttemptsTotal</c> the group stops
+/// retrying for that integration until this row is cleared. A row that has reached the cap is cleared
+/// by the integration's next success — the only signal that the downstream it was failing against has
+/// recovered — or by one of the reset endpoints. A row still below the cap is left alone by a success:
+/// the cap is there for a downstream that fails some messages and succeeds others, which is exactly
+/// when crediting it back would stop it ever being reached.
+/// </remarks>
+public class RetryGroupUsage
+{
+    /// <summary>The integration whose budget this is. A shared policy gives each one its own total.</summary>
+    public int SubscriptionId { get; set; }
+
+    /// <summary><c>RetryGroup.Id</c>, which survives policy edits, so the total does too.</summary>
+    public Guid GroupId { get; set; }
+
+    public int AttemptsUsed { get; set; }
+
+    /// <summary>When the last attempt was claimed — the only clue left once a group is exhausted.</summary>
+    public DateTime LastAttemptOn { get; set; }
+
+    /// <summary>
+    /// When the exhaustion alert for this integration and group was claimed, or <c>null</c> while
+    /// the budget still has room.
+    /// </summary>
+    /// <remarks>
+    /// Claiming this is what makes the alert fire exactly once: every failure after the budget runs
+    /// out would otherwise raise another one. Reset deletes the whole row, which re-arms the alert
+    /// along with the budget.
+    /// </remarks>
+    public DateTime? ExhaustedNotifiedOn { get; set; }
+}

SW.Bitween.Api/Domain/RetryPolicy.cs

@@ -9,6 +9,15 @@ public class RetryPolicy : BaseEntity, IAudited, IRetryPolicy
 {
     public string Name { get; set; }
     public List<RetryGroup> Groups { get; set; } = [];
+
+    /// <summary>
+    /// Default destination for "retry budget exhausted" alerts, used by every group that does not
+    /// override it. Null means no alert unless a group or a subscription+group override defines one.
+    /// </summary>
+    public string AlertHandlerId { get; set; }
+
+    /// <summary>That adapter's own settings — api key, recipients, subject.</summary>
+    public IReadOnlyDictionary<string, string> AlertHandlerProperties { get; set; }
     public DateTime CreatedOn { get; set; }
     public string CreatedBy { get; set; }
     public DateTime? ModifiedOn { get; set; }