From 8ed46c26d725273a8211d388c95ea393445e8af1 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Wed, 8 Jul 2026 11:30:27 +0300 Subject: [PATCH 1/2] feat: add native S3 upload handler and receiver Port the serverless S3 handler and receiver into in-process native adapters (NativeS3UploadHandler, NativeS3Receiver), using the same SimplyWorks.CloudFiles.S3 library already referenced by Bitween-api. - Receiver's DeleteFile now always removes the fetched object; the original only deleted when DeleteMovesFileTo was set, so files were reprocessed on every poll cycle by default - DeleteMovesFileTo is reinterpreted as a copy to another key/prefix in the same bucket; the original wrote to local disk, which has no meaning for an in-process API pod - Receiver's ContentType field renamed to ResponseEncoding (utf8/base64) to match the FTP/Pop3 native adapters, since it was actually selecting response encoding, not a MIME content type - Fields renamed for clarity (Url -> ServiceUrl, TargetPath -> BucketName) to match the underlying CloudFilesOptions shape --- .../S3Receiver/NativeS3Receiver.cs | 82 +++++++++++++++++++ .../S3Receiver/S3ReceiverInput.cs | 30 +++++++ .../S3UploadHandler/NativeS3UploadHandler.cs | 46 +++++++++++ .../S3UploadHandler/S3UploadHandlerInput.cs | 29 +++++++ .../SW.Bitween.NativeAdapters.csproj | 1 + .../ServiceCollectionExtensions.cs | 8 ++ 6 files changed, 196 insertions(+) create mode 100644 SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs create mode 100644 SW.Bitween.NativeAdapters/S3Receiver/S3ReceiverInput.cs create mode 100644 SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs create mode 100644 SW.Bitween.NativeAdapters/S3UploadHandler/S3UploadHandlerInput.cs diff --git a/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs b/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs new file mode 100644 index 00000000..bbc8dc35 --- /dev/null +++ b/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs @@ -0,0 +1,82 @@ +using System.Text; +using SW.CloudFiles.S3; +using SW.PrimitiveTypes; + +namespace SW.Bitween.NativeAdapters.S3Receiver; + +public class NativeS3Receiver : INativeInfolinkReceiver +{ + private S3ReceiverInput _options = new(); + private CloudFilesService _cloudFiles = null!; + + public Task Initialize() + { + _cloudFiles = new CloudFilesService(new CloudFilesOptions + { + AccessKeyId = _options.AccessKeyId, + SecretAccessKey = _options.SecretAccessKey, + ServiceUrl = _options.ServiceUrl, + BucketName = _options.BucketName, + }); + + return Task.CompletedTask; + } + + public Task Finalize() + { + _cloudFiles.Dispose(); + return Task.CompletedTask; + } + + public async Task> ListFiles() + { + var files = await _cloudFiles.ListAsync(_options.FolderName ?? string.Empty); + + return files + .Where(f => !f.Key.EndsWith("/")) + .Select(f => f.Key) + .Take(_options.BatchSize) + .ToList(); + } + + public async Task GetFile(string fileId) + { + await using var stream = await _cloudFiles.OpenReadAsync(fileId); + using var memoryStream = new MemoryStream(); + await stream.CopyToAsync(memoryStream); + var bytes = memoryStream.ToArray(); + + return _options.ResponseEncoding.ToLower() switch + { + "base64" => new XchangeFile(Convert.ToBase64String(bytes), fileId), + "utf8" => new XchangeFile(Encoding.UTF8.GetString(bytes), fileId), + _ => throw new ArgumentException( + $"Unknown {nameof(S3ReceiverInput.ResponseEncoding)} '{_options.ResponseEncoding}'") + }; + } + + public async Task DeleteFile(string fileId) + { + if (!string.IsNullOrWhiteSpace(_options.DeleteMovesFileTo)) + { + await using var sourceStream = await _cloudFiles.OpenReadAsync(fileId); + using var buffer = new MemoryStream(); + await sourceStream.CopyToAsync(buffer); + buffer.Position = 0; + + var targetKey = $"{_options.DeleteMovesFileTo}/{Path.GetFileName(fileId)}"; + await _cloudFiles.WriteAsync(buffer, new WriteFileSettings { Key = targetKey }); + } + + await _cloudFiles.DeleteAsync(fileId); + } + + public string Name => "NativeS3Receiver"; + + public void InitializeStartupValues(IDictionary settings) + { + _options = settings.ConvertTo(); + } + + public Type StartupValuesType => typeof(S3ReceiverInput); +} diff --git a/SW.Bitween.NativeAdapters/S3Receiver/S3ReceiverInput.cs b/SW.Bitween.NativeAdapters/S3Receiver/S3ReceiverInput.cs new file mode 100644 index 00000000..a97bfe3d --- /dev/null +++ b/SW.Bitween.NativeAdapters/S3Receiver/S3ReceiverInput.cs @@ -0,0 +1,30 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; + +namespace SW.Bitween.NativeAdapters.S3Receiver; + +public class S3ReceiverInput +{ + [Required] + public string AccessKeyId { get; set; } = string.Empty; + + [Required] + [Secure] + public string SecretAccessKey { get; set; } = string.Empty; + + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; + + public string? FolderName { get; set; } + + [DefaultValue(50)] + public int BatchSize { get; set; } = 50; + + [DefaultValue("utf8")] + public string ResponseEncoding { get; set; } = "utf8"; + + public string? DeleteMovesFileTo { get; set; } +} diff --git a/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs b/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs new file mode 100644 index 00000000..015c865f --- /dev/null +++ b/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs @@ -0,0 +1,46 @@ +using SW.CloudFiles.S3; +using SW.PrimitiveTypes; + +namespace SW.Bitween.NativeAdapters.S3UploadHandler; + +public class NativeS3UploadHandler : INativeInfolinkHandler +{ + private S3UploadHandlerInput _options = new(); + + public async Task Handle(XchangeFile xchangeFile) + { + using var cloudFiles = new CloudFilesService(new CloudFilesOptions + { + AccessKeyId = _options.AccessKeyId, + SecretAccessKey = _options.SecretAccessKey, + ServiceUrl = _options.ServiceUrl, + BucketName = _options.BucketName, + }); + + var key = _options.FileName; + if (string.IsNullOrWhiteSpace(key)) + { + key = $"{DateTime.UtcNow:yyyyMMddHHmmss}.{_options.FileExtension}"; + + if (!string.IsNullOrWhiteSpace(_options.FolderName)) + key = $"{_options.FolderName}/{key}"; + } + + await cloudFiles.WriteTextAsync(xchangeFile.Data, new WriteFileSettings + { + Key = key, + ContentType = _options.ContentType, + }); + + return new XchangeFile(key, xchangeFile.Filename); + } + + public string Name => "NativeS3UploadHandler"; + + public void InitializeStartupValues(IDictionary settings) + { + _options = settings.ConvertTo(); + } + + public Type StartupValuesType => typeof(S3UploadHandlerInput); +} diff --git a/SW.Bitween.NativeAdapters/S3UploadHandler/S3UploadHandlerInput.cs b/SW.Bitween.NativeAdapters/S3UploadHandler/S3UploadHandlerInput.cs new file mode 100644 index 00000000..eb9bbce2 --- /dev/null +++ b/SW.Bitween.NativeAdapters/S3UploadHandler/S3UploadHandlerInput.cs @@ -0,0 +1,29 @@ +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; + +namespace SW.Bitween.NativeAdapters.S3UploadHandler; + +public class S3UploadHandlerInput +{ + [Required] + public string AccessKeyId { get; set; } = string.Empty; + + [Required] + [Secure] + public string SecretAccessKey { get; set; } = string.Empty; + + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; + + public string? FolderName { get; set; } + + public string? FileName { get; set; } + + public string? FileExtension { get; set; } + + [DefaultValue("text/plain")] + public string ContentType { get; set; } = "text/plain"; +} diff --git a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj index 561103de..db3cb05e 100644 --- a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj +++ b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj @@ -22,6 +22,7 @@ + diff --git a/SW.Bitween.NativeAdapters/ServiceCollectionExtensions.cs b/SW.Bitween.NativeAdapters/ServiceCollectionExtensions.cs index aaae520c..a4189224 100644 --- a/SW.Bitween.NativeAdapters/ServiceCollectionExtensions.cs +++ b/SW.Bitween.NativeAdapters/ServiceCollectionExtensions.cs @@ -4,6 +4,8 @@ using SW.Bitween.NativeAdapters.RebexFtpReceiver; using SW.Bitween.NativeAdapters.RebexFtpUploadHandler; using SW.Bitween.NativeAdapters.RebexPop3Receiver; +using SW.Bitween.NativeAdapters.S3Receiver; +using SW.Bitween.NativeAdapters.S3UploadHandler; namespace SW.Bitween.NativeAdapters; @@ -38,6 +40,12 @@ public static void AddNativeAdapters(this IServiceCollection serviceCollection, serviceCollection.AddScoped(); serviceCollection.AddScoped(); + serviceCollection.AddScoped(); + serviceCollection.AddScoped(); + + serviceCollection.AddScoped(); + serviceCollection.AddScoped(); + if (!string.IsNullOrEmpty(rebexLicenseKey)) { serviceCollection.AddScoped(_ => new NativeRebexPop3Receiver(rebexLicenseKey)); From b88b006cfda1a33047d38033449bb23ff6eda5d9 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Wed, 8 Jul 2026 12:50:08 +0300 Subject: [PATCH 2/2] fix: harden native S3 adapters per code review Guard against null encoding, use S3 server-side copy instead of buffering in memory, avoid filename collisions on move and upload, and dispose the S3 client reliably via IDisposable. --- .../S3Receiver/NativeS3Receiver.cs | 41 +++++++++++++------ .../S3UploadHandler/NativeS3UploadHandler.cs | 4 +- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs b/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs index bbc8dc35..05148909 100644 --- a/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs +++ b/SW.Bitween.NativeAdapters/S3Receiver/NativeS3Receiver.cs @@ -1,33 +1,47 @@ using System.Text; +using Amazon.S3; using SW.CloudFiles.S3; using SW.PrimitiveTypes; namespace SW.Bitween.NativeAdapters.S3Receiver; -public class NativeS3Receiver : INativeInfolinkReceiver +public class NativeS3Receiver : INativeInfolinkReceiver, IDisposable { private S3ReceiverInput _options = new(); - private CloudFilesService _cloudFiles = null!; + private CloudFilesService? _cloudFiles; + private AmazonS3Client? _s3Client; public Task Initialize() { - _cloudFiles = new CloudFilesService(new CloudFilesOptions + var options = new CloudFilesOptions { AccessKeyId = _options.AccessKeyId, SecretAccessKey = _options.SecretAccessKey, ServiceUrl = _options.ServiceUrl, BucketName = _options.BucketName, - }); + }; + + _cloudFiles = new CloudFilesService(options); + _s3Client = options.CreateClient(); return Task.CompletedTask; } public Task Finalize() { - _cloudFiles.Dispose(); + _cloudFiles?.Dispose(); + _s3Client?.Dispose(); return Task.CompletedTask; } + // Safety net: the DI container disposes scoped instances when the job's scope ends, + // even if Initialize/ListFiles/GetFile/DeleteFile threw and Finalize was never reached. + public void Dispose() + { + _cloudFiles?.Dispose(); + _s3Client?.Dispose(); + } + public async Task> ListFiles() { var files = await _cloudFiles.ListAsync(_options.FolderName ?? string.Empty); @@ -46,7 +60,7 @@ public async Task GetFile(string fileId) await stream.CopyToAsync(memoryStream); var bytes = memoryStream.ToArray(); - return _options.ResponseEncoding.ToLower() switch + return (_options.ResponseEncoding ?? "utf8").ToLower() switch { "base64" => new XchangeFile(Convert.ToBase64String(bytes), fileId), "utf8" => new XchangeFile(Encoding.UTF8.GetString(bytes), fileId), @@ -59,13 +73,16 @@ public async Task DeleteFile(string fileId) { if (!string.IsNullOrWhiteSpace(_options.DeleteMovesFileTo)) { - await using var sourceStream = await _cloudFiles.OpenReadAsync(fileId); - using var buffer = new MemoryStream(); - await sourceStream.CopyToAsync(buffer); - buffer.Position = 0; + // Preserve the path relative to FolderName so files with the same name in + // different subdirectories don't collide at the destination. + var relativePath = !string.IsNullOrEmpty(_options.FolderName) && fileId.StartsWith(_options.FolderName + "/") + ? fileId[(_options.FolderName.Length + 1)..] + : fileId; - var targetKey = $"{_options.DeleteMovesFileTo}/{Path.GetFileName(fileId)}"; - await _cloudFiles.WriteAsync(buffer, new WriteFileSettings { Key = targetKey }); + // Server-side copy: S3 moves the object internally, so no bytes are + // downloaded or re-uploaded through this process. + var targetKey = $"{_options.DeleteMovesFileTo}/{relativePath}"; + await _s3Client!.CopyObjectAsync(_options.BucketName, fileId, _options.BucketName, targetKey); } await _cloudFiles.DeleteAsync(fileId); diff --git a/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs b/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs index 015c865f..674d88bd 100644 --- a/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs +++ b/SW.Bitween.NativeAdapters/S3UploadHandler/NativeS3UploadHandler.cs @@ -20,7 +20,9 @@ public async Task Handle(XchangeFile xchangeFile) var key = _options.FileName; if (string.IsNullOrWhiteSpace(key)) { - key = $"{DateTime.UtcNow:yyyyMMddHHmmss}.{_options.FileExtension}"; + var extension = _options.FileExtension?.TrimStart('.'); + var name = $"{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + key = string.IsNullOrEmpty(extension) ? name : $"{name}.{extension}"; if (!string.IsNullOrWhiteSpace(_options.FolderName)) key = $"{_options.FolderName}/{key}";