feat: implement async background post content extraction to save HTML in blob storage (#185)

Co-authored-by: fboucher-os <fboucheros+github@gmail.com>

Author: fboucher

src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs

@@ -0,0 +1,74 @@
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+using NoteBookmark.Api.Tests.Fixtures;
+using NoteBookmark.Domain;
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Threading.Tasks;
+using Xunit;
+using Azure.Storage.Blobs;
+
+namespace NoteBookmark.Api.Tests.Endpoints;
+
+public class PostExtractionTests : IClassFixture<NoteBookmarkApiTestFactory>
+{
+    private readonly NoteBookmarkApiTestFactory _factory;
+    private readonly HttpClient _client;
+
+    public PostExtractionTests(NoteBookmarkApiTestFactory factory)
+    {
+        _factory = factory;
+        _client = _factory.CreateClient();
+    }
+
+    [Fact]
+    public async Task ExtractPostDetails_TriggersBackgroundWorkerAndSavesHtmlToBlobStorage()
+    {
+        // Arrange
+        var url = "https://example.com/blog/test-post-" + Guid.NewGuid();
+        var extractRequest = new
+        {
+            url = url,
+            tags = "test",
+            category = "Test"
+        };
+
+        // Act - Call the API to extract metadata and save the post
+        var response = await _client.PostAsJsonAsync("/api/posts/extractPostDetails", extractRequest);
+
+        // Assert API response is OK
+        response.StatusCode.Should().Be(HttpStatusCode.OK);
+        
+        var post = await response.Content.ReadFromJsonAsync<Post>();
+        post.Should().NotBeNull();
+        var postId = post!.Id ?? post.RowKey;
+        postId.Should().NotBeNullOrEmpty();
+
+        // Since the extraction happens asynchronously in a BackgroundWorker,
+        // we poll the blob storage for a short time to verify the file was created.
+        var blobServiceClient = _factory.Services.GetRequiredService<BlobServiceClient>();
+        var containerClient = blobServiceClient.GetBlobContainerClient("cleanedposts");
+        var blobClient = containerClient.GetBlobClient($"{postId}.html");
+
+        // Wait up to 5 seconds for the background worker to process
+        bool blobExists = false;
+        for (int i = 0; i < 25; i++)
+        {
+            if (await blobClient.ExistsAsync())
+            {
+                blobExists = true;
+                break;
+            }
+            await Task.Delay(200);
+        }
+
+        blobExists.Should().BeTrue("HTML content should be processed by the background worker and saved to blob storage");
+
+        // Verify the content saved matches the fake content
+        var downloadResult = await blobClient.DownloadContentAsync();
+        var content = downloadResult.Value.Content.ToString();
+        content.Should().Contain(url);
+    }
+}

src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs

@@ -0,0 +1,13 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NoteBookmark.Api.Tests.Fixtures;
+
+public class FakePostParserClient : IPostParserClient
+{
+    public Task<string?> ExtractContentAsync(string url, CancellationToken cancellationToken = default)
+    {
+        // Return a mock HTML snippet for testing
+        return Task.FromResult<string?>($"<div>Extracted HTML content for {url}</div>");
+    }
+}

src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs

@@ -34,6 +34,9 @@ protected override void ConfigureWebHost(IWebHostBuilder builder)
                 services.AddSingleton(new TableServiceClient(connectionString));
                 services.AddSingleton(new BlobServiceClient(connectionString));
             }
+
+            // Register FakePostParserClient for integration tests
+            services.AddSingleton<NoteBookmark.Api.IPostParserClient, FakePostParserClient>();
         });
     }
 

src/NoteBookmark.Api/IPostParserClient.cs

@@ -0,0 +1,9 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NoteBookmark.Api;
+
+public interface IPostParserClient
+{
+    Task<string?> ExtractContentAsync(string url, CancellationToken cancellationToken = default);
+}

src/NoteBookmark.Api/PostEndpoints.cs

@@ -94,7 +94,11 @@ static Results<Ok, BadRequest> SavePost(Post post, TableServiceClient tblClient,
 		}
 		return TypedResults.BadRequest();
 	}
-	static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(ExtractPostRequest request, TableServiceClient tblClient, BlobServiceClient blobClient)
+	static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(
+		ExtractPostRequest request,
+		TableServiceClient tblClient,
+		BlobServiceClient blobClient,
+		PostExtractionQueue queue)
 	{
 		var dataStorageService = new DataStorageService(tblClient, blobClient);
 
@@ -105,6 +109,10 @@ static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(ExtractPostR
 			if (post != null)
 			{
 				dataStorageService.SavePost(post);
+				
+				// Queue background HTML extraction task
+				queue.QueueBackgroundWorkItem(new ExtractionTask(post.Id ?? post.RowKey, post.Url ?? decodeUrl));
+
 				return TypedResults.Ok(post);
 			}
 			return TypedResults.BadRequest();

src/NoteBookmark.Api/PostExtractionBackgroundWorker.cs

@@ -0,0 +1,87 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace NoteBookmark.Api;
+
+public class PostExtractionBackgroundWorker : BackgroundService
+{
+    private readonly PostExtractionQueue _queue;
+    private readonly IServiceProvider _serviceProvider;
+    private readonly ILogger<PostExtractionBackgroundWorker> _logger;
+
+    public PostExtractionBackgroundWorker(
+        PostExtractionQueue queue,
+        IServiceProvider serviceProvider,
+        ILogger<PostExtractionBackgroundWorker> logger)
+    {
+        _queue = queue;
+        _serviceProvider = serviceProvider;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("Post Extraction Background Worker started.");
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                var task = await _queue.DequeueAsync(stoppingToken);
+                _logger.LogInformation("Processing extraction for Post: {PostId}, URL: {Url}", task.PostId, task.Url);
+
+                await ProcessExtractionAsync(task, stoppingToken);
+            }
+            catch (OperationCanceledException)
+            {
+                // Normal shutdown
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error occurred executing background extraction task.");
+            }
+        }
+
+        _logger.LogInformation("Post Extraction Background Worker stopped.");
+    }
+
+    private async Task ProcessExtractionAsync(ExtractionTask task, CancellationToken cancellationToken)
+    {
+        using var scope = _serviceProvider.CreateScope();
+        var parserClient = scope.ServiceProvider.GetRequiredService<IPostParserClient>();
+        var blobServiceClient = scope.ServiceProvider.GetRequiredService<BlobServiceClient>();
+
+        try
+        {
+            var content = await parserClient.ExtractContentAsync(task.Url, cancellationToken);
+            if (string.IsNullOrEmpty(content))
+            {
+                _logger.LogWarning("No content returned for URL: {Url}. Skipping blob upload.", task.Url);
+                return;
+            }
+
+            var containerClient = blobServiceClient.GetBlobContainerClient("cleanedposts");
+            await containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken);
+
+            var blobClient = containerClient.GetBlobClient($"{task.PostId}.html");
+            
+            byte[] contentBytes = Encoding.UTF8.GetBytes(content);
+            using var stream = new MemoryStream(contentBytes);
+            
+            await blobClient.UploadAsync(stream, overwrite: true, cancellationToken: cancellationToken);
+            _logger.LogInformation("Successfully saved extracted HTML for Post {PostId} to Blob Storage.", task.PostId);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to process extraction for Post {PostId} / URL: {Url}", task.PostId, task.Url);
+        }
+    }
+}

src/NoteBookmark.Api/PostExtractionQueue.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Threading;
+using System.Threading.Channels;
+using System.Threading.Tasks;
+
+namespace NoteBookmark.Api;
+
+public record ExtractionTask(string PostId, string Url);
+
+public class PostExtractionQueue
+{
+    private readonly Channel<ExtractionTask> _queue;
+
+    public PostExtractionQueue()
+    {
+        // Unbounded channel is simple and suitable for this task queue.
+        _queue = Channel.CreateUnbounded<ExtractionTask>(new UnboundedChannelOptions
+        {
+            SingleReader = true,
+            SingleWriter = false
+        });
+    }
+
+    public void QueueBackgroundWorkItem(ExtractionTask task)
+    {
+        ArgumentNullException.ThrowIfNull(task);
+        _queue.Writer.TryWrite(task);
+    }
+
+    public async ValueTask<ExtractionTask> DequeueAsync(CancellationToken cancellationToken)
+    {
+        return await _queue.Reader.ReadAsync(cancellationToken);
+    }
+}

src/NoteBookmark.Api/PostParserClient.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Text.Json.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+
+namespace NoteBookmark.Api;
+
+public class PostParserClient : IPostParserClient
+{
+    private readonly HttpClient _httpClient;
+    private readonly ILogger<PostParserClient> _logger;
+
+    public PostParserClient(HttpClient httpClient, ILogger<PostParserClient> logger)
+    {
+        _httpClient = httpClient;
+        _logger = logger;
+        // Configure base address or default headers if needed, but since URL is fully specified we can just configure it or call it directly.
+        if (_httpClient.BaseAddress == null)
+        {
+            _httpClient.BaseAddress = new Uri("https://azpostlight-parser.azurewebsites.net/");
+        }
+    }
+
+    public async Task<string?> ExtractContentAsync(string url, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            _logger.LogInformation("Calling parser API for URL: {Url}", url);
+            var requestBody = new { url = url };
+            var response = await _httpClient.PostAsJsonAsync("parser", requestBody, cancellationToken);
+            
+            if (!response.IsSuccessStatusCode)
+            {
+                _logger.LogWarning("Parser API returned error status: {StatusCode}", response.StatusCode);
+                return null;
+            }
+
+            var result = await response.Content.ReadFromJsonAsync<ParserResponse>(cancellationToken: cancellationToken);
+            return result?.Content;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to extract content for URL: {Url}", url);
+            return null;
+        }
+    }
+
+    private class ParserResponse
+    {
+        [JsonPropertyName("content")]
+        public string? Content { get; set; }
+    }
+}

src/NoteBookmark.Api/Program.cs

@@ -15,6 +15,11 @@
 // Register data storage service
 builder.Services.AddScoped<IDataStorageService, DataStorageService>();
 
+// Register background extraction queue and worker
+builder.Services.AddHttpClient<IPostParserClient, PostParserClient>();
+builder.Services.AddSingleton<PostExtractionQueue>();
+builder.Services.AddHostedService<PostExtractionBackgroundWorker>();
+
 // Register AI settings provider
 builder.Services.AddScoped<IAISettingsProvider, AISettingsProvider>();