Version 1.4.0 - Re-design async post content extraction

Directory.Build.props

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <Version>1.3.2</Version>
+    <Version>1.4.0</Version>
     <TargetFramework>net10.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

src/NoteBookmark.AIServices/ResearchService.cs

@@ -151,7 +151,9 @@ private async Task SaveToFile(string prefix, string responseContent)
     {
         string datetime = DateTime.Now.ToString("yyyy-MM-dd_HH-mm");
         string fileName = $"{prefix}_{datetime}.json";
-        string folderPath = "Data";
+        // Use the app's sandboxed data directory so this works on Android/iOS as well as desktop
+        string folderPath = Path.Combine(
+            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Data");
         Directory.CreateDirectory(folderPath);
         string filePath = Path.Combine(folderPath, fileName);
         await File.WriteAllTextAsync(filePath, responseContent);

src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs

@@ -0,0 +1,74 @@
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+using NoteBookmark.Api.Tests.Fixtures;
+using NoteBookmark.Domain;
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Threading.Tasks;
+using Xunit;
+using Azure.Storage.Blobs;
+
+namespace NoteBookmark.Api.Tests.Endpoints;
+
+public class PostExtractionTests : IClassFixture<NoteBookmarkApiTestFactory>
+{
+    private readonly NoteBookmarkApiTestFactory _factory;
+    private readonly HttpClient _client;
+
+    public PostExtractionTests(NoteBookmarkApiTestFactory factory)
+    {
+        _factory = factory;
+        _client = _factory.CreateClient();
+    }
+
+    [Fact]
+    public async Task ExtractPostDetails_TriggersBackgroundWorkerAndSavesHtmlToBlobStorage()
+    {
+        // Arrange
+        var url = "https://example.com/blog/test-post-" + Guid.NewGuid();
+        var extractRequest = new
+        {
+            url = url,
+            tags = "test",
+            category = "Test"
+        };
+
+        // Act - Call the API to extract metadata and save the post
+        var response = await _client.PostAsJsonAsync("/api/posts/extractPostDetails", extractRequest);
+
+        // Assert API response is OK
+        response.StatusCode.Should().Be(HttpStatusCode.OK);
+
+        var post = await response.Content.ReadFromJsonAsync<Post>();
+        post.Should().NotBeNull();
+        var postId = post!.Id ?? post.RowKey;
+        postId.Should().NotBeNullOrEmpty();
+
+        // Since the extraction happens asynchronously in a BackgroundWorker,
+        // we poll the blob storage for a short time to verify the file was created.
+        var blobServiceClient = _factory.Services.GetRequiredService<BlobServiceClient>();
+        var containerClient = blobServiceClient.GetBlobContainerClient("cleanedposts");
+        var blobClient = containerClient.GetBlobClient($"{postId}.html");
+
+        // Wait up to 5 seconds for the background worker to process
+        bool blobExists = false;
+        for (int i = 0; i < 25; i++)
+        {
+            if (await blobClient.ExistsAsync())
+            {
+                blobExists = true;
+                break;
+            }
+            await Task.Delay(200);
+        }
+
+        blobExists.Should().BeTrue("HTML content should be processed by the background worker and saved to blob storage");
+
+        // Verify the content saved matches the fake content
+        var downloadResult = await blobClient.DownloadContentAsync();
+        var content = downloadResult.Value.Content.ToString();
+        content.Should().Contain(url);
+    }
+}

src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs

@@ -0,0 +1,13 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NoteBookmark.Api.Tests.Fixtures;
+
+public class FakePostParserClient : IPostParserClient
+{
+    public Task<string?> ExtractContentAsync(string url, CancellationToken cancellationToken = default)
+    {
+        // Return a mock HTML snippet for testing
+        return Task.FromResult<string?>($"<div>Extracted HTML content for {url}</div>");
+    }
+}

src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs

@@ -34,6 +34,9 @@ protected override void ConfigureWebHost(IWebHostBuilder builder)
                 services.AddSingleton(new TableServiceClient(connectionString));
                 services.AddSingleton(new BlobServiceClient(connectionString));
             }
+
+            // Register FakePostParserClient for integration tests
+            services.AddSingleton<NoteBookmark.Api.IPostParserClient, FakePostParserClient>();
         });
     }
 

src/NoteBookmark.Api.Tests/Services/PostParserClientTests.cs

@@ -0,0 +1,163 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq.Protected;
+
+namespace NoteBookmark.Api.Tests.Services;
+
+public class PostParserClientTests
+{
+    private readonly Mock<IConfiguration> _mockConfig;
+    private readonly Mock<HttpMessageHandler> _mockHandler;
+
+    public PostParserClientTests()
+    {
+        _mockConfig = new Mock<IConfiguration>();
+        _mockHandler = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+
+        // Default config setups
+        _mockConfig.Setup(c => c["Parser:BaseUrl"]).Returns((string?)null);
+        _mockConfig.Setup(c => c["Parser:ApiKey"]).Returns((string?)null);
+    }
+
+    private PostParserClient CreateSut(HttpClient httpClient) =>
+        new(httpClient, _mockConfig.Object, NullLogger<PostParserClient>.Instance);
+
+    [Fact]
+    public async Task ExtractContentAsync_WithDefaults_CallsDefaultUrlWithoutApiKey()
+    {
+        // Arrange
+        var expectedUrl = "https://azpostlight-parser.azurewebsites.net/api/parser";
+        var sourceUrl = "https://example.com/blog-post";
+
+        _mockHandler.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.Is<HttpRequestMessage>(req =>
+                    req.Method == HttpMethod.Post &&
+                    req.RequestUri != null &&
+                    req.RequestUri.ToString() == expectedUrl &&
+                    !req.Headers.Contains("x-functions-key")),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent("{\"content\":\"extracted blog content\"}", Encoding.UTF8, "application/json")
+            });
+
+        var httpClient = new HttpClient(_mockHandler.Object);
+        var sut = CreateSut(httpClient);
+
+        // Act
+        var result = await sut.ExtractContentAsync(sourceUrl);
+
+        // Assert
+        result.Should().Be("extracted blog content");
+        _mockHandler.Protected().Verify(
+            "SendAsync",
+            Times.Once(),
+            ItExpr.Is<HttpRequestMessage>(req => req.RequestUri != null && req.RequestUri.ToString() == expectedUrl),
+            ItExpr.IsAny<CancellationToken>());
+    }
+
+    [Fact]
+    public async Task ExtractContentAsync_WithApiKey_SendsXFunctionsKeyHeader()
+    {
+        // Arrange
+        var expectedUrl = "https://azpostlight-parser.azurewebsites.net/api/parser";
+        var sourceUrl = "https://example.com/blog-post";
+        var apiKey = "test-api-key-123";
+
+        _mockConfig.Setup(c => c["Parser:ApiKey"]).Returns(apiKey);
+
+        _mockHandler.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.Is<HttpRequestMessage>(req =>
+                    req.Method == HttpMethod.Post &&
+                    req.RequestUri != null &&
+                    req.RequestUri.ToString() == expectedUrl &&
+                    req.Headers.Contains("x-functions-key") &&
+                    string.Join("", req.Headers.GetValues("x-functions-key")) == apiKey),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent("{\"content\":\"content with auth\"}", Encoding.UTF8, "application/json")
+            });
+
+        var httpClient = new HttpClient(_mockHandler.Object);
+        var sut = CreateSut(httpClient);
+
+        // Act
+        var result = await sut.ExtractContentAsync(sourceUrl);
+
+        // Assert
+        result.Should().Be("content with auth");
+    }
+
+    [Fact]
+    public async Task ExtractContentAsync_WithCustomUrl_CallsCustomUrl()
+    {
+        // Arrange
+        var customUrl = "https://my-custom-parser.com/api/parser";
+        var sourceUrl = "https://example.com/blog-post";
+
+        _mockConfig.Setup(c => c["Parser:BaseUrl"]).Returns(customUrl);
+
+        _mockHandler.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.Is<HttpRequestMessage>(req =>
+                    req.Method == HttpMethod.Post &&
+                    req.RequestUri != null &&
+                    req.RequestUri.ToString() == customUrl),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.OK,
+                Content = new StringContent("{\"content\":\"custom url content\"}", Encoding.UTF8, "application/json")
+            });
+
+        var httpClient = new HttpClient(_mockHandler.Object);
+        var sut = CreateSut(httpClient);
+
+        // Act
+        var result = await sut.ExtractContentAsync(sourceUrl);
+
+        // Assert
+        result.Should().Be("custom url content");
+    }
+
+    [Fact]
+    public async Task ExtractContentAsync_ParserReturnsErrorCode_ReturnsNull()
+    {
+        // Arrange
+        var sourceUrl = "https://example.com/blog-post";
+
+        _mockHandler.Protected()
+            .Setup<Task<HttpResponseMessage>>(
+                "SendAsync",
+                ItExpr.IsAny<HttpRequestMessage>(),
+                ItExpr.IsAny<CancellationToken>())
+            .ReturnsAsync(new HttpResponseMessage
+            {
+                StatusCode = HttpStatusCode.InternalServerError
+            });
+
+        var httpClient = new HttpClient(_mockHandler.Object);
+        var sut = CreateSut(httpClient);
+
+        // Act
+        var result = await sut.ExtractContentAsync(sourceUrl);
+
+        // Assert
+        result.Should().BeNull();
+    }
+}

src/NoteBookmark.Api/IPostParserClient.cs

@@ -0,0 +1,9 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NoteBookmark.Api;
+
+public interface IPostParserClient
+{
+    Task<string?> ExtractContentAsync(string url, CancellationToken cancellationToken = default);
+}

src/NoteBookmark.Api/PostEndpoints.cs

@@ -94,7 +94,11 @@ static Results<Ok, BadRequest> SavePost(Post post, TableServiceClient tblClient,
 		}
 		return TypedResults.BadRequest();
 	}
-	static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(ExtractPostRequest request, TableServiceClient tblClient, BlobServiceClient blobClient)
+	static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(
+		ExtractPostRequest request,
+		TableServiceClient tblClient,
+		BlobServiceClient blobClient,
+		PostExtractionQueue queue)
 	{
 		var dataStorageService = new DataStorageService(tblClient, blobClient);
 
@@ -105,6 +109,10 @@ static async Task<Results<Ok<Post>, BadRequest>> ExtractPostDetails(ExtractPostR
 			if (post != null)
 			{
 				dataStorageService.SavePost(post);
+
+				// Queue background HTML extraction task
+				queue.QueueBackgroundWorkItem(new ExtractionTask(post.Id ?? post.RowKey, post.Url ?? decodeUrl));
+
 				return TypedResults.Ok(post);
 			}
 			return TypedResults.BadRequest();