Create crawler sessions before navigating to a destination

Sources/HtmlTinkerX.Tests/HtmlBrowserInitialReadinessTests.cs

@@ -0,0 +1,74 @@
+using Microsoft.Playwright;
+using Moq;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace HtmlTinkerX.Tests;
+
+[Collection("Playwright collection")]
+public sealed class HtmlBrowserInitialReadinessTests {
+    [Theory]
+    [InlineData(false, false)]
+    [InlineData(false, true)]
+    [InlineData(true, false)]
+    [InlineData(true, true)]
+    public async Task DeferredInitialNavigationPreservesLoginAndNormalNavigationPolicy(bool deferInitialNavigation, bool login) {
+        var page = new Mock<IPage>();
+        var context = new Mock<IBrowserContext>();
+        var browser = new Mock<IBrowser>();
+        var browserType = new Mock<IBrowserType>();
+        var playwright = new Mock<IPlaywright>();
+        var navigations = new List<(string Url, WaitUntilState? State, float? Timeout)>();
+        page.Setup(value => value.GotoAsync(It.IsAny<string>(), It.IsAny<PageGotoOptions>()))
+            .Callback<string, PageGotoOptions>((url, options) => navigations.Add((url, options.WaitUntil, options.Timeout)))
+            .ReturnsAsync((IResponse?)null);
+        context.Setup(value => value.NewPageAsync()).ReturnsAsync(page.Object);
+        browser.Setup(value => value.NewContextAsync(It.IsAny<BrowserNewContextOptions>())).ReturnsAsync(context.Object);
+        browserType.Setup(value => value.LaunchAsync(It.IsAny<BrowserTypeLaunchOptions>())).ReturnsAsync(browser.Object);
+        playwright.SetupGet(value => value.Chromium).Returns(browserType.Object);
+        var originalFactory = HtmlBrowser.PlaywrightFactory;
+        HtmlBrowser.PlaywrightFactory = () => Task.FromResult(playwright.Object);
+        try {
+            var options = new HtmlBrowserLaunchOptions {
+                BrowserChannel = "msedge",
+                Timeout = 1379,
+                FormLogin = login ? new HtmlFormLogin { LoginUrl = "https://example.test/login", SubmitSelector = "#submit" } : null
+            };
+            await using var session = deferInitialNavigation
+                ? await HtmlBrowser.OpenSessionForNavigationAsync(options, default)
+                : await HtmlBrowser.OpenSessionAsync("about:blank", options);
+            Assert.Equal((login ? 1 : 0) + (deferInitialNavigation ? 0 : 1), navigations.Count);
+            if (!deferInitialNavigation) {
+                Assert.Equal(("about:blank", (WaitUntilState?)WaitUntilState.NetworkIdle, (float?)1379), navigations[navigations.Count - 1]);
+            }
+            if (login) {
+                Assert.Equal(("https://example.test/login", (WaitUntilState?)WaitUntilState.NetworkIdle, (float?)1379), navigations[0]);
+                page.Verify(value => value.WaitForLoadStateAsync(LoadState.NetworkIdle,
+                    It.Is<PageWaitForLoadStateOptions>(options => options.Timeout == 1379)), Times.Once);
+            }
+        } finally {
+            HtmlBrowser.PlaywrightFactory = originalFactory;
+        }
+    }
+
+    [Theory]
+    [InlineData(HtmlBrowserEngine.Chromium)]
+    [InlineData(HtmlBrowserEngine.Firefox)]
+    [InlineData(HtmlBrowserEngine.WebKit)]
+    public async Task DeferredInitialNavigationRunsInitScriptsOnTheActualDestination(HtmlBrowserEngine browser) {
+        var options = new HtmlBrowserLaunchOptions {
+            Browser = browser,
+            InitScripts = { "window.initializationCount = (window.initializationCount || 0) + 1" }
+        };
+        await using var session = await HtmlBrowser.OpenSessionForNavigationAsync(options, default);
+        await session.Page.RouteAsync("https://initialization.test/**", route => route.FulfillAsync(new RouteFulfillOptions {
+            ContentType = "text/html", Body = "<h1>Destination</h1>"
+        }));
+
+        await HtmlBrowser.NavigateAsync(session, "https://initialization.test/", HtmlBrowserLoadState.DomContentLoaded);
+
+        Assert.Equal("https://initialization.test/", session.Page.Url);
+        Assert.Equal(1, await session.Page.EvaluateAsync<int>("() => window.initializationCount"));
+    }
+}

Sources/HtmlTinkerX/HtmlCrawler.Browser.cs

@@ -18,7 +18,7 @@ private static async Task<CrawlRenderSession> CreateRenderSessionAsync(HtmlCrawl
             && !string.IsNullOrEmpty(options.Username) && options.Password != null) {
             headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.Username}:{options.Password}"));
         }
-        HtmlBrowserSession session = await HtmlBrowser.OpenSessionAsync("about:blank", new HtmlBrowserLaunchOptions {
+        HtmlBrowserSession session = await HtmlBrowser.OpenSessionForNavigationAsync(new HtmlBrowserLaunchOptions {
             Browser = options.Browser, Clean = options.CleanBrowserInstall, Headless = options.Headless,
             Username = imported || (options.FormLogin == null && options.Browser == HtmlBrowserEngine.Chromium) ? null : options.Username,
             Password = imported || (options.FormLogin == null && options.Browser == HtmlBrowserEngine.Chromium) ? null : options.Password,

Sources/HtmlTinkerX/Playwright/HtmlBrowser.Navigation.cs

@@ -18,7 +18,8 @@ private static async Task NavigateAsync(
         string? password,
         HtmlBrowserLoadState loadState,
         int timeout,
-        CancellationToken cancellationToken) {
+        CancellationToken cancellationToken,
+        bool skipInitialNavigation = false) {
         if (formLogin != null) {
             cancellationToken.ThrowIfCancellationRequested();
             await page.GotoAsync(formLogin.LoginUrl, new PageGotoOptions {
@@ -36,6 +37,7 @@ private static async Task NavigateAsync(
         }
 
         cancellationToken.ThrowIfCancellationRequested();
+        if (skipInitialNavigation) return;
         await page.GotoAsync(url, new PageGotoOptions {
             Timeout = timeout,
             WaitUntil = ToWaitUntilState(loadState)

Sources/HtmlTinkerX/Playwright/HtmlBrowser.cs

@@ -492,7 +492,8 @@ private static void ApplySharedContextOptions(BrowserTypeLaunchPersistentContext
     private static async Task<HtmlBrowserSession> CreatePageAsync(
         string url,
         HtmlBrowserLaunchOptions options,
-        CancellationToken cancellationToken = default) {
+        CancellationToken cancellationToken = default,
+        bool skipInitialNavigation = false) {
         if (!string.IsNullOrWhiteSpace(options.UserDataDirectory) && !string.IsNullOrWhiteSpace(options.StorageStatePath)) {
             throw new ArgumentException("Use either UserDataDirectory for a persistent profile or StorageStatePath for imported context state, not both.");
         }
@@ -597,7 +598,7 @@ await HtmlBrowserPdfCapture.ExecuteWithCancellationAsync(
                 AbortOwnershipAsync,
                 cancellationToken).ConfigureAwait(false);
             await HtmlBrowserPdfCapture.ExecuteWithCancellationAsync(
-                () => NavigateAsync(page!, url, options.FormLogin, options.Username, options.Password, options.LoadState, options.Timeout, cancellationToken),
+                () => NavigateAsync(page!, url, options.FormLogin, options.Username, options.Password, options.LoadState, options.Timeout, cancellationToken, skipInitialNavigation),
                 AbortOwnershipAsync,
                 cancellationToken).ConfigureAwait(false);
             if (options.ManualLogin) {
@@ -666,6 +667,10 @@ private static bool HasCdpContextOnlyOptions(HtmlBrowserLaunchOptions options) =
         || !string.IsNullOrWhiteSpace(options.Timezone)
         || options.Permissions.Count > 0;
 
+    /// <summary>Creates a session for installing routes before its first destination. Configured form login still runs.</summary>
+    internal static Task<HtmlBrowserSession> OpenSessionForNavigationAsync(HtmlBrowserLaunchOptions options, CancellationToken cancellationToken) =>
+        CreatePageAsync("about:blank", options, cancellationToken, skipInitialNavigation: true);
+
     /// <summary>
     /// Creates a new <see cref="HtmlBrowserSession"/> using reusable launch options and navigates to the specified URL.
     /// </summary>