diff --git a/Sources/HtmlTinkerX.Tests/HtmlHttpResponsePolicyTests.cs b/Sources/HtmlTinkerX.Tests/HtmlHttpResponsePolicyTests.cs
index 179ca09e..f8a79f30 100644
--- a/Sources/HtmlTinkerX.Tests/HtmlHttpResponsePolicyTests.cs
+++ b/Sources/HtmlTinkerX.Tests/HtmlHttpResponsePolicyTests.cs
@@ -18,17 +18,17 @@ public class HtmlHttpResponsePolicyTests {
private static async Task ReadAsync(string consumer, HttpClient client, HtmlHttpFetchOptions? options, CancellationToken token = default) {
if (consumer == "url") {
- return options == null ? await HtmlUtilities.GetStringWithProperEncodingAsync(client, Url, token)
- : await HtmlUtilities.GetStringWithProperEncodingAsync(client, Url, options, token);
+ return options == null ? await HtmlUtilities.GetStringWithProperEncodingAsync(client, Url, token).ConfigureAwait(false)
+ : await HtmlUtilities.GetStringWithProperEncodingAsync(client, Url, options, token).ConfigureAwait(false);
}
if (consumer == "relay") {
- var result = await HtmlFormRelayClient.FollowAsync(Relay, new Uri(Url), client, new HtmlFormRelayOptions { FetchOptions = options }, token);
+ var result = await HtmlFormRelayClient.FollowAsync(Relay, new Uri(Url), client, new HtmlFormRelayOptions { FetchOptions = options }, token).ConfigureAwait(false);
return result.FinalContent;
}
var fields = new Dictionary { ["field"] = "value" };
FormMethod method = consumer == "get" ? FormMethod.Get : FormMethod.Post;
- return options == null ? await HtmlFormSubmitter.SubmitAsync(Url, method, fields, client, token)
- : await HtmlFormSubmitter.SubmitAsync(Url, method, fields, client, options, token);
+ return options == null ? await HtmlFormSubmitter.SubmitAsync(Url, method, fields, client, token).ConfigureAwait(false)
+ : await HtmlFormSubmitter.SubmitAsync(Url, method, fields, client, options, token).ConfigureAwait(false);
}
[Theory]
@@ -36,29 +36,35 @@ private static async Task ReadAsync(string consumer, HttpClient client,
[InlineData("get")]
[InlineData("post")]
[InlineData("relay")]
- public async Task LegacyConsumers_RejectOversizeStreamingResponses(string consumer) {
- byte[] bytes = new byte[HtmlHttpFetchOptions.DefaultMaximumResponseBytes + 1];
- using var handler = new ResponseHandler(() => new StreamContent(new MemoryStream(bytes)));
+ public async Task LegacyConsumers_RejectOversizeDeclaredResponses(string consumer) {
+ using var handler = new ResponseHandler(() => {
+ var content = new ByteArrayContent(Array.Empty());
+ content.Headers.ContentLength = HtmlHttpFetchOptions.DefaultMaximumResponseBytes + 1L;
+ return content;
+ });
using var client = new HttpClient(handler);
await Assert.ThrowsAsync(() => ReadAsync(consumer, client, null));
}
[Theory]
- [InlineData("url")]
- [InlineData("get")]
- [InlineData("post")]
- [InlineData("relay")]
- public async Task ConsumerLimits_RejectDeclaredAndChunkedBodiesAndAllowExplicitOverride(string consumer) {
- foreach (bool declared in new[] { false, true }) {
- using var handler = new ResponseHandler(() => {
- HttpContent content = declared ? new ByteArrayContent(Encoding.UTF8.GetBytes("response"))
- : new StreamContent(new MemoryStream(Encoding.UTF8.GetBytes("response")));
- return content;
- });
- using var client = new HttpClient(handler);
- await Assert.ThrowsAsync(() => ReadAsync(consumer, client, new HtmlHttpFetchOptions { MaximumResponseBytes = 7 }));
- Assert.Equal("response", await ReadAsync(consumer, client, new HtmlHttpFetchOptions { MaximumResponseBytes = 8 }));
- }
+ [InlineData("url", false)]
+ [InlineData("url", true)]
+ [InlineData("get", false)]
+ [InlineData("get", true)]
+ [InlineData("post", false)]
+ [InlineData("post", true)]
+ [InlineData("relay", false)]
+ [InlineData("relay", true)]
+ public async Task ConsumerLimits_RejectDeclaredAndUnknownLengthBodiesAndAllowExplicitOverride(string consumer, bool declared) {
+ using var handler = new ResponseHandler(() => {
+ HttpContent content = declared ? new ByteArrayContent(Encoding.UTF8.GetBytes("response"))
+ : new StreamContent(new UnknownLengthReadStream(Encoding.UTF8.GetBytes("response")));
+ Assert.Equal(declared ? 8L : (long?)null, content.Headers.ContentLength);
+ return content;
+ });
+ using var client = new HttpClient(handler);
+ await Assert.ThrowsAsync(() => ReadAsync(consumer, client, new HtmlHttpFetchOptions { MaximumResponseBytes = 7 }));
+ Assert.Equal("response", await ReadAsync(consumer, client, new HtmlHttpFetchOptions { MaximumResponseBytes = 8 }));
}
[Theory]
@@ -80,25 +86,27 @@ public async Task Consumers_UseBomBeforeConflictingHeaders(string consumer) {
}
[Theory]
- [InlineData("url")]
- [InlineData("get")]
- [InlineData("post")]
- [InlineData("relay")]
- public async Task Consumers_CancelStalledStreamsThatIgnoreReadTokens(string consumer) {
- foreach (bool callerCancellation in new[] { false, true }) {
- using var stream = new BlockingStream();
- using var handler = new ResponseHandler(() => new StreamContent(stream));
- using var client = new HttpClient(handler) { Timeout = callerCancellation ? TimeSpan.FromSeconds(5) : TimeSpan.FromMilliseconds(100) };
- using var cancellation = new CancellationTokenSource();
- Task reading = ReadAsync(consumer, client, null, cancellation.Token);
- if (callerCancellation) {
- Assert.Same(stream.Entered.Task, await Task.WhenAny(stream.Entered.Task, Task.Delay(2000)));
- cancellation.Cancel();
- }
- Assert.Same(reading, await Task.WhenAny(reading, Task.Delay(2000)));
- await Assert.ThrowsAnyAsync(() => reading);
- Assert.True(stream.Disposed);
+ [InlineData("url", false)]
+ [InlineData("url", true)]
+ [InlineData("get", false)]
+ [InlineData("get", true)]
+ [InlineData("post", false)]
+ [InlineData("post", true)]
+ [InlineData("relay", false)]
+ [InlineData("relay", true)]
+ public async Task Consumers_CancelStalledStreamsThatIgnoreReadTokens(string consumer, bool callerCancellation) {
+ using var stream = new BlockingStream();
+ using var handler = new ResponseHandler(() => new StreamContent(stream));
+ using var client = new HttpClient(handler) { Timeout = callerCancellation ? TimeSpan.FromSeconds(5) : TimeSpan.FromMilliseconds(100) };
+ using var cancellation = new CancellationTokenSource();
+ Task reading = ReadAsync(consumer, client, null, cancellation.Token);
+ if (callerCancellation) {
+ Assert.Same(stream.Entered.Task, await Task.WhenAny(stream.Entered.Task, Task.Delay(2000)));
+ cancellation.Cancel();
}
+ Assert.Same(reading, await Task.WhenAny(reading, Task.Delay(2000)));
+ await Assert.ThrowsAnyAsync(() => reading);
+ Assert.True(stream.Disposed);
}
[Fact]
diff --git a/Sources/HtmlTinkerX.Tests/HtmlUtilitiesHttpTests.cs b/Sources/HtmlTinkerX.Tests/HtmlUtilitiesHttpTests.cs
index 922bc6af..5ad1aa8c 100644
--- a/Sources/HtmlTinkerX.Tests/HtmlUtilitiesHttpTests.cs
+++ b/Sources/HtmlTinkerX.Tests/HtmlUtilitiesHttpTests.cs
@@ -25,9 +25,11 @@ public async Task ReadResponseContent_RejectsDeclaredContentLengthBeforeReading(
[Fact]
public async Task ReadResponseContent_EnforcesLimitWhileStreamingUnknownLength() {
using HttpResponseMessage response = new(HttpStatusCode.OK) {
- Content = new StreamContent(new MemoryStream(new byte[11]))
+ Content = new StreamContent(new UnknownLengthReadStream(new byte[11]))
};
+ Assert.Null(response.Content.Headers.ContentLength);
+
InvalidDataException exception = await Assert.ThrowsAsync(() =>
HtmlUtilities.ReadResponseContentWithProperEncodingAsync(
response,
diff --git a/Sources/HtmlTinkerX.Tests/UnknownLengthReadStream.cs b/Sources/HtmlTinkerX.Tests/UnknownLengthReadStream.cs
new file mode 100644
index 00000000..c8489f20
--- /dev/null
+++ b/Sources/HtmlTinkerX.Tests/UnknownLengthReadStream.cs
@@ -0,0 +1,8 @@
+using System.IO;
+
+namespace HtmlTinkerX.Tests;
+
+// StreamContent derives Content-Length from seekable streams.
+internal sealed class UnknownLengthReadStream(byte[] bytes) : MemoryStream(bytes, writable: false) {
+ public override bool CanSeek => false;
+}
diff --git a/Tests/Invoke-HTMLCrawl.Tests.ps1 b/Tests/Invoke-HTMLCrawl.Tests.ps1
index 21878da3..3a216a17 100644
--- a/Tests/Invoke-HTMLCrawl.Tests.ps1
+++ b/Tests/Invoke-HTMLCrawl.Tests.ps1
@@ -13,8 +13,8 @@ using System.Threading.Tasks;
public sealed class PesterTestHttpServer : IDisposable {
private sealed class ServerResponse {
- public string Body { get; set; } = string.Empty;
- public string ContentType { get; set; } = "text/html; charset=utf-8";
+ public string Body { get; set; }
+ public string ContentType { get; set; }
}
private readonly HttpListener _listener = new HttpListener();
@@ -26,10 +26,10 @@ public sealed class PesterTestHttpServer : IDisposable {
Prefix = prefix;
_listener.Prefixes.Add(prefix);
_listener.Start();
- _serverTask = Task.Run(ListenAsync);
+ _serverTask = Task.Run(new Func(ListenAsync));
}
- public string Prefix { get; }
+ public string Prefix { get; private set; }
public void AddResponse(string path, string body, string contentType) {
_responses[path] = new ServerResponse {
@@ -44,14 +44,17 @@ public sealed class PesterTestHttpServer : IDisposable {
try {
context = await _listener.GetContextAsync().ConfigureAwait(false);
- } catch (HttpListenerException) when (_cancellation.IsCancellationRequested || !_listener.IsListening) {
+ } catch (HttpListenerException) {
+ if (!_cancellation.IsCancellationRequested && _listener.IsListening) throw;
break;
- } catch (ObjectDisposedException) when (_cancellation.IsCancellationRequested) {
+ } catch (ObjectDisposedException) {
+ if (!_cancellation.IsCancellationRequested) throw;
break;
}
string rawUrl = string.IsNullOrWhiteSpace(context.Request.RawUrl) ? "/" : context.Request.RawUrl;
- if (!_responses.TryGetValue(rawUrl, out ServerResponse response)) {
+ ServerResponse response;
+ if (!_responses.TryGetValue(rawUrl, out response)) {
context.Response.StatusCode = 404;
context.Response.Close();
continue;
@@ -143,8 +146,9 @@ public sealed class PesterTestHttpServer : IDisposable {
[string] $ToFilePath
)
- $fromDirectory = Split-Path $FromFilePath -Parent
- [System.IO.Path]::GetRelativePath($fromDirectory, $ToFilePath).Replace('\', '/')
+ $fromUri = [Uri]::new([System.IO.Path]::GetFullPath($FromFilePath))
+ $toUri = [Uri]::new([System.IO.Path]::GetFullPath($ToFilePath))
+ [Uri]::UnescapeDataString($fromUri.MakeRelativeUri($toUri).ToString())
}
}