Continue refactoring and changes for 1.2.0

Author: Decimation

SmartImage.Lib/Engines/Results/SearchResultItem.cs

@@ -217,7 +217,7 @@ public override async ValueTask<bool> AllocImageAsync(CancellationToken ct = def
 		}
 
 		bool allocImgOk = false;
-		var  allocOk    = await AllocAsync(ct);
+		var  allocOk    = await AllocSourceAsync(ct);
 
 		if (allocOk) {
 			allocImgOk = await base.AllocImageAsync(ct);
@@ -286,7 +286,7 @@ await Parallel.ForEachAsync(urls, ct, async (s, token) =>
 				IsScanResult = true,
 			};
 
-			// var sriNew = CloneToChildWithUrl(s);
+			// var sriNew = MemberwiseCloneWithUrl(s);
 
 			var allocImgOk = await sriNew.AllocImageAsync(token);
 
@@ -321,8 +321,8 @@ public async ValueTask<bool> ScanAsync(CancellationToken ct = default)
 
 		var task = ScanAsync(cw.Writer, s =>
 		{
-			var obj= CloneWithUrl(s);
-			return obj;
+			var clone= PartialCopyCloneWithUrl(s);
+			return clone;
 		}, ct);
 
 		while (await cw.Reader.WaitToReadAsync(ct)) {
@@ -334,13 +334,11 @@ public async ValueTask<bool> ScanAsync(CancellationToken ct = default)
 		var ok = await task;
 
 		return ok;
-
-		// return sriNews;
 	}
 
 #endregion
 
-	public SearchResultItem CloneWithUrl(Url s)
+	public SearchResultItem PartialCopyCloneWithUrl(Url s)
 	{
 		return new SearchResultItem(Root, false)
 		{
@@ -353,11 +351,12 @@ public SearchResultItem CloneWithUrl(Url s)
 			Site        = Site,
 			Source      = Source,
 			Time        = Time,
+			Metadata = Metadata,
 			IsCloned    = true,
 		};
 	}
 
-	public SearchResultItem CloneToChildWithUrl(Url u)
+	public SearchResultItem MemberwiseCloneWithUrl(Url u)
 	{
 		var clone = (MemberwiseClone() as SearchResultItem);
 		clone.Url    = u;

SmartImage.Lib/Engines/Search/TinEyeEngine.cs

@@ -121,7 +121,7 @@ public override async Task<SearchResult> GetResultAsync(SearchQuery query, Cance
 				for (int m = 1; m < backlinks.Count; m++) {
 					var bl = backlinks[m];
 
-					var resultItemSister = resultItem.CloneToChildWithUrl(bl.Backlink);
+					var resultItemSister = resultItem.MemberwiseCloneWithUrl(bl.Backlink);
 					resultItemSister.Description = bl.ImageName;
 					resultItemSister.Title       = bl.ImageName;
 					resultItemSister.Time        = DateTime.Parse(bl.CrawlDate);

SmartImage.Lib/Images/ImageScanner.cs

@@ -130,25 +130,22 @@ static ImageScanner()
 
 #endregion
 
-#region Scanning
+#region Scanning/parsing
 
-	public static readonly string[] UrlSegmentBlacklist = ["thumbs", ".svg", ".ico", "twitter.svg", "pinterest.svg"];
+	public static readonly string[] UrlSegmentBlacklist = ["thumbs", ".svg", ".ico", "twitter.svg", "pinterest.svg", "favicon"];
 
 	public static readonly string[] LegalSchemeWhitelist = ["http", "https"];
 
 	internal const char URL_DELIM = '/';
 
-	/// <summary>
-	/// Scans for images within the webpage located at <paramref name="url"/>; if <paramref name="url"/> itself
-	/// points to binary image data, it is returned.
-	/// </summary>
-	public static async Task<bool> ScanForImagesAsync(Url url, ChannelWriter<UniImage> cw, CancellationToken ct = default)
+
+	/*public static async Task<bool> ScanForImagesAsync(Url url, ChannelWriter<UniImage> cw, CancellationToken ct = default)
 	{
 		string sz = null;
 
 		IHtmlDocument doc = null;
 
-		/* Immediate search  */
+		/* Immediate search  #1#
 		var uf = await UniImage.TryCreateAsync(url, autoInit: true, autoDisposeOnError: false, ct: ct);
 
 		IFlurlResponse res;
@@ -176,7 +173,7 @@ public static async Task<bool> ScanForImagesAsync(Url url, ChannelWriter<UniImag
 		/*if (!stream.CanRead) {
 			stream.Dispose();
 			goto ret;
-		}*/
+		}#1#
 
 		var dp = new HtmlParser();
 
@@ -214,7 +211,7 @@ async ValueTask Body(string s, CancellationToken token)
 				uni?.Dispose();
 			}
 		}
-	}
+	}*/
 
 	public static IEnumerable<string> ParseImageUrlsByRegex(string html, Url url, bool heuristicFilter = true)
 	{
@@ -239,6 +236,19 @@ public static IEnumerable<string> ParseImageUrlsByRegex(string html, Url url, bo
 			}
 		}
 
+		imgUrls = imgUrls.Distinct().Where(e =>
+		{
+			if (e.StartsWith("url(")) {
+				return false;
+			}
+
+			return Url.IsValid(e);
+		});
+
+		if (heuristicFilter) {
+			imgUrls = imgUrls.Where(static u => !UrlSegmentBlacklist.Any(u.Contains));
+		}
+
 		var abs = imgUrls.Select(u =>
 		{
 			if (u.StartsWith("http"))
@@ -253,11 +263,7 @@ public static IEnumerable<string> ParseImageUrlsByRegex(string html, Url url, bo
 
 			// return baseUrl + URL_DELIM + u;
 			return Url.Combine(baseUrl, URL_DELIM.ToString(), u);
-		}).Select(static u => Url.Decode(u, true)).Where(Url.IsValid).Distinct();
-
-		if (heuristicFilter) {
-			abs = abs.Where(static u => !UrlSegmentBlacklist.Any(u.Contains));
-		}
+		});
 
 		return abs;
 	}
@@ -278,10 +284,19 @@ public static IEnumerable<string> ParseImageUrlsByDoc(IHtmlDocument doc)
 
 	public static async ValueTask<IFlurlResponse> GetResponseAsync(Url value, CancellationToken ct)
 	{
-		var req1 = await Client.Request(value)
-			           .GetAsync(cancellationToken: ct);
+		var request = Client.Request(value);
+
+		if (value.ToString().Contains("zerochan")) {
+			// request = request.WithHeader("User-Agent", R1.Name);
+
+			request = new FlurlRequest(value) { };
+		}
+
+		var response = await request
+			               .OnError(act => { act.ExceptionHandled = true; })
+			               .GetAsync(cancellationToken: ct);
 
-		return req1;
+		return response;
 	}
 
 #endregion

SmartImage.Lib/Images/Uni/UniImage.cs

@@ -15,11 +15,13 @@
 using SixLabors.ImageSharp.Formats.Png;
 using SixLabors.ImageSharp.PixelFormats;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Runtime.InteropServices;
 using CoenM.ImageHash;
 using CommunityToolkit.HighPerformance;
 using Kantan.Net.Utilities;
 using Microsoft.IO;
+using SixLabors.ImageSharp.Memory;
 using SmartImage.Lib.Model;
 using SmartImage.Lib.Utilities;
 
@@ -128,7 +130,7 @@ protected set
 
 #endregion
 
-	#region 
+#region
 
 	public double? Similarity
 	{
@@ -142,7 +144,7 @@ public virtual bool CalculateSimilarity(IHashable hashable)
 		return Similarity.HasValue;
 	}
 
-	#endregion
+#endregion
 
 #region
 
@@ -172,9 +174,6 @@ public Stream GetStream()
 #endregion
 
 
-	public static readonly UniImage Null = null;
-
-
 	private protected UniImage(string value, UniImageType type)
 	{
 		Value = value;
@@ -184,10 +183,13 @@ private protected UniImage(string value, UniImageType type)
 
 #region
 
-	protected abstract ValueTask<bool> AllocAsync(CancellationToken ct = default);
+	/// <summary>
+	/// Allocates <see cref="Bytes"/>
+	/// </summary>
+	protected abstract ValueTask<bool> AllocSourceAsync(CancellationToken ct = default);
 
 	/// <summary>
-	/// Allocates <see cref="Image"/>
+	/// Allocates <see cref="Image"/> from <see cref="Bytes"/>
 	/// </summary>
 	public virtual async ValueTask<bool> AllocImageAsync(CancellationToken ct = default)
 	{
@@ -211,13 +213,26 @@ public virtual async ValueTask<bool> AllocImageAsync(CancellationToken ct = defa
 
 	}
 
+	/// <returns><see cref="AllocSourceAsync"/>, <see cref="AllocImageAsync"/></returns>
+	protected virtual async ValueTask<(bool AllocOk, bool AllocImageOk)> AllocAll(CancellationToken ct)
+	{
+		bool allocOk    = await AllocSourceAsync(ct);
+		bool allocImgOk = false;
+
+		if (allocOk) {
+			allocImgOk = await AllocImageAsync(ct);
+		}
+
+		return (allocOk, allocImgOk);
+	}
+
 	/// <summary>
 	/// Attempts to create the appropriate <see cref="UniImage" /> for <paramref name="o" />.
 	/// </summary>
 	public static async Task<UniImage> TryCreateAsync(object o, bool autoInit = true, bool autoDisposeOnError = true,
 	                                                  CancellationToken ct = default)
 	{
-		UniImage ui = Null;
+		UniImage ui = null;
 
 		try {
 
@@ -235,11 +250,7 @@ public static async Task<UniImage> TryCreateAsync(object o, bool autoInit = true
 				bool allocOk    = false;
 				bool allocImgOk = false;
 
-				allocOk = await ui.AllocAsync(ct);
-
-				if (allocOk) {
-					allocImgOk = await ui.AllocImageAsync(ct);
-				}
+				(allocOk,allocImgOk) = await ui.AllocAll(ct);
 
 				s_logger.LogTrace("{Value} :: {AllocOk} {AllocImgOk}", o, allocOk, allocImgOk);
 
@@ -270,7 +281,7 @@ public static bool IsValidSourceType(object o)
 
 #endregion
 
-	#region New region
+#region New region
 
 	public bool TryWriteOrGetFile(string fn = null)
 	{
@@ -314,13 +325,14 @@ public virtual string WriteImageToFile([CBN] string fn = null)
 		return path;
 	}
 
-	#endregion
+#endregion
 
 
 	public virtual void Dispose()
 	{
 		GC.SuppressFinalize(this);
 		Image?.Dispose();
+		s_logger.LogTrace("Disposing {Uv} {Ut}", Value, Type);
 	}
 
 	public override string ToString()

SmartImage.Lib/Images/Uni/UniImageFile.cs

@@ -20,7 +20,7 @@ internal UniImageFile(FileInfo fi) : base(fi.FullName, UniImageType.File)
 
 	public FileInfo LocalFileInfo { get; }
 
-	protected override async ValueTask<bool> AllocAsync(CancellationToken ct = default)
+	protected override async ValueTask<bool> AllocSourceAsync(CancellationToken ct = default)
 	{
 		if (HasBytes) {
 			goto ret;