Skip to content

Commit 6a3e6db

Browse files
DecimationDecimation
committed
Fixes; refactoring; fix item index selection
1 parent 4fdd882 commit 6a3e6db

9 files changed

Lines changed: 190 additions & 160 deletions

File tree

‎SmartImage.Lib/Clients/FlareSolverrClient.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ public FlareSolverrClient([CBN] string api = FLARE_SOLVERR_API_URL_DEFAULT)
5454

5555
static FlareSolverrClient() { }
5656

57+
public async ValueTask<FlareSolverrIndexResponse> GetIndexAsync()
58+
{
59+
return (await Clearance.Solverr.GetIndexAsync());
60+
}
5761

5862
public void Dispose()
5963
{
@@ -63,13 +67,19 @@ public void Dispose()
6367
Client = null;
6468
}
6569

66-
public ValueTask<bool> ApplyConfigAsync(SearchConfig cfg, CancellationToken ct = default)
70+
public async ValueTask<bool> ApplyConfigAsync(SearchConfig cfg, CancellationToken ct = default)
6771
{
6872
var ok = false;
73+
FlareSolverrIndexResponse idx = null;
74+
75+
if (!cfg.FlareSolverr) {
76+
return false;
77+
}
6978

70-
ok = Configure(cfg.FlareSolverrApiUrl);
79+
ok = Configure(cfg.FlareSolverrApiUrl);
80+
idx = await GetIndexAsync();
7181

72-
return ValueTask.FromResult(ok);
82+
return ok && idx != null;
7383
}
7484

7585

‎SmartImage.Lib/Engines/Results/SearchResultItem.cs‎

Lines changed: 27 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -251,23 +251,6 @@ public async ValueTask<bool> LoadThumbnailAsync(CancellationToken ct = default)
251251
return HasThumbnail;
252252
}
253253

254-
public SearchResultItem CloneWithUrl(Url s)
255-
{
256-
return new SearchResultItem(Root, false)
257-
{
258-
Parent = this,
259-
Url = s,
260-
Artist = Artist,
261-
Character = Character,
262-
Description = Description,
263-
Title = Title,
264-
Site = Site,
265-
Source = Source,
266-
Time = Time,
267-
IsCloned = true,
268-
};
269-
}
270-
271254
/*public async ValueTask<bool> ScanAsync(CancellationToken ct = default)
272255
{
273256
if (!(await AllocImageAsync(ct))) {
@@ -328,57 +311,48 @@ await Parallel.ForEachAsync(urls, ct, async (s, token) =>
328311
[MNNW(true, nameof(ScannedItems))]
329312
public bool HasScannedItems => ScannedItems is { Count: > 0 };
330313

331-
public async ValueTask<bool> ScanAsync2(CancellationToken ct = default)
314+
public async ValueTask<bool> ScanAsync(CancellationToken ct = default)
332315
{
333-
if (!(await AllocImageAsync(ct))) {
334-
// return false;
335-
}
336-
337316
if (HasScannedItems) {
338317
return true;
339318
}
340319

341-
await using var stream = GetStream();
342-
using var sr = new StreamReader(stream);
343-
var str = await sr.ReadToEndAsync(ct);
320+
var cw = Channel.CreateUnbounded<SearchResultItem>();
344321

345-
var hp = new HtmlParser();
346-
var urls = ImageScanner.ParseImageUrlsByRegex(str, Url);
347-
using var doc = await hp.ParseDocumentAsync(str);
348-
var sriNews = new ConcurrentBag<SearchResultItem>();
322+
var task = ScanAsync(cw.Writer, s => { return CloneWithUrl(s); }, ct);
349323

350-
await Parallel.ForEachAsync(urls, ct, async (s, token) =>
351-
{
352-
var sriNew = CloneWithUrl(s);
324+
while (await cw.Reader.WaitToReadAsync(ct)) {
325+
var val = await cw.Reader.ReadAsync(ct);
353326

354-
// var sriNew = CloneToChildWithUrl(s);
355-
356-
var allocImgOk = await sriNew.AllocImageAsync(token);
357-
358-
if (allocImgOk) {
359-
sriNews.Add(sriNew);
360-
}
361-
else {
362-
sriNew?.Dispose();
363-
}
364-
});
365-
366-
367-
// Root.Results.InsertRange(Root.Results.IndexOf(this), sriNews);
368-
// ScannedItems = sriNews.ToList();
327+
ScannedItems.Add(val);
328+
}
369329

370-
// var sriIdx = Results.IndexOf(sri);
371-
// Results.InsertRange(sriIdx + 1, sriNews);
330+
var ok = await task;
372331

373-
// return ScannedResults.TryAdd(sri, sriNews.ToArray());
374-
ScannedItems.AddRange(sriNews);
375-
return true;
332+
return ok;
376333

377334
// return sriNews;
378335
}
379336

380337
#endregion
381338

339+
public SearchResultItem CloneWithUrl(Url s)
340+
{
341+
return new SearchResultItem(Root, false)
342+
{
343+
Parent = this,
344+
Url = s,
345+
Artist = Artist,
346+
Character = Character,
347+
Description = Description,
348+
Title = Title,
349+
Site = Site,
350+
Source = Source,
351+
Time = Time,
352+
IsCloned = true,
353+
};
354+
}
355+
382356
public SearchResultItem CloneToChildWithUrl(Url u)
383357
{
384358
var clone = (MemberwiseClone() as SearchResultItem);
@@ -405,6 +379,7 @@ public override void Dispose()
405379
foreach (SearchResultItem item in ScannedItems) {
406380
item.Dispose();
407381
}
382+
408383
ScannedItems.Clear();
409384
}
410385

‎SmartImage.Lib/Engines/Search/Ascii2DEngine.cs‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ public sealed class Ascii2DEngine : WebSearchEngine<Ascii2DItem, IList<INode>>,
4848

4949
public Ascii2DEngine() : base(MAIN_URL)
5050
{
51-
Timeout = TimeSpan.FromSeconds(30);
52-
MaxLength = 10_000_000;
53-
Jar = new CookieJar();
51+
Timeout = TimeSpan.FromSeconds(30);
52+
MaxLength = 10_000_000;
53+
Jar = new CookieJar();
5454
m_fsClient = new FlareSolverrClient();
5555
}
5656

@@ -146,16 +146,21 @@ protected override async Task<IDocument> GetSourceAsync(SearchResult sr, SearchQ
146146

147147
if (m_fsClient.IsInitialized) {
148148

149-
var msg = new HttpRequestMessage(HttpMethod.Get, origin);
149+
try {
150+
var msg = new HttpRequestMessage(HttpMethod.Get, origin);
150151

151-
var fsr = await m_fsClient.Clearance.Solverr.SolveAsync(msg).ConfigureAwait(false);
152-
var cookies = fsr.Solution.Cookies;
153-
var newUrl = fsr.Solution.Url;
152+
var fsr = await m_fsClient.Clearance.Solverr.SolveAsync(msg).ConfigureAwait(false);
153+
var cookies = fsr.Solution.Cookies;
154+
var newUrl = fsr.Solution.Url;
154155

155-
Logger.LogTrace("{Name} using {Fs}: {CookieCnt} {NewUrl}", Name, fsr, cookies.Length, newUrl);
156+
Logger.LogTrace("{Name} using {Fs}: {CookieCnt} {NewUrl}", Name, fsr, cookies.Length, newUrl);
156157

157-
foreach (FlareSolverrCookie cookie in cookies) {
158-
Jar.AddOrReplace(new FlurlCookie(cookie.Name, cookie.Value, fsr.Solution.Url));
158+
foreach (FlareSolverrCookie cookie in cookies) {
159+
Jar.AddOrReplace(new FlurlCookie(cookie.Name, cookie.Value, fsr.Solution.Url));
160+
}
161+
}
162+
catch (Exception e) {
163+
Logger.LogError(e, "{Name}", Name);
159164
}
160165
}
161166

@@ -213,13 +218,13 @@ public async ValueTask<bool> ApplyCookiesAsync(ICookiesSource source, Cancellati
213218
return true;
214219
}
215220

216-
public ValueTask<bool> ApplyConfigAsync(SearchConfig cfg, CancellationToken ct = default)
221+
public async ValueTask<bool> ApplyConfigAsync(SearchConfig cfg, CancellationToken ct = default)
217222
{
218-
if (cfg.FlareSolverr) {
219-
m_fsClient.Configure(cfg.FlareSolverrApiUrl);
220-
}
223+
bool b = true;
224+
225+
b = await m_fsClient.ApplyConfigAsync(cfg, ct);
221226

222-
return ValueTask.FromResult(m_fsClient.IsInitialized);
227+
return b;
223228
}
224229

225230
}

‎SmartImage.Lib/Images/Uni/UniImage.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,8 @@ public virtual async ValueTask<bool> AllocImageAsync(CancellationToken ct = defa
199199
Image = await ISImage.LoadAsync(stream, ct);
200200
stream.Rewind();
201201
Hash = ImageUtilities.Hasher.Hash(stream);
202+
CalculateSimilarity(this);
203+
202204
}
203205
catch (Exception exception) {
204206
s_logger.LogError(exception, "{Value} failed to allocate image", Value);

‎SmartImage.Lib/Images/Uni/UniImageUrl.cs‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
// Author: Deci | Project: SmartImage.Lib | Name: UniImageUrl.cs
22
// Date: 2024/07/17 @ 02:07:26
33

4-
using System.Collections.Immutable;
5-
using System.Net;
6-
using System.Text.Json.Serialization;
4+
using AngleSharp.Html.Parser;
75
using Flurl.Http;
86
using Kantan.Net.Utilities;
97
using Microsoft.Extensions.Logging;
108
using Microsoft.IO;
119
using SixLabors.ImageSharp.PixelFormats;
1210
using SmartImage.Lib.Engines.Results;
1311
using SmartImage.Lib.Model;
12+
using System.Collections.Concurrent;
13+
using System.Collections.Immutable;
14+
using System.Net;
15+
using System.Text.Json.Serialization;
16+
using System.Threading.Channels;
1417

1518
namespace SmartImage.Lib.Images.Uni;
1619

@@ -29,6 +32,45 @@ internal UniImageUrl(Url url) : base(url?.ToString(), UniImageType.Uri)
2932
// public override string Name => Url?.GetFileName();
3033

3134

35+
public async ValueTask<bool> ScanAsync<TUniImageUrl>(ChannelWriter<TUniImageUrl> cw, Func<string, TUniImageUrl> newItem, CancellationToken ct = default)
36+
where TUniImageUrl : UniImageUrl
37+
{
38+
var allocImageAsync = await AllocImageAsync(ct);
39+
40+
if (allocImageAsync) {
41+
cw.TryComplete();
42+
return true;
43+
}
44+
45+
await using var stream = GetStream();
46+
47+
using var sr = new StreamReader(stream);
48+
var str = await sr.ReadToEndAsync(ct);
49+
50+
var hp = new HtmlParser();
51+
var urls = ImageScanner.ParseImageUrlsByRegex(str, Url);
52+
using var doc = await hp.ParseDocumentAsync(str);
53+
54+
await cw.WaitToWriteAsync(ct);
55+
56+
await Parallel.ForEachAsync(urls, ct, async (s, token) =>
57+
{
58+
var item = newItem(s);
59+
60+
var allocImgOk = await item.AllocImageAsync(token);
61+
62+
if (allocImgOk) {
63+
await cw.WriteAsync(item, token);
64+
}
65+
else {
66+
item?.Dispose();
67+
}
68+
});
69+
cw.TryComplete();
70+
71+
return true;
72+
}
73+
3274
protected override async ValueTask<bool> AllocAsync(CancellationToken ct = default)
3375
{
3476
IFlurlResponse fres = null;

0 commit comments

Comments
 (0)