From 8a4cb0ed2e23c8a14935c6929304b747577fca62 Mon Sep 17 00:00:00 2001 From: chchatzop <35049131+chchatzop@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:22:20 +0300 Subject: [PATCH 1/6] The list can say how long each track is and how good (#567) LIST_SHOW_AUDIO_INFO (off by default) adds duration and quality after the size on every MP3 and FLAC row, in the spelling other servers' lists use: "::INFO:: 10.3MB 4m31s 320/44.1/JS". A VBR average is "~245". audio_info.py reads both with the standard library: ID3v2 skipped, the first frame confirmed by its successor, Xing/Info/VBRI for the frame count, ID3v1 excluded from the audio; FLAC's STREAMINFO with the real bitrate. Anything unreadable keeps its size only; read() never raises. A SQLite cache (LIST_AUDIO_INFO_CACHE) keyed by the row and checked against size and mtime means only new or changed files are opened after the first rebuild. Each list prunes only its own rows, and only when its rebuild publishes. @find sends a row without its audio tail before it would cut the name. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AP6LSxkr4n9dMFNSNMogmW --- audio_info.py | 336 ++++++++++++++++ defaults.py | 13 + docs/INSTALL.md | 6 + docs/UPDATES-PUBLIC.md | 1 + docs/UPDATES.md | 44 +++ list.py | 20 +- settings.conf.sample | 22 ++ settings_help.py | 2 + tests/support.py | 3 +- ...est_the_list_says_how_long_and_how_good.py | 364 ++++++++++++++++++ update_list.py | 29 ++ web/app.js | 2 + web/lang/en.json | 2 + web/lang/es.json | 4 + web/lang/fr.json | 4 + webserver.py | 6 +- 16 files changed, 855 insertions(+), 3 deletions(-) create mode 100644 audio_info.py create mode 100644 tests/test_the_list_says_how_long_and_how_good.py diff --git a/audio_info.py b/audio_info.py new file mode 100644 index 00000000..fc9c278d --- /dev/null +++ b/audio_info.py @@ -0,0 +1,336 @@ +"""Duration and quality for the list's file rows (#567). + + !DCCore Artist - Album - 01 - Track.mp3 ::INFO:: 10.3MB 4m31s 320/44.1/JS + !DCCore Artist - Album - 02 - Track.flac ::INFO:: 16.7MB 2m5s 1115/44.1/S + +After the size: the duration, then bitrate kbps / sample rate kHz / channels. +The spelling is the one other servers' lists already use, so every reader that +parses theirs - this project's own list.py included - reads ours unchanged. A +VBR MP3's number is its average, marked with a leading "~" (~245/44.1/JS); a +FLAC's is its real bitrate, what the file actually costs per second. Channels: +S stereo, JS joint stereo, DC dual channel, M mono, and "6ch" beyond two. + +STANDARD LIBRARY ONLY. Reading audio metadata normally means mutagen, and zero +third-party packages is a property of the project. MP3 and FLAC are the two the +issue asked for and the two these lists are made of; both answer from a seek +and a few KB, never a full read: + +- MP3: skip any ID3v2 tags, find the first frame whose header decodes AND whose + successor sits where the header says, decode version / layer / bitrate / + sample rate / channel mode, then look inside that frame for a Xing / Info / + VBRI header, which carries the frame count - the only honest duration for + VBR. Without one it is CBR and the duration is the audio bytes over the + bitrate (an ID3v1 tag at the end is not audio). +- FLAC: the "fLaC" magic, then the metadata blocks; STREAMINFO has the sample + rate, channels and total samples, and where the blocks end is where the audio + starts - which is what the real bitrate is measured from. + +Anything else, and anything these cannot make sense of, is None: the row keeps +its size and nothing more. A malformed file must never take a list build down, +so read() never raises. + +THE CACHE. The scan otherwise asks each file for nothing but its size; this +opens every one. Kept in SQLite (stdlib) at LIST_AUDIO_INFO_CACHE, keyed by the +row's list path and checked against the file's size and mtime, so a rebuild +re-reads only what changed - the first one pays, the rest are a stat each. +Rows for files no longer in the library are dropped when a rebuild publishes. +""" + +import os +import sqlite3 +import time + +import defaults as config + +# Extensions read at all. Everything else is size-only without being opened. +AUDIO_EXTENSIONS = (".mp3", ".flac") + +# How far past the ID3v2 tag the first MP3 frame may start. Encoders pad; a +# file whose first frame is further out than this is size-only, not a stall. +MP3_SYNC_WINDOW = 64 * 1024 +# How many FLAC metadata blocks are walked before giving up on a file. +FLAC_MAX_BLOCKS = 256 + +_BITRATES = { + (1, 1): (0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448), + (1, 2): (0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384), + (1, 3): (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320), + (2, 1): (0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256), + (2, 2): (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160), + (2, 3): (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160), +} +_SAMPLE_RATES = {1: (44100, 48000, 32000), 2: (22050, 24000, 16000), 25: (11025, 12000, 8000)} +_MP3_MODES = {0: "S", 1: "JS", 2: "DC", 3: "M"} + + +def is_audio(name): + return name.lower().endswith(AUDIO_EXTENSIONS) + + +def _id3v2_end(handle, start=0): + """The offset just past every ID3v2 tag at `start` (there may be more + than one, back to back), or `start` when there is none.""" + offset = start + for _ in range(8): + handle.seek(offset) + head = handle.read(10) + if len(head) < 10 or head[:3] != b"ID3": + break + size = 0 + for byte in head[6:10]: + if byte & 0x80: # not a synchsafe integer: not a real tag + return offset + size = (size << 7) | byte + offset += 10 + size + (10 if head[5] & 0x10 else 0) + return offset + + +def _mp3_header(data, at): + """The frame header at data[at:at+4], decoded, or None.""" + if at + 4 > len(data): + return None + word = int.from_bytes(data[at:at + 4], "big") + if (word >> 21) & 0x7FF != 0x7FF: + return None + version = {0: 25, 2: 2, 3: 1}.get((word >> 19) & 3) + layer = 4 - ((word >> 17) & 3) + bitrate_index = (word >> 12) & 0xF + rate_index = (word >> 10) & 3 + if version is None or layer == 4 or bitrate_index in (0, 15) or rate_index == 3: + return None + kbps = _BITRATES[(1 if version == 1 else 2, layer)][bitrate_index] + rate = _SAMPLE_RATES[version][rate_index] + padding = (word >> 9) & 1 + samples = 384 if layer == 1 else (1152 if layer == 2 or version == 1 else 576) + if layer == 1: + length = (12 * kbps * 1000 // rate + padding) * 4 + else: + length = samples // 8 * kbps * 1000 // rate + padding + return {"version": version, "layer": layer, "kbps": kbps, "rate": rate, + "mode": (word >> 6) & 3, "samples": samples, "length": length} + + +def _first_frame(data): + """(offset, header) of the first frame whose NEXT frame is where it says, + or None. A lone 0xFFE bit pattern inside tag padding or cover art decodes + as a header often enough that one match proves nothing.""" + at = data.find(b"\xff") + while 0 <= at < len(data) - 4: + header = _mp3_header(data, at) + if header and header["length"] > 4: + following = at + header["length"] + if following + 4 > len(data): + return at, header # nothing to confirm against; take it + nxt = _mp3_header(data, following) + if nxt and (nxt["version"], nxt["layer"], nxt["rate"]) == \ + (header["version"], header["layer"], header["rate"]): + return at, header + at = data.find(b"\xff", at + 1) + return None + + +def _read_mp3(handle, size): + start = _id3v2_end(handle) + handle.seek(start) + data = handle.read(MP3_SYNC_WINDOW) + found = _first_frame(data) + if not found: + return None + at, header = found + audio_end = size + if size >= 128: + handle.seek(size - 128) + if handle.read(3) == b"TAG": + audio_end = size - 128 + audio_bytes = audio_end - (start + at) + if audio_bytes <= 0: + return None + + frames = total_bytes = None + vbr = False + if header["version"] == 1: + side = 17 if header["mode"] == 3 else 32 + else: + side = 9 if header["mode"] == 3 else 17 + tag_at = at + 4 + side + tag = data[tag_at:tag_at + 4] + if tag in (b"Xing", b"Info") and len(data) >= tag_at + 16: + flags = int.from_bytes(data[tag_at + 4:tag_at + 8], "big") + cursor = tag_at + 8 + if flags & 1: + frames = int.from_bytes(data[cursor:cursor + 4], "big") + cursor += 4 + if flags & 2: + total_bytes = int.from_bytes(data[cursor:cursor + 4], "big") + vbr = tag == b"Xing" + elif data[at + 36:at + 40] == b"VBRI" and len(data) >= at + 54: + total_bytes = int.from_bytes(data[at + 46:at + 50], "big") + frames = int.from_bytes(data[at + 50:at + 54], "big") + vbr = True + + if frames: + seconds = frames * header["samples"] / header["rate"] + else: + vbr = False # a VBR header without a frame count says nothing usable + seconds = audio_bytes * 8 / (header["kbps"] * 1000) + if seconds <= 0: + return None + if vbr: + kbps = round((total_bytes or audio_bytes) * 8 / seconds / 1000) + else: + kbps = header["kbps"] + return {"seconds": seconds, "kbps": kbps, "rate": header["rate"], + "channels": _MP3_MODES[header["mode"]], "vbr": vbr} + + +def _read_flac(handle, size): + start = _id3v2_end(handle) + handle.seek(start) + if handle.read(4) != b"fLaC": + return None + offset = start + 4 + info = None + for _ in range(FLAC_MAX_BLOCKS): + head = handle.read(4) + if len(head) < 4: + return None + length = int.from_bytes(head[1:4], "big") + if head[0] & 0x7F == 0: + body = handle.read(length) + if length < 34 or len(body) < 34: + return None + word = int.from_bytes(body[10:18], "big") + info = {"rate": word >> 44, "channels": ((word >> 41) & 7) + 1, + "samples": word & 0xFFFFFFFFF} + else: + handle.seek(length, 1) + offset += 4 + length + if head[0] & 0x80: + break + else: + return None + if not info or not info["rate"] or not info["samples"] or offset >= size: + return None + seconds = info["samples"] / info["rate"] + channels = {1: "M", 2: "S"}.get(info["channels"], f"{info['channels']}ch") + return {"seconds": seconds, "kbps": round((size - offset) * 8 / seconds / 1000), + "rate": info["rate"], "channels": channels, "vbr": False} + + +def read(path, size=None): + """{"seconds", "kbps", "rate", "channels", "vbr"} for an MP3 or FLAC, or + None - for any other file, and for any file that cannot be read or makes + no sense. Never raises.""" + name = path.lower() + try: + if size is None: + size = os.path.getsize(path) + with open(path, "rb") as handle: + if name.endswith(".mp3"): + return _read_mp3(handle, size) + if name.endswith(".flac"): + return _read_flac(handle, size) + except Exception: + return None + return None + + +def describe(info): + """"4m31s 320/44.1/JS" from what read() returned; "" for None.""" + if not info: + return "" + seconds = int(round(info["seconds"])) + quality = f"{'~' if info.get('vbr') else ''}{info['kbps']}/{info['rate'] / 1000:g}/{info['channels']}" + return f"{seconds // 60}m{seconds % 60}s {quality}" + + +def cache_path(): + return getattr(config, "LIST_AUDIO_INFO_CACHE", "./data/audio_info.db") + + +class Cache: + """What a rebuild knows about each audio file, kept between rebuilds. + + observe() is called from the walk, once per listed audio file: it answers + from the stored row when the file's size and mtime still match, and reads + the file otherwise. suffix() is called while the list is written. publish() + drops every row this rebuild did not observe - a file removed from the + library - and is called only when the rebuild publishes, so an aborted scan + that saw half the library does not throw away the other half. + + Opening can fail (a read-only data directory, a damaged file); then open() + returns None, the caller says so, and the list is written size-only. + """ + + COMMIT_EVERY = 2000 + + def __init__(self, conn, reader=None, scope=""): + self.conn = conn + self.scope = scope + self.run = time.time_ns() + self.reader = reader or read + self.pending = 0 + self.read_count = 0 + self.reused_count = 0 + + @classmethod + def open(cls, path=None, reader=None, log=print, scope=""): + """`scope` is the list being built: each list prunes only its own + rows, so rebuilding one never empties another's.""" + path = path or cache_path() + try: + folder = os.path.dirname(path) + if folder: + os.makedirs(folder, exist_ok=True) + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE IF NOT EXISTS audio (scope TEXT, key TEXT, size INTEGER, " + "mtime INTEGER, suffix TEXT, run INTEGER, PRIMARY KEY (scope, key))") + conn.commit() + return cls(conn, reader, scope or "") + except (sqlite3.Error, OSError) as err: + log(f"[LIST-GEN] Could not open the audio info cache at {path!r} ({err}); " + f"this list is written with sizes only.") + return None + + def observe(self, key, path, size): + try: + mtime = os.stat(path).st_mtime_ns + except OSError: + return + row = self.conn.execute("SELECT size, mtime FROM audio WHERE scope = ? AND key = ?", + (self.scope, key)).fetchone() + if row and row[0] == size and row[1] == mtime: + self.conn.execute("UPDATE audio SET run = ? WHERE scope = ? AND key = ?", + (self.run, self.scope, key)) + self.reused_count += 1 + else: + suffix = describe(self.reader(path, size)) + self.conn.execute("INSERT OR REPLACE INTO audio (scope, key, size, mtime, suffix, run) " + "VALUES (?, ?, ?, ?, ?, ?)", (self.scope, key, size, mtime, suffix, self.run)) + self.read_count += 1 + self.pending += 1 + if self.pending >= self.COMMIT_EVERY: + self.conn.commit() + self.pending = 0 + + def suffix(self, key): + row = self.conn.execute("SELECT suffix FROM audio WHERE scope = ? AND key = ? AND run = ?", + (self.scope, key, self.run)).fetchone() + return row[0] if row and row[0] else "" + + def publish(self): + self.conn.execute("DELETE FROM audio WHERE scope = ? AND run != ?", (self.scope, self.run)) + self.conn.commit() + + def close(self): + try: + self.conn.commit() + self.conn.close() + except sqlite3.Error: + pass + + +def row_key(folder, filename): + """The cache key for one list row: its folder heading and its name, which + is what the list itself identifies a file by.""" + return f"{folder}\x00{filename}" diff --git a/defaults.py b/defaults.py index 9fed1c62..5790db6e 100644 --- a/defaults.py +++ b/defaults.py @@ -341,6 +341,19 @@ # and rebuilt from the lists on disk (#628); the copy can be deleted. LIST_INDEX_FILE: str = "./data/list_index.db" +# Duration and quality after the size on the list's MP3 and FLAC rows (#567): +# "::INFO:: 10.3MB 4m31s 320/44.1/JS" - the spelling other servers' lists use. +# Off by default because it OPENS every audio file, where the scan otherwise +# asks for nothing but sizes: the first rebuild with it on takes noticeably +# longer. What it read is kept in LIST_AUDIO_INFO_CACHE, checked against each +# file's size and modification time, so later rebuilds open only new or changed +# files. Read with the standard library (audio_info.py); a file it cannot read +# keeps its size and nothing more. +LIST_SHOW_AUDIO_INFO: bool = False # Put duration and bitrate after the size on MP3 and FLAC rows +# One row per audio file in the lists (about 150 bytes each). Safe to delete: +# the next rebuild reads every file again. +LIST_AUDIO_INFO_CACHE: str = "./data/audio_info.db" + # One row per thing this bot has ever sent, {relative path or archive name -> # {name, kind, count}}. Feeds the Stats page's "Most downloaded" table. Not # bounded on purpose: a bot can only send what it shares, so the row count is diff --git a/docs/INSTALL.md b/docs/INSTALL.md index da7662dd..191363be 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -250,6 +250,12 @@ Two settings decide how the result is split up: - **`SEPARATE_VIDEO_LIST`** — publishes film and series as their own list rather than mixing them in with the music. Both travel in the same archive people get by typing your bot's name, so there is no second command to learn. `LIST_VIDEO_EXTENSIONS` says which formats count, and `LIST_VIDEO_COMPANION_EXTENSIONS` (subtitles, `.nfo`, `.sfv`) says which files follow a film into its list when they sit in the same folder - so a release travels whole, while an album's `.nfo` stays with the album. Turn it off if your films and music are already in separate folders and you would rather split by folder. - **`RAR_EXTENSIONS`** — which formats make a folder packable with `!rar`. A folder needs one of these to get a row in the album list. Everything else stays listed and directly requestable; this only decides what can be packed. **`MAX_RAR_FOLDER_SIZE`** bounds how large a folder `!rar` will pack — 10 GB by default, which passes a large box set and refuses the folder somebody names hoping it is a library. Set it to 0 for no limit. +**`LIST_SHOW_AUDIO_INFO`** (off by default) adds each MP3 and FLAC file's length and quality after its size - +`::INFO:: 10.3MB 4m31s 320/44.1/JS`, the way other servers' lists show it (`~245` is a VBR average). The first +rebuild with it on opens every audio file, so on a large library expect it to take noticeably longer; what it read +is kept in `data/audio_info.db` and later rebuilds only open new or changed files. Deleting that file is safe - the +next rebuild reads everything again. + ### If your users queue with AutoQ AutoQ (the mIRC queue script most of these channels use) pastes list rows into diff --git a/docs/UPDATES-PUBLIC.md b/docs/UPDATES-PUBLIC.md index c0a277ff..1076d787 100644 --- a/docs/UPDATES-PUBLIC.md +++ b/docs/UPDATES-PUBLIC.md @@ -2,6 +2,7 @@ ## Unreleased +- **Added: your list can say how long each track is and at what quality.** Turn on *Length and quality in the list* on the Settings page (`LIST_SHOW_AUDIO_INFO`), and every MP3 and FLAC row gets its duration and bitrate/sample rate/channels after the size - `::INFO:: 10.3MB 4m31s 320/44.1/JS` - the way other servers' lists already show it. It is off by default because the first rebuild with it on opens every audio file and takes longer; after that only new or changed files are read. A file it cannot read keeps just its size, and search results drop the extra detail before they would cut a filename short. - **Added: the bot tells you when a new version of DCCore is out.** Once a day it asks GitHub for the latest release - one request, carrying nothing about your bot - and says so in the dashboard's sidebar, in the console's `status` and in the mIRC window. The dashboard has a **Check now** button, and the console (and the mIRC menu) a `checkversion` command. If it cannot reach GitHub it says why, rather than staying quiet. It is on by default and says so at every start; untick *Tell me when a new version is out* on the Settings page (or set `CHECK_FOR_UPDATES = false`) to turn it off. Update `dccore.mrc` for the menu entry. - **Added: the list can rebuild itself on a schedule.** Set **Rebuild the list automatically** (Settings, List rebuild) to `daily 04:00`, `weekly sun 04:00`, `monthly 1 03:30` or `every 12h`, and the bot runs the same rebuild `!update` does, on its own clock. If it was off at the scheduled time it catches up once when it starts; a rebuild that fails is tried again at the next scheduled time rather than over and over. The Tools page and the console's `status` show when the next one is. Off by default. - **Added: put words in quotes to search for them together.** `@find Metal Church` finds every file with both words anywhere in its name - thousands, in a big library. `@find "Metal Church"` finds only files where those words appear together, in that order (`Metal Church`, `Metal_Church`, `metal.church`), and words outside the quotes still narrow it: `@find "Metal Church" 1986`. Searches without quotes work exactly as before. The same works in the dashboard's Search tab. Other bots in the channel may not understand quotes and answer nothing to a quoted search. diff --git a/docs/UPDATES.md b/docs/UPDATES.md index c2615e6b..b49f2218 100644 --- a/docs/UPDATES.md +++ b/docs/UPDATES.md @@ -4,6 +4,50 @@ All version changes, optimizations, and bug fixes made over time in the DCCore p ## 🟨 Unreleased +### 🎚️ The list can say how long each track is and how good (#567) + +The list gave size and nothing else, while other servers' lists give duration and quality too - which is what +someone choosing between two copies of an album wants to know: 320 or VBR, is that FLAC really lossless-sized, is +track 4 the seven-minute version. **`LIST_SHOW_AUDIO_INFO`** (off by default; *Your list* on the Settings page) adds +them after the size on every MP3 and FLAC row, in the spelling those lists already use, so our own parser and other +bots' read it unchanged: + +``` +!DCCore Artist - Album - 01 - Track.mp3 ::INFO:: 10.3MB 4m31s 320/44.1/JS +!DCCore Artist - Album - 02 - Track.flac ::INFO:: 16.7MB 2m5s 1115/44.1/S +!DCCore Artist - Album - Front.jpg ::INFO:: 94.4KB +``` + +- **Standard library only** (`audio_info.py`, no mutagen). MP3: skip every ID3v2 tag, find the first frame whose + successor is where its header says (a lone sync pattern in padding proves nothing), decode it, then read a + Xing / Info / VBRI header for the frame count - the only honest duration for VBR; without one it is CBR and the + audio bytes over the bitrate, an ID3v1 tag at the end excluded. FLAC: `fLaC`, the metadata blocks walked (a big + picture block is seeked over, not read), STREAMINFO for rate, channels and samples; the bitrate is the real one, + audio bytes over duration. A few KB per file, never a full read. +- **Spelling decided here** (the issue left it open): duration `4m31s` (minutes go past 59: `72m10s`), + then `kbps/kHz/channels` with channels `S` `JS` `DC` `M` or `6ch`. A VBR average is marked with a leading + `~` (`~245/44.1/JS`) - no spelling for it was on record. +- **Anything it cannot read keeps its size and nothing more** - a malformed file, an unknown format, a read error. + `read()` never raises; a list build is never taken down by one file. +- **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index): keyed by the + row's folder and name, checked against the file's size and mtime, so only the first rebuild opens every file and + later ones only what changed. Each list prunes only its own rows, and only when its rebuild **publishes** - an + aborted scan that saw half the library does not forget the other half. The rebuild says how many it read and + how many were unchanged. A cache that cannot be opened is said, and that list is written size-only. +- **`@find` keeps the name first.** A result row goes through the line budget as before, but one that would be cut + is sent without its audio tail first, so no letter of the name - the part people paste back - is spent on it. + Search words still match the whole row, so `@find 320` narrows to 320 kbps copies. +- AutoQ is unaffected: the tail is on file rows only, after the size, where its file branch never looks; the + `!rar` rows stay exactly as they were. + +Stacked on #913, where *Your list* has room since #776 moved the rebuild limits out. +`tests/test_the_list_says_how_long_and_how_good.py` (31) builds every MP3 and FLAC byte by byte - CBR, both ID3 +tags, a tag bigger than the search window, a false sync, all four channel modes, Xing, Info, VBRI, MPEG-2, FLAC +mono / 6ch / 96 kHz / a 3 MB picture block, six kinds of broken file - plus the cache (reuse, change, prune, +per-list scope, cannot open), a real rebuild on and off, and `@find` at the exact length where the tail decides +whether the name is cut. Mutation-checked: removing the sync confirmation, the ID3v1 exclusion, the ID3v2 skip, +the per-list prune, the size/mtime check, the prune on publish or the search fallback each fails a test. + ### 🆕 The bot says when a new version of DCCore is out (#572) Nothing told an operator that a release existed: the ones who never read the repository - most of them - ran old diff --git a/list.py b/list.py index 22259516..8d60c6b9 100644 --- a/list.py +++ b/list.py @@ -560,6 +560,18 @@ def strip_info_suffix(rest): return filename.strip(), size.strip() +# The duration-and-quality tail one of OUR rows carries after its size when +# LIST_SHOW_AUDIO_INFO is on (#567): "::INFO:: 10.3MB 4m31s 320/44.1/JS", or +# "~245/44.1/JS" for a VBR average. Anchored to the end, and to the size +# token right after the marker, so nothing in a filename can match it. +_AUDIO_TAIL_RE = re.compile(r'(::INFO::\s*\S+)\s+\d+m\d+s\s+~?\d+/[\d.]+/\w+\s*$') + + +def without_audio_info(row): + """The row with its audio tail removed, or the row unchanged.""" + return _AUDIO_TAIL_RE.sub(r'\1', row) + + def _split_entry_line(line_strip): """Pull the filename and size back out of one "!..." master-list line. @@ -1336,7 +1348,13 @@ def _build(shown_match): f"{shown_match}{R} {BG_CYAN_BLOCK} {BG_RED_BLOCK} ") return f"PRIVMSG {user} :{block_match}\r\n" - oserve.queue_message(user, announce.fit_irc_line(_build, match)) + line = announce.fit_irc_line(_build, match) + # A row the budget would cut loses its audio tail + # (#567) before a single letter of its name: the name + # is what the reader pastes back to ask for the file. + if line != _build(match) and without_audio_info(match) != match: + line = announce.fit_irc_line(_build, without_audio_info(match)) + oserve.queue_message(user, line) else: print(f"[SEARCH RESULT] 0 Match(es) found for {user} in {channel} on '{search_term}'") diff --git a/settings.conf.sample b/settings.conf.sample index a5aad075..bb914879 100644 --- a/settings.conf.sample +++ b/settings.conf.sample @@ -490,6 +490,28 @@ # and rebuilt from the lists on disk (#628); the copy can be deleted. #LIST_INDEX_FILE = ./data/list_index.db +# Add each MP3 and FLAC file's length and quality after its size in your list, +# e.g. 10.3MB 4m31s 320/44.1/JS. The first rebuild with it on opens every +# audio file and takes longer; later ones only read new or changed files. +# +# Duration and quality after the size on the list's MP3 and FLAC rows (#567): +# "::INFO:: 10.3MB 4m31s 320/44.1/JS" - the spelling other servers' lists use. +# Off by default because it OPENS every audio file, where the scan otherwise +# asks for nothing but sizes: the first rebuild with it on takes noticeably +# longer. What it read is kept in LIST_AUDIO_INFO_CACHE, checked against each +# file's size and modification time, so later rebuilds open only new or changed +# files. Read with the standard library (audio_info.py); a file it cannot read +# keeps its size and nothing more. +# Put duration and bitrate after the size on MP3 and FLAC rows +#LIST_SHOW_AUDIO_INFO = false + +# Where the length and quality read from your audio files are kept between +# rebuilds. Safe to delete; the next rebuild reads every file again. +# +# One row per audio file in the lists (about 150 bytes each). Safe to delete: +# the next rebuild reads every file again. +#LIST_AUDIO_INFO_CACHE = ./data/audio_info.db + # Where the count of how often each file was sent is kept, for the Most # downloaded table. # diff --git a/settings_help.py b/settings_help.py index 085b63ca..2a4a2d0d 100644 --- a/settings_help.py +++ b/settings_help.py @@ -275,6 +275,8 @@ def help_text(name): 'STATS_FILE': 'Where the lifetime totals, the speed record and the daily figures are saved.', 'KNOWN_BOTS_FILE': 'Where the bot remembers the other bots it has seen advertising.', 'FETCHED_BOT_LISTS_FILE': "Where the bot remembers which other bots' lists it holds.", + 'LIST_SHOW_AUDIO_INFO': 'Add each MP3 and FLAC file\'s length and quality after its size in your list, e.g. 10.3MB 4m31s 320/44.1/JS. The first rebuild with it on opens every audio file and takes longer; later ones only read new or changed files.', + 'LIST_AUDIO_INFO_CACHE': 'Where the length and quality read from your audio files are kept between rebuilds. Safe to delete; the next rebuild reads every file again.', 'LIST_INDEX_FILE': 'The search index over every list you have fetched from other bots. Can be large; safe to delete, it is rebuilt at the next fetch.', 'FETCH_HISTORY_FILE': 'Where finished downloads from other bots are recorded for the Downloads page.', 'DOWNLOAD_COUNTS_FILE': 'Where the count of how often each file was sent is kept, for the Most downloaded table.', diff --git a/tests/support.py b/tests/support.py index d6a2e319..f98288fa 100644 --- a/tests/support.py +++ b/tests/support.py @@ -698,7 +698,8 @@ def setUp(self): self.set_config( BANS_FILE=os.path.join(self._fetch_history_dir, "bans.txt"), STATS_FILE=os.path.join(self._fetch_history_dir, "stats.txt"), - LIST_INDEX_FILE=os.path.join(self._fetch_history_dir, "list_index.db")) + LIST_INDEX_FILE=os.path.join(self._fetch_history_dir, "list_index.db"), + LIST_AUDIO_INFO_CACHE=os.path.join(self._fetch_history_dir, "audio_info.db")) def tearDown(self): restore_daemon_functions() diff --git a/tests/test_the_list_says_how_long_and_how_good.py b/tests/test_the_list_says_how_long_and_how_good.py new file mode 100644 index 00000000..43625cce --- /dev/null +++ b/tests/test_the_list_says_how_long_and_how_good.py @@ -0,0 +1,364 @@ +"""#567: duration and quality after the size, on the list's MP3 and FLAC rows. + + !DCCore Artist - Album - 01 - Track.mp3 ::INFO:: 10.3MB 4m31s 320/44.1/JS + +Every audio file here is built byte by byte in the test - frame headers, a +Xing / Info / VBRI header, a FLAC STREAMINFO block - so the expected numbers +come from the formats' own arithmetic, not from a sample file nobody can +regenerate. Names are invented. +""" + +import io +import os +import sys +import unittest +from contextlib import redirect_stdout + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +import announce # noqa: E402 +import audio_info # noqa: E402 +import list as list_mod # noqa: E402 +import update_list # noqa: E402 + +from tests.support import DCCoreTestCase, RecordingSocket # noqa: E402 +from tests.test_webserver import write_master_list # noqa: E402 + +# MPEG-1 Layer III, 128 kbps, 44.1 kHz, no CRC, no padding. The last byte is +# the channel mode: 0x00 stereo, 0x40 joint stereo, 0x80 dual, 0xC0 mono. +MPEG1_128 = b"\xff\xfb\x90" +FRAME_128 = 417 # 144 * 128000 / 44100, rounded down +# MPEG-2 Layer III, 64 kbps, 22.05 kHz: 72 * 64000 / 22050 -> 208 bytes. +MPEG2_64 = b"\xff\xf3\x80" +FRAME_64 = 208 + + +def frames(count, header=MPEG1_128, mode=0x40, length=FRAME_128): + one = header + bytes([mode]) + b"\x00" * (length - 4) + return one * count + + +def id3v2(body_size=1000): + size = bytes([(body_size >> 21) & 0x7F, (body_size >> 14) & 0x7F, + (body_size >> 7) & 0x7F, body_size & 0x7F]) + # A frame-sync pattern inside the tag, which must be skipped with it. + body = (b"\xff\xfb\x90\x40" * 8).ljust(body_size, b"\x00") + return b"ID3\x03\x00\x00" + size + body + + +def id3v1(): + return b"TAG" + b"\x00" * 125 + + +def xing_frame(tag, frame_count, byte_count, mode=0x40, side=32): + frame = bytearray(frames(1, mode=mode)) + at = 4 + side + frame[at:at + 16] = (tag + (3).to_bytes(4, "big") + frame_count.to_bytes(4, "big") + + byte_count.to_bytes(4, "big")) + return bytes(frame) + + +def vbri_frame(frame_count, byte_count): + frame = bytearray(frames(1)) + frame[36:54] = (b"VBRI" + (1).to_bytes(2, "big") + b"\x00\x00" + b"\x00\x4b" + + byte_count.to_bytes(4, "big") + frame_count.to_bytes(4, "big")) + return bytes(frame) + + +def flac(rate=44100, channels=2, bits=16, samples=44100 * 2, audio_bytes=278750, padding=100): + word = (rate << 44) | ((channels - 1) << 41) | ((bits - 1) << 36) | samples + streaminfo = (b"\x10\x00\x10\x00" + b"\x00" * 6 + word.to_bytes(8, "big") + b"\x00" * 16) + blocks = (bytes([0x00]) + len(streaminfo).to_bytes(3, "big") + streaminfo + + bytes([0x81]) + padding.to_bytes(3, "big") + b"\x00" * padding) + return b"fLaC" + blocks + b"\x55" * audio_bytes + + +class FileCase(DCCoreTestCase): + def setUp(self): + super().setUp() + self.tree = self.make_tree() + + def write(self, name, data, folder=None): + directory = os.path.join(self.tree.music, folder) if folder else self.tree.music + os.makedirs(directory, exist_ok=True) + path = os.path.join(directory, name) + with open(path, "wb") as handle: + handle.write(data) + return path + + def described(self, name, data): + return audio_info.describe(audio_info.read(self.write(name, data))) + + +class ReadingMp3(FileCase): + def test_cbr_from_its_bytes_and_bitrate(self): + # 1000 frames of 417 bytes at 128 kbps: 417000 * 8 / 128000 = 26.06 s. + self.assertEqual(self.described("cbr.mp3", frames(1000)), "0m26s 128/44.1/JS") + + def test_tags_at_either_end_are_not_audio(self): + # 940 frames: 24.499 s of audio. Counting the ID3v1 tag's 128 bytes + # as audio would make it 24.507 s, and the row would say 0m25s. + data = id3v2() + frames(940) + id3v1() + self.assertEqual(self.described("tagged.mp3", data), "0m24s 128/44.1/JS") + + def test_a_tag_bigger_than_the_search_window_is_skipped(self): + """Embedded cover art makes ID3v2 tags of hundreds of KB; the frame + search only looks MP3_SYNC_WINDOW past where the tag ends.""" + data = id3v2(body_size=audio_info.MP3_SYNC_WINDOW * 2) + frames(1000) + self.assertEqual(self.described("cover-art.mp3", data), "0m26s 128/44.1/JS") + + def test_a_false_sync_before_the_audio_is_passed_over(self): + """A header-shaped pattern whose "next frame" is not there.""" + junk = bytes([0xFF, 0xFB, 0xE0, 0x40]) + bytes(10) # claims 320 kbps + info = audio_info.read(self.write("junk.mp3", junk + frames(1000))) + self.assertEqual(audio_info.describe(info), "0m26s 128/44.1/JS") + + def test_the_channel_modes(self): + for mode, word in ((0x00, "S"), (0x40, "JS"), (0x80, "DC"), (0xC0, "M")): + self.assertTrue(self.described(f"mode{mode}.mp3", frames(1000, mode=mode)).endswith("/" + word)) + + def test_vbr_is_the_xing_frame_count_and_an_average(self): + # 2297 frames * 1152 / 44100 = 60.003 s; 1837592 bytes over that = 245 kbps. + data = xing_frame(b"Xing", 2297, 1837592) + frames(5) + self.assertEqual(self.described("vbr.mp3", data), "1m0s ~245/44.1/JS") + + def test_a_mono_xing_header_sits_closer(self): + data = xing_frame(b"Xing", 2297, 1837592, mode=0xC0, side=17) + frames(5, mode=0xC0) + self.assertEqual(self.described("vbr-mono.mp3", data), "1m0s ~245/44.1/M") + + def test_an_info_header_is_cbr_with_an_exact_length(self): + data = xing_frame(b"Info", 2297, 0) + frames(5) + self.assertEqual(self.described("info.mp3", data), "1m0s 128/44.1/JS") + + def test_vbri(self): + data = vbri_frame(2297, 1837592) + frames(5) + self.assertEqual(self.described("vbri.mp3", data), "1m0s ~245/44.1/JS") + + def test_mpeg2_rates(self): + # 1000 frames of 208 bytes at 64 kbps: 208000 * 8 / 64000 = 26 s. + data = frames(1000, header=MPEG2_64, mode=0x00, length=FRAME_64) + self.assertEqual(self.described("low.mp3", data), "0m26s 64/22.05/S") + + +class ReadingFlac(FileCase): + def test_duration_and_real_bitrate(self): + # 88200 samples at 44.1 kHz = 2 s; 278750 audio bytes * 8 / 2 = 1115 kbps. + self.assertEqual(self.described("track.flac", flac()), "0m2s 1115/44.1/S") + + def test_channels_and_rates(self): + self.assertEqual(self.described("mono.flac", flac(channels=1)), "0m2s 1115/44.1/M") + six = self.described("six.flac", flac(rate=96000, channels=6, samples=96000 * 2)) + self.assertEqual(six, "0m2s 1115/96/6ch") + + def test_a_big_picture_block_is_skipped_not_read(self): + data = flac(padding=3 * 1024 * 1024) + self.assertEqual(self.described("art.flac", data), "0m2s 1115/44.1/S") + + +class NothingItCannotReadIsGuessed(FileCase): + def test_every_bad_file_is_none(self): + cases = { + "empty.mp3": b"", + "noise.mp3": bytes(range(256)) * 40, + "cut.flac": flac()[:20], + "notflac.flac": b"OggS" + b"\x00" * 100, + "zero-samples.flac": flac(samples=0), + "tag-only.mp3": id3v2(), + } + for name, data in cases.items(): + self.assertIsNone(audio_info.read(self.write(name, data)), name) + + def test_other_files_are_never_opened(self): + self.assertIsNone(audio_info.read(self.write("cover.jpg", frames(10)))) + self.assertFalse(audio_info.is_audio("cover.jpg")) + self.assertTrue(audio_info.is_audio("Loud.MP3")) + + def test_a_missing_file_is_none_not_an_exception(self): + self.assertIsNone(audio_info.read(os.path.join(self.tree.music, "gone.mp3"))) + + def test_describe_of_nothing_is_empty(self): + self.assertEqual(audio_info.describe(None), "") + + +class TheCache(FileCase): + def counting(self): + self.reads = [] + + def reader(path, size=None): + self.reads.append(os.path.basename(path)) + return audio_info.read(path, size) + return reader + + def open(self, scope=""): + cache = audio_info.Cache.open(reader=self.reader, scope=scope) + self.addCleanup(cache.close) + return cache + + def setUp(self): + super().setUp() + self.reader = self.counting() + self.path = self.write("a.mp3", frames(1000)) + self.size = os.path.getsize(self.path) + + def test_an_unchanged_file_is_not_read_again(self): + first = self.open() + first.observe("k", self.path, self.size) + first.publish() + first.close() + second = self.open() + second.observe("k", self.path, self.size) + self.assertEqual(self.reads, ["a.mp3"]) + self.assertEqual(second.suffix("k"), "0m26s 128/44.1/JS") + self.assertEqual((second.read_count, second.reused_count), (0, 1)) + + def test_a_changed_file_is(self): + first = self.open() + first.observe("k", self.path, self.size) + first.close() + self.write("a.mp3", frames(2000)) + second = self.open() + second.observe("k", self.path, os.path.getsize(self.path)) + self.assertEqual(self.reads, ["a.mp3", "a.mp3"]) + self.assertEqual(second.suffix("k"), "0m52s 128/44.1/JS") + + def test_a_published_rebuild_forgets_what_it_did_not_see(self): + first = self.open() + first.observe("gone", self.path, self.size) + first.publish() + first.close() + second = self.open() + second.publish() + second.close() + third = self.open() + third.observe("gone", self.path, self.size) + self.assertEqual(len(self.reads), 2, "the row was dropped, so it was read again") + + def test_a_rebuild_that_did_not_see_a_file_has_no_suffix_for_it(self): + first = self.open() + first.observe("k", self.path, self.size) + first.close() + self.assertEqual(self.open().suffix("k"), "") + + def test_one_list_never_prunes_another(self): + other = self.open(scope="other") + other.observe("k", self.path, self.size) + other.publish() + other.close() + primary = self.open(scope="") + primary.publish() + primary.close() + again = self.open(scope="other") + again.observe("k", self.path, self.size) + self.assertEqual(self.reads, ["a.mp3"], "the other list's row survived") + + def test_a_cache_that_cannot_open_says_so(self): + blocker = os.path.join(self.tree.root, "not-a-dir") + with open(blocker, "w") as handle: + handle.write("x") + said = [] + self.assertIsNone(audio_info.Cache.open(path=os.path.join(blocker, "audio.db"), log=said.append)) + self.assertIn("written with sizes only", said[0]) + + +class TheList(FileCase): + def setUp(self): + super().setUp() + self.set_config(LOCAL_LIST_DIR=self.tree.lists, LIST_BASE_NAME="DCCoreTest", + NICKNAME="DCCoreTest", RAR_ENABLED=False, LIST_FORMAT="txt") + self.write("Example Artist - 01 - Opening.mp3", frames(1000), folder="Album") + self.write("Example Artist - 02 - Closing.flac", flac(), folder="Album") + self.write("Front.jpg", b"\xff\xd8" + b"\x00" * 500, folder="Album") + self.write("Broken.mp3", b"not audio at all", folder="Album") + + def rows(self, **overrides): + self.set_config(**overrides) + buffer = io.StringIO() + with redirect_stdout(buffer): + built = update_list.generate_master_list() + self.assertTrue(built, buffer.getvalue()) + path = list_mod.find_latest_list() + with open(path, encoding="utf-8") as handle: + return {line.split(" ::INFO:: ")[0].split(" ", 1)[1]: line.rstrip("\n").split(" ::INFO:: ")[1] + for line in handle if line.startswith("!")}, buffer.getvalue() + + def test_on_the_audio_rows_carry_it_and_nothing_else_does(self): + rows, said = self.rows(LIST_SHOW_AUDIO_INFO=True) + self.assertRegex(rows["Example Artist - 01 - Opening.mp3"], r"^\S+ 0m26s 128/44\.1/JS$") + self.assertRegex(rows["Example Artist - 02 - Closing.flac"], r"^\S+ 0m2s 1115/44\.1/S$") + self.assertNotIn(" ", rows["Front.jpg"]) + self.assertNotIn(" ", rows["Broken.mp3"], "unreadable: size only") + # make_tree() ships a few audio files of its own; every one is read. + self.assertRegex(said, r"\[LIST-GEN\] Audio info: [1-9]\d* file\(s\) read, 0 unchanged") + + def test_off_nothing_is_opened_and_the_rows_are_as_before(self): + rows, said = self.rows(LIST_SHOW_AUDIO_INFO=False) + self.assertTrue(all(" " not in tail for tail in rows.values()), rows) + self.assertNotIn("Audio info", said) + self.assertFalse(os.path.exists(audio_info.cache_path())) + + def test_the_second_rebuild_reads_nothing(self): + self.rows(LIST_SHOW_AUDIO_INFO=True) + rows, said = self.rows(LIST_SHOW_AUDIO_INFO=True) + self.assertRegex(said, r"\[LIST-GEN\] Audio info: 0 file\(s\) read, [1-9]\d* unchanged") + self.assertRegex(rows["Example Artist - 01 - Opening.mp3"], r" 0m26s 128/44\.1/JS$") + + def test_a_published_rebuild_forgets_a_removed_file(self): + import sqlite3 + self.rows(LIST_SHOW_AUDIO_INFO=True) + os.remove(os.path.join(self.tree.music, "Album", "Example Artist - 02 - Closing.flac")) + self.rows(LIST_SHOW_AUDIO_INFO=True) + conn = sqlite3.connect(audio_info.cache_path()) + self.addCleanup(conn.close) + keys = [row[0] for row in conn.execute("SELECT key FROM audio")] + self.assertTrue(any(key.endswith("Opening.mp3") for key in keys), keys) + self.assertFalse(any(key.endswith("Closing.flac") for key in keys), keys) + + def test_our_own_parser_reads_the_row_back(self): + self.rows(LIST_SHOW_AUDIO_INFO=True) + entries, _total = list_mod.find_matching_entries(["opening"]) + self.assertEqual([e["filename"] for e in entries], ["Example Artist - 01 - Opening.mp3"]) + + +class Search(DCCoreTestCase): + """The name comes before the audio tail when a result must be cut.""" + + def setUp(self): + super().setUp() + self.tree = self.make_tree() + os.makedirs(self.tree.lists, exist_ok=True) + self.long_name = "Example Artist - " + "Very Long Title " * 19 + ".mp3" # fits only without its tail + write_master_list(self.tree.lists, "DCCoreTest", [(None, [ + ("Short Song.mp3", "4.1MB 4m31s 320/44.1/JS"), + (self.long_name, "9.9MB 7m2s ~245/44.1/JS"), + ])]) + self.set_config(FILE_DIRECTORY=self.tree.music, LOCAL_LIST_DIR=self.tree.lists, + LIST_BASE_NAME="DCCoreTest", NICKNAME="DCCoreTest", CHANNEL="#chan", + search_inprogress=False, update_inprogress=False) + + def find(self, term): + self.oserve.queued.clear() + list_mod.execute_search(RecordingSocket(), "dave", term, "#chan") + return [m for _u, m, *_ in self.oserve.queued if "::INFO::" in m] + + def test_a_row_that_fits_keeps_its_tail(self): + rows = self.find("short song") + self.assertEqual(len(rows), 1) + self.assertIn("Short Song.mp3 ::INFO:: 4.1MB 4m31s 320/44.1/JS", rows[0]) + + def test_a_row_that_does_not_loses_the_tail_before_the_name(self): + rows = self.find("very long title") + self.assertEqual(len(rows), 1) + self.assertLessEqual(len(rows[0].encode("utf-8")), announce.IRC_LINE_BUDGET) + self.assertIn(self.long_name + " ::INFO:: 9.9MB", rows[0]) + self.assertNotIn("7m2s", rows[0]) + + def test_the_tail_pattern_touches_only_a_tail(self): + row = "!Bot 4m31s 320/44.1/JS.mp3 ::INFO:: 4.1MB 4m31s 320/44.1/JS" + self.assertEqual(list_mod.without_audio_info(row), "!Bot 4m31s 320/44.1/JS.mp3 ::INFO:: 4.1MB") + self.assertEqual(list_mod.without_audio_info("!Bot a.mp3 ::INFO:: 4.1MB"), "!Bot a.mp3 ::INFO:: 4.1MB") + + +if __name__ == "__main__": + unittest.main() diff --git a/update_list.py b/update_list.py index e898b0d0..b9f02bca 100644 --- a/update_list.py +++ b/update_list.py @@ -13,6 +13,7 @@ import defaults as config import library import platform_compat +import audio_info # BEFORE ANYTHING PRINTS A FILENAME. This runs as its own process - the daemon # starts it with subprocess.run() and configure.py runs it directly - so @@ -1337,6 +1338,13 @@ def _on_walk_error(err): write_progress("scanning", folder_count=len(scan_folders), force=True) + # Duration and quality on the file rows (#567), when asked for. The cache + # is what keeps it affordable: only files new or changed since the last + # rebuild are opened. Unopenable, it says so and the list is size-only. + audio = None + if getattr(config, "LIST_SHOW_AUDIO_INFO", False): + audio = audio_info.Cache.open(scope=list_name or "") + for folder_number, scan_folder in enumerate(scan_folders, start=1): # Reported per folder because the folder COUNT is the one total known # before the walk starts - a file total would need a full pass to @@ -1480,10 +1488,15 @@ def _on_walk_error(err): video_files_data.append((rel_dir, file, file_bytes)) else: all_files_data.append((rel_dir, file, file_bytes)) + if audio is not None and audio_info.is_audio(file): + audio.observe(audio_info.row_key(rel_dir, file), + os.path.join(root, file), file_bytes) if walk_errors: print(f"[LIST-GEN ERROR] {len(walk_errors)} part(s) of the library could not be " "read - keeping the previous index rather than publishing a truncated one.") + if audio is not None: + audio.close() return False if denied_dirs: @@ -1793,6 +1806,13 @@ def format_size_human(b): f_rar.write(f"!{config.NICKNAME} !rar {_one_line(display_rar_folder)}\n") written_rar_folders.add(display_rar_folder) single_file_size = format_size_human(bytes_size) + # "4m31s 320/44.1/JS" after the size (#567), or nothing. After + # the size, where AutoQ never looks (see the !rar note above) + # and where other servers' lists already put it. + if audio is not None: + tail = audio.suffix(audio_info.row_key(folder, filename)) + if tail: + single_file_size = f"{single_file_size} {tail}" f.write(f"!{config.NICKNAME} {_one_line(filename)} ::INFO:: {single_file_size}\n") # The film and series list. Written after the music one and from the @@ -2032,6 +2052,12 @@ def _every_listed_row(): # left beside a fresh .rar would go on being handed out to somebody the # day the operator switched formats and the build failed. _prune_superseded_lists(keep=keep, directory=directory) + if audio is not None: + # Only a PUBLISHED rebuild forgets the files it did not see; a + # failed one may have seen half the library. + audio.publish() + print(f"[LIST-GEN] Audio info: {audio.read_count:,} file(s) read, " + f"{audio.reused_count:,} unchanged since the last rebuild.") return True except Exception as e: @@ -2050,6 +2076,9 @@ def _every_listed_row(): print("[LIST-GEN] The previous list was left untouched and is still in use.") _discard_temp_lists(*tmp_all_paths) return False + finally: + if audio is not None: + audio.close() def generate_all_lists(log=print): """Build every configured list. True only if every one of them succeeded. diff --git a/web/app.js b/web/app.js index db65bd41..9ef807ab 100644 --- a/web/app.js +++ b/web/app.js @@ -3840,6 +3840,8 @@ DCC_QUEUE_FILE: "settings.field.DCC_QUEUE_FILE", KNOWN_BOTS_FILE: "settings.field.KNOWN_BOTS_FILE", LIST_INDEX_FILE: "settings.field.LIST_INDEX_FILE", + LIST_AUDIO_INFO_CACHE: "settings.field.LIST_AUDIO_INFO_CACHE", + LIST_SHOW_AUDIO_INFO: "settings.field.LIST_SHOW_AUDIO_INFO", DOWNLOAD_COUNTS_FILE: "settings.field.DOWNLOAD_COUNTS_FILE", FETCHED_BOT_LISTS_FILE: "settings.field.FETCHED_BOT_LISTS_FILE", FETCH_HISTORY_FILE: "settings.field.FETCH_HISTORY_FILE", diff --git a/web/lang/en.json b/web/lang/en.json index c721fcff..a5af3485 100644 --- a/web/lang/en.json +++ b/web/lang/en.json @@ -448,6 +448,8 @@ "settings.field.DCC_QUEUE_FILE": "DCC queue file", "settings.field.KNOWN_BOTS_FILE": "Known bots file", "settings.field.LIST_INDEX_FILE": "Cross-list search index", + "settings.field.LIST_AUDIO_INFO_CACHE": "Audio info cache", + "settings.field.LIST_SHOW_AUDIO_INFO": "Length and quality in the list", "settings.field.DOWNLOAD_COUNTS_FILE": "Download counts file", "settings.field.FETCHED_BOT_LISTS_FILE": "Fetched bot lists file", "settings.field.FETCH_HISTORY_FILE": "Fetch history file", diff --git a/web/lang/es.json b/web/lang/es.json index 96ad4e9f..3b6a2566 100644 --- a/web/lang/es.json +++ b/web/lang/es.json @@ -448,6 +448,8 @@ "settings.field.DCC_QUEUE_FILE": "Archivo de la cola DCC", "settings.field.KNOWN_BOTS_FILE": "Archivo de bots conocidos", "settings.field.LIST_INDEX_FILE": "Índice de búsqueda entre listas", + "settings.field.LIST_AUDIO_INFO_CACHE": "Caché de información de audio", + "settings.field.LIST_SHOW_AUDIO_INFO": "Duración y calidad en la lista", "settings.field.DOWNLOAD_COUNTS_FILE": "Archivo de contadores de descargas", "settings.field.FETCHED_BOT_LISTS_FILE": "Archivo de listas de bots obtenidas", "settings.field.FETCH_HISTORY_FILE": "Archivo de historial de obtenciones", @@ -571,6 +573,8 @@ "settings.field.DCC_QUEUE_FILE.help": "Dónde se guarda la cola de envío por usuario. El bloqueo de instancia única vive junto a este archivo.", "settings.field.KNOWN_BOTS_FILE.help": "Dónde recuerda el bot a los otros bots que ha visto anunciarse.", "settings.field.LIST_INDEX_FILE.help": "El índice de búsqueda sobre todas las listas descargadas de otros bots. Puede ser grande; se puede borrar sin problema, se reconstruye en la siguiente descarga.", + "settings.field.LIST_AUDIO_INFO_CACHE.help": "Dónde se guardan entre reconstrucciones la duración y la calidad leídas de sus archivos de audio. Se puede borrar sin problema; la siguiente reconstrucción vuelve a leer todos los archivos.", + "settings.field.LIST_SHOW_AUDIO_INFO.help": "Añade tras el tamaño de cada archivo MP3 y FLAC de su lista su duración y su calidad, p. ej. 10.3MB 4m31s 320/44.1/JS. La primera reconstrucción con esta opción abre todos los archivos de audio y tarda más; las siguientes solo leen los archivos nuevos o modificados.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Dónde se guarda el recuento de cuántas veces se envió cada archivo, para la tabla de los más descargados.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Dónde recuerda el bot qué listas de otros bots tiene.", "settings.field.FETCH_HISTORY_FILE.help": "Dónde se registran las descargas terminadas desde otros bots, para la página Descargas.", diff --git a/web/lang/fr.json b/web/lang/fr.json index 19a8a3f0..910feb41 100644 --- a/web/lang/fr.json +++ b/web/lang/fr.json @@ -448,6 +448,8 @@ "settings.field.DCC_QUEUE_FILE": "Fichier de la file DCC", "settings.field.KNOWN_BOTS_FILE": "Fichier des bots connus", "settings.field.LIST_INDEX_FILE": "Index de recherche multi-listes", + "settings.field.LIST_AUDIO_INFO_CACHE": "Cache des infos audio", + "settings.field.LIST_SHOW_AUDIO_INFO": "Durée et qualité dans la liste", "settings.field.DOWNLOAD_COUNTS_FILE": "Fichier des compteurs de téléchargement", "settings.field.FETCHED_BOT_LISTS_FILE": "Fichier des listes de bots récupérées", "settings.field.FETCH_HISTORY_FILE": "Fichier d'historique des récupérations", @@ -571,6 +573,8 @@ "settings.field.DCC_QUEUE_FILE.help": "Où est enregistrée la file d'envoi par utilisateur. Le verrou d'instance unique se trouve à côté de ce fichier.", "settings.field.KNOWN_BOTS_FILE.help": "Où le bot se souvient des autres bots qu'il a vus s'annoncer.", "settings.field.LIST_INDEX_FILE.help": "L'index de recherche sur toutes les listes récupérées d'autres bots. Peut être gros ; peut être supprimé sans risque, il est reconstruit à la prochaine récupération.", + "settings.field.LIST_AUDIO_INFO_CACHE.help": "L'endroit où la durée et la qualité lues dans vos fichiers audio sont gardées entre deux reconstructions. Peut être supprimé sans risque ; la reconstruction suivante relit tous les fichiers.", + "settings.field.LIST_SHOW_AUDIO_INFO.help": "Ajoute après la taille de chaque fichier MP3 et FLAC de votre liste sa durée et sa qualité, par ex. 10.3MB 4m31s 320/44.1/JS. La première reconstruction avec cette option ouvre tous les fichiers audio et prend plus de temps ; les suivantes ne lisent que les fichiers nouveaux ou modifiés.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Où est conservé le compte des envois de chaque fichier, pour le tableau des plus téléchargés.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Où le bot se souvient des listes d'autres bots qu'il détient.", "settings.field.FETCH_HISTORY_FILE.help": "Où sont enregistrés les téléchargements terminés depuis d'autres bots, pour la page Téléchargements.", diff --git a/webserver.py b/webserver.py index a2f88f5f..1d4dd6ea 100644 --- a/webserver.py +++ b/webserver.py @@ -2386,7 +2386,8 @@ def start_list_update(): "RAR_ENABLED", "RAR_EXTENSIONS", "RAR_BINARY", "MAX_RAR_FOLDER_SIZE", "RAR_TIMEOUT", "LIST_HEADER_FILE", - "LIST_HEADER_MAX_BYTES"]), + "LIST_HEADER_MAX_BYTES", + "LIST_SHOW_AUDIO_INFO"]), # #776: when the list rebuilds by itself, beside the two limits every # rebuild runs under. Its own category because "Your list" had reached # the sixteen the grouping test allows before one becomes a dumping ground. @@ -2449,6 +2450,7 @@ def start_list_update(): "FETCHED_FILES_DIR", "BANS_FILE", "HARD_BANS_FILE", "STATS_FILE", "KNOWN_BOTS_FILE", "FETCHED_BOT_LISTS_FILE", "LIST_INDEX_FILE", + "LIST_AUDIO_INFO_CACHE", "FETCH_HISTORY_FILE", "DOWNLOAD_COUNTS_FILE", "LIST_SIZE_FILE", "LIST_RAWBYTES_FILE", "LIST_PROGRESS_FILE", "LIBRARY_FOLDERS_FILE", @@ -2530,6 +2532,8 @@ def start_list_update(): "DCC_QUEUE_FILE": "DCC queue file", "KNOWN_BOTS_FILE": "Known bots file", "LIST_INDEX_FILE": "Cross-list search index", + "LIST_AUDIO_INFO_CACHE": "Audio info cache", + "LIST_SHOW_AUDIO_INFO": "Length and quality in the list", "DOWNLOAD_COUNTS_FILE": "Download counts file", "FETCHED_BOT_LISTS_FILE": "Fetched bot lists file", "FETCH_HISTORY_FILE": "Fetch history file", From 19139e3794a4f1c54c403f1b6bab16156aab74e7 Mon Sep 17 00:00:00 2001 From: chchatzop <35049131+chchatzop@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:56:27 +0300 Subject: [PATCH 2/6] Audio info reads many files at once, one request each, within a time limit (#567) Neo's live test on a real 64,136-file NFS library measured the first version at 9.8 files a second: two hours for one rebuild, with every search and request paused. On a network mount the time is round trips. Now, the way QuickList (OmenServe's list maker) does it: - files are only noted during the walk; the ones to read are read after it, LIST_AUDIO_INFO_THREADS (16) at a time; - each file is opened unbuffered and read through a window: one 16 KB request for an ordinary MP3 or FLAC, one more for cover art; the ID3v1 tag is no longer looked for; - the cache is loaded into memory and checked against the size the directory listing gave: no stat, no read for an unchanged file; - LIST_AUDIO_INFO_MINUTES (5) bounds the reading one rebuild does; the rest show their size and the next rebuild reads them. A stopped rebuild keeps what it read. The dashboard shows the reading as its own progress phase. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AP6LSxkr4n9dMFNSNMogmW --- audio_info.py | 234 +++++++++++----- defaults.py | 10 + docs/INSTALL.md | 11 +- docs/UPDATES-PUBLIC.md | 2 +- docs/UPDATES.md | 47 +++- settings.conf.sample | 28 +- settings_help.py | 4 +- ...est_the_list_says_how_long_and_how_good.py | 249 ++++++++++++++---- update_list.py | 29 +- web/app.js | 8 + web/lang/en.json | 3 + web/lang/es.json | 7 +- web/lang/fr.json | 7 +- webserver.py | 6 +- 14 files changed, 508 insertions(+), 137 deletions(-) diff --git a/audio_info.py b/audio_info.py index fc9c278d..73273204 100644 --- a/audio_info.py +++ b/audio_info.py @@ -29,22 +29,45 @@ its size and nothing more. A malformed file must never take a list build down, so read() never raises. -THE CACHE. The scan otherwise asks each file for nothing but its size; this -opens every one. Kept in SQLite (stdlib) at LIST_AUDIO_INFO_CACHE, keyed by the -row's list path and checked against the file's size and mtime, so a rebuild -re-reads only what changed - the first one pays, the rest are a stat each. -Rows for files no longer in the library are dropped when a rebuild publishes. +FEW REQUESTS PER FILE, MANY FILES AT ONCE. Measured on a real library on an +NFS mount (#914): read one file at a time, with a stat, a 64 KB read and a +seek to the end each, it managed 9.8 files a second - two hours for 64,136 +files, every search blocked meanwhile. On a network mount the time is round +trips, not bytes. So each file is opened unbuffered and read in as few +requests as the format allows: one 16 KB read covers the ID3 header, the +first frame and its Xing header, or FLAC's STREAMINFO, in the ordinary case; +a big tag (cover art) or a big picture block costs one more. The ID3v1 tag at +the end is no longer looked for - a request to shave 128 bytes, a few +milliseconds, off a CBR duration. And the files are read several at a +time (LIST_AUDIO_INFO_THREADS), the way QuickList - OmenServe's list maker - +reads them: round trips overlap, bytes do not matter. + +THE CACHE, the way QuickList keeps its own: loaded into memory once, checked +against the SIZE the directory scan already knows - no stat, no request, for a +file that has not changed - and written back once. SQLite (stdlib) at +LIST_AUDIO_INFO_CACHE. Rows for files no longer in the library are dropped only +when a rebuild publishes; what a failed or stopped rebuild read is kept. + +A TIME LIMIT, so the first run cannot hold the bot. A rebuild pauses searches +and requests (PAUSE_ON_UPDATE), and the first one with this on has every audio +file to read. LIST_AUDIO_INFO_MINUTES bounds it: past the limit no new read is +started, the list publishes with what was read, and the rest keep their size +alone until the next rebuild reads them. Later rebuilds read only new files. """ import os import sqlite3 import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait import defaults as config # Extensions read at all. Everything else is size-only without being opened. AUDIO_EXTENSIONS = (".mp3", ".flac") +# The first request's size: the ordinary file's ID3 header, first frame and +# Xing header, or FLAC's STREAMINFO, all fit. +FIRST_READ = 16 * 1024 # How far past the ID3v2 tag the first MP3 frame may start. Encoders pad; a # file whose first frame is further out than this is size-only, not a stall. MP3_SYNC_WINDOW = 64 * 1024 @@ -67,13 +90,38 @@ def is_audio(name): return name.lower().endswith(AUDIO_EXTENSIONS) -def _id3v2_end(handle, start=0): +class _Window: + """The bytes of one file, fetched in as few requests as possible. + + Every access asks for (offset, length); it is answered from the last read + when that covers it, and otherwise costs ONE read of at least FIRST_READ + from that offset. The file is opened unbuffered, so a read is a request + and nothing is fetched behind it.""" + + def __init__(self, handle): + self.handle = handle + self.base = 0 + self.data = b"" + + def get(self, offset, length, need=None): + """Up to `length` bytes at `offset`; `need` of them must already be + in the window for it to be enough, else they are fetched.""" + need = length if need is None else need + start = offset - self.base + if 0 <= start and start + need <= len(self.data): + return self.data[start:start + length] + self.handle.seek(offset) + self.data = self.handle.read(max(length, FIRST_READ)) + self.base = offset + return self.data[:length] + + +def _id3v2_end(window, start=0): """The offset just past every ID3v2 tag at `start` (there may be more than one, back to back), or `start` when there is none.""" offset = start for _ in range(8): - handle.seek(offset) - head = handle.read(10) + head = window.get(offset, 10) if len(head) < 10 or head[:3] != b"ID3": break size = 0 @@ -129,20 +177,19 @@ def _first_frame(data): return None -def _read_mp3(handle, size): - start = _id3v2_end(handle) - handle.seek(start) - data = handle.read(MP3_SYNC_WINDOW) +def _read_mp3(window, size): + start = _id3v2_end(window) + # What the first read already holds past the tag, then - only if no frame + # is found in it - the whole search window in one more request. + data = window.get(start, FIRST_READ, need=1) found = _first_frame(data) + if not found and len(data) < MP3_SYNC_WINDOW and start + len(data) < size: + data = window.get(start, MP3_SYNC_WINDOW) + found = _first_frame(data) if not found: return None at, header = found - audio_end = size - if size >= 128: - handle.seek(size - 128) - if handle.read(3) == b"TAG": - audio_end = size - 128 - audio_bytes = audio_end - (start + at) + audio_bytes = size - (start + at) if audio_bytes <= 0: return None @@ -183,27 +230,26 @@ def _read_mp3(handle, size): "channels": _MP3_MODES[header["mode"]], "vbr": vbr} -def _read_flac(handle, size): - start = _id3v2_end(handle) - handle.seek(start) - if handle.read(4) != b"fLaC": +def _read_flac(window, size): + start = _id3v2_end(window) + if window.get(start, 4) != b"fLaC": return None offset = start + 4 info = None for _ in range(FLAC_MAX_BLOCKS): - head = handle.read(4) + head = window.get(offset, 4) if len(head) < 4: return None length = int.from_bytes(head[1:4], "big") if head[0] & 0x7F == 0: - body = handle.read(length) + body = window.get(offset + 4, length) if length < 34 or len(body) < 34: return None word = int.from_bytes(body[10:18], "big") info = {"rate": word >> 44, "channels": ((word >> 41) & 7) + 1, "samples": word & 0xFFFFFFFFF} - else: - handle.seek(length, 1) + # Any other block - a picture of megabytes - is stepped over: the next + # header is fetched where it is, not read up to. offset += 4 + length if head[0] & 0x80: break @@ -225,11 +271,14 @@ def read(path, size=None): try: if size is None: size = os.path.getsize(path) - with open(path, "rb") as handle: + if not name.endswith(AUDIO_EXTENSIONS): + return None + # Unbuffered: a read is exactly one request, nothing fetched behind it. + with open(path, "rb", buffering=0) as handle: + window = _Window(handle) if name.endswith(".mp3"): - return _read_mp3(handle, size) - if name.endswith(".flac"): - return _read_flac(handle, size) + return _read_mp3(window, size) + return _read_flac(window, size) except Exception: return None return None @@ -251,80 +300,137 @@ def cache_path(): class Cache: """What a rebuild knows about each audio file, kept between rebuilds. - observe() is called from the walk, once per listed audio file: it answers - from the stored row when the file's size and mtime still match, and reads - the file otherwise. suffix() is called while the list is written. publish() - drops every row this rebuild did not observe - a file removed from the - library - and is called only when the rebuild publishes, so an aborted scan - that saw half the library does not throw away the other half. + note() is called from the walk for each listed audio file: a file whose + size matches the stored row is answered from memory, anything else is put + aside. read_pending() then reads what was put aside, many at once and + within a time limit. suffix() answers while the list is written. publish() + stores what this rebuild knows and drops every row it did not see - a file + removed from the library - and is called only when the rebuild publishes; + close() without it keeps what was read and drops nothing, so an aborted + scan that saw half the library does not forget the other half. Opening can fail (a read-only data directory, a damaged file); then open() returns None, the caller says so, and the list is written size-only. """ - COMMIT_EVERY = 2000 - def __init__(self, conn, reader=None, scope=""): self.conn = conn self.scope = scope - self.run = time.time_ns() self.reader = reader or read - self.pending = 0 + self.known = {key: (size, suffix or "") for key, size, suffix in conn.execute( + "SELECT key, size, suffix FROM audio WHERE scope = ?", (scope,))} + self.seen = {} # key -> suffix, for every audio file this rebuild listed and knows + self.sizes = {} # key -> size, for the same + self.fresh = {} # key -> (size, suffix) read by this rebuild + self.pending = [] # (key, path, size) still to read self.read_count = 0 self.reused_count = 0 + self.left_count = 0 + self.published = False @classmethod def open(cls, path=None, reader=None, log=print, scope=""): """`scope` is the list being built: each list prunes only its own rows, so rebuilding one never empties another's.""" path = path or cache_path() + conn = None try: folder = os.path.dirname(path) if folder: os.makedirs(folder, exist_ok=True) conn = sqlite3.connect(path) + # mtime and run are kept for a cache written before #914's rework; + # nothing reads them now. conn.execute("CREATE TABLE IF NOT EXISTS audio (scope TEXT, key TEXT, size INTEGER, " "mtime INTEGER, suffix TEXT, run INTEGER, PRIMARY KEY (scope, key))") conn.commit() return cls(conn, reader, scope or "") except (sqlite3.Error, OSError) as err: + if conn is not None: + conn.close() log(f"[LIST-GEN] Could not open the audio info cache at {path!r} ({err}); " f"this list is written with sizes only.") return None - def observe(self, key, path, size): - try: - mtime = os.stat(path).st_mtime_ns - except OSError: - return - row = self.conn.execute("SELECT size, mtime FROM audio WHERE scope = ? AND key = ?", - (self.scope, key)).fetchone() - if row and row[0] == size and row[1] == mtime: - self.conn.execute("UPDATE audio SET run = ? WHERE scope = ? AND key = ?", - (self.run, self.scope, key)) + def note(self, key, path, size): + """One listed audio file. No request is made here: a file whose size + has not changed is answered from the cache, anything else waits for + read_pending().""" + hit = self.known.get(key) + if hit is not None and hit[0] == size: + self.seen[key] = hit[1] + self.sizes[key] = size self.reused_count += 1 else: - suffix = describe(self.reader(path, size)) - self.conn.execute("INSERT OR REPLACE INTO audio (scope, key, size, mtime, suffix, run) " - "VALUES (?, ?, ?, ?, ?, ?)", (self.scope, key, size, mtime, suffix, self.run)) - self.read_count += 1 - self.pending += 1 - if self.pending >= self.COMMIT_EVERY: - self.conn.commit() - self.pending = 0 + self.pending.append((key, path, size)) + + def read_pending(self, workers=16, budget=None, clock=time.monotonic, progress=None): + """Read what note() put aside, `workers` at a time. With `budget` + (seconds), no read is STARTED after it runs out - the ones in flight + finish - and the rest are left for the next rebuild. `progress(done, + total)` is called as reads complete, at least once a second.""" + total = len(self.pending) + if not total: + return + workers = max(1, int(workers)) + deadline = None if not budget else clock() + budget + queue = iter(self.pending) + running = {} + done = 0 + + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="audio-info") as pool: + def top_up(): + while len(running) < workers * 2: + if deadline is not None and clock() >= deadline: + return + item = next(queue, None) + if item is None: + return + key, path, size = item + running[pool.submit(self.reader, path, size)] = (key, size) + + top_up() + while running: + finished, _ = wait(running, timeout=1.0, return_when=FIRST_COMPLETED) + for future in finished: + key, size = running.pop(future) + try: + suffix = describe(future.result()) + except Exception: + suffix = "" + self.seen[key] = suffix + self.sizes[key] = size + self.fresh[key] = (size, suffix) + done += 1 + if progress is not None: + progress(done, total) + top_up() + + self.read_count = done + self.left_count = total - done + self.pending = [] def suffix(self, key): - row = self.conn.execute("SELECT suffix FROM audio WHERE scope = ? AND key = ? AND run = ?", - (self.scope, key, self.run)).fetchone() - return row[0] if row and row[0] else "" + return self.seen.get(key, "") + + def _save(self, rows): + self.conn.executemany( + "INSERT OR REPLACE INTO audio (scope, key, size, mtime, suffix, run) VALUES (?, ?, ?, 0, ?, 0)", + ((self.scope, key, size, suffix) for key, (size, suffix) in rows)) def publish(self): - self.conn.execute("DELETE FROM audio WHERE scope = ? AND run != ?", (self.scope, self.run)) - self.conn.commit() + """This rebuild published: keep what it saw, forget the rest.""" + with self.conn: + self.conn.execute("DELETE FROM audio WHERE scope = ?", (self.scope,)) + self._save((key, (self.sizes[key], suffix)) for key, suffix in self.seen.items()) + self.published = True def close(self): + """Without publish(): keep what was read, drop nothing.""" try: - self.conn.commit() + if not self.published and self.fresh: + with self.conn: + self._save(self.fresh.items()) self.conn.close() except sqlite3.Error: pass diff --git a/defaults.py b/defaults.py index 5790db6e..d44023a4 100644 --- a/defaults.py +++ b/defaults.py @@ -353,6 +353,16 @@ # One row per audio file in the lists (about 150 bytes each). Safe to delete: # the next rebuild reads every file again. LIST_AUDIO_INFO_CACHE: str = "./data/audio_info.db" +# How many audio files are read at once (#914). On a network mount (NFS, SMB) +# the time goes into round trips, which overlap: one at a time measured 9.8 +# files a second on a real NFS library. 1 to 64. +LIST_AUDIO_INFO_THREADS: int = 16 # Audio files read at once for length and quality +# The most time one rebuild spends reading audio files it has not read before +# (#914). A rebuild pauses searches and requests, and the first one with +# LIST_AUDIO_INFO on has the whole library to read: past this, the list +# publishes with what was read and the rest wait for the next rebuild. 0 = no +# limit. +LIST_AUDIO_INFO_MINUTES: int = 5 # Minutes one rebuild may spend reading new audio files; 0 = no limit # One row per thing this bot has ever sent, {relative path or archive name -> # {name, kind, count}}. Feeds the Stats page's "Most downloaded" table. Not diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 191363be..7e84f78d 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -251,10 +251,13 @@ Two settings decide how the result is split up: - **`RAR_EXTENSIONS`** — which formats make a folder packable with `!rar`. A folder needs one of these to get a row in the album list. Everything else stays listed and directly requestable; this only decides what can be packed. **`MAX_RAR_FOLDER_SIZE`** bounds how large a folder `!rar` will pack — 10 GB by default, which passes a large box set and refuses the folder somebody names hoping it is a library. Set it to 0 for no limit. **`LIST_SHOW_AUDIO_INFO`** (off by default) adds each MP3 and FLAC file's length and quality after its size - -`::INFO:: 10.3MB 4m31s 320/44.1/JS`, the way other servers' lists show it (`~245` is a VBR average). The first -rebuild with it on opens every audio file, so on a large library expect it to take noticeably longer; what it read -is kept in `data/audio_info.db` and later rebuilds only open new or changed files. Deleting that file is safe - the -next rebuild reads everything again. +`::INFO:: 10.3MB 4m31s 320/44.1/JS`, the way other servers' lists show it (`~245` is a VBR average). Every audio +file has to be read once. The files are read several at a time (`LIST_AUDIO_INFO_THREADS`, 16), and each rebuild +spends at most `LIST_AUDIO_INFO_MINUTES` (5) on it, since searches wait while a rebuild runs: on a large library, +or one on a network drive, the first few rebuilds each publish with part of the library read and the rest showing +its size alone, until everything has been read once. After that only new files are read, and a rebuild costs what +it did without the setting. What was read is kept in `data/audio_info.db`; deleting it is safe - the files are +read again. ### If your users queue with AutoQ diff --git a/docs/UPDATES-PUBLIC.md b/docs/UPDATES-PUBLIC.md index 1076d787..4b6edcd4 100644 --- a/docs/UPDATES-PUBLIC.md +++ b/docs/UPDATES-PUBLIC.md @@ -2,7 +2,7 @@ ## Unreleased -- **Added: your list can say how long each track is and at what quality.** Turn on *Length and quality in the list* on the Settings page (`LIST_SHOW_AUDIO_INFO`), and every MP3 and FLAC row gets its duration and bitrate/sample rate/channels after the size - `::INFO:: 10.3MB 4m31s 320/44.1/JS` - the way other servers' lists already show it. It is off by default because the first rebuild with it on opens every audio file and takes longer; after that only new or changed files are read. A file it cannot read keeps just its size, and search results drop the extra detail before they would cut a filename short. +- **Added: your list can say how long each track is and at what quality.** Turn on *Length and quality in the list* on the Settings page (`LIST_SHOW_AUDIO_INFO`), and every MP3 and FLAC row gets its duration and bitrate/sample rate/channels after the size - `::INFO:: 10.3MB 4m31s 320/44.1/JS` - the way other servers' lists already show it. Every audio file has to be read once, several at a time; on a large library or a network drive that is spread over a few rebuilds, each spending at most 5 minutes on it (*Time limit for reading audio files* under *List rebuild*), so searches are never held up for long. After that only new or changed files are read. A file it cannot read keeps just its size, and search results drop the extra detail before they would cut a filename short. - **Added: the bot tells you when a new version of DCCore is out.** Once a day it asks GitHub for the latest release - one request, carrying nothing about your bot - and says so in the dashboard's sidebar, in the console's `status` and in the mIRC window. The dashboard has a **Check now** button, and the console (and the mIRC menu) a `checkversion` command. If it cannot reach GitHub it says why, rather than staying quiet. It is on by default and says so at every start; untick *Tell me when a new version is out* on the Settings page (or set `CHECK_FOR_UPDATES = false`) to turn it off. Update `dccore.mrc` for the menu entry. - **Added: the list can rebuild itself on a schedule.** Set **Rebuild the list automatically** (Settings, List rebuild) to `daily 04:00`, `weekly sun 04:00`, `monthly 1 03:30` or `every 12h`, and the bot runs the same rebuild `!update` does, on its own clock. If it was off at the scheduled time it catches up once when it starts; a rebuild that fails is tried again at the next scheduled time rather than over and over. The Tools page and the console's `status` show when the next one is. Off by default. - **Added: put words in quotes to search for them together.** `@find Metal Church` finds every file with both words anywhere in its name - thousands, in a big library. `@find "Metal Church"` finds only files where those words appear together, in that order (`Metal Church`, `Metal_Church`, `metal.church`), and words outside the quotes still narrow it: `@find "Metal Church" 1986`. Searches without quotes work exactly as before. The same works in the dashboard's Search tab. Other bots in the channel may not understand quotes and answer nothing to a quoted search. diff --git a/docs/UPDATES.md b/docs/UPDATES.md index b49f2218..37fd600a 100644 --- a/docs/UPDATES.md +++ b/docs/UPDATES.md @@ -21,19 +21,37 @@ bots' read it unchanged: - **Standard library only** (`audio_info.py`, no mutagen). MP3: skip every ID3v2 tag, find the first frame whose successor is where its header says (a lone sync pattern in padding proves nothing), decode it, then read a Xing / Info / VBRI header for the frame count - the only honest duration for VBR; without one it is CBR and the - audio bytes over the bitrate, an ID3v1 tag at the end excluded. FLAC: `fLaC`, the metadata blocks walked (a big - picture block is seeked over, not read), STREAMINFO for rate, channels and samples; the bitrate is the real one, - audio bytes over duration. A few KB per file, never a full read. + audio bytes over the bitrate. FLAC: `fLaC`, the metadata blocks walked (a big picture block is stepped over, not + read), STREAMINFO for rate, channels and samples; the bitrate is the real one, audio bytes over duration. - **Spelling decided here** (the issue left it open): duration `4m31s` (minutes go past 59: `72m10s`), then `kbps/kHz/channels` with channels `S` `JS` `DC` `M` or `6ch`. A VBR average is marked with a leading `~` (`~245/44.1/JS`) - no spelling for it was on record. - **Anything it cannot read keeps its size and nothing more** - a malformed file, an unknown format, a read error. `read()` never raises; a list build is never taken down by one file. -- **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index): keyed by the - row's folder and name, checked against the file's size and mtime, so only the first rebuild opens every file and - later ones only what changed. Each list prunes only its own rows, and only when its rebuild **publishes** - an - aborted scan that saw half the library does not forget the other half. The rebuild says how many it read and - how many were unchanged. A cache that cannot be opened is said, and that list is written size-only. +- **Built for a network mount, after a live test on one.** The first version read one file at a time during the + walk - a stat, a 64 KB read and a seek to the end for an ID3v1 tag each - and on the operator's real library + (64,136 files, 1.85 TB, NFS) managed **9.8 files a second**: two hours for one rebuild, with `PAUSE_ON_UPDATE` + refusing every search and request meanwhile (Neo's report on #914; the run was stopped, cleanly). On a network + mount the time is round trips, not bytes, so it now works the way QuickList - OmenServe's list maker - does: + - **One request for an ordinary file.** Opened unbuffered and read through a window that fetches only what it + does not hold: one 16 KB read covers the ID3 header, the first frame and its Xing header, or FLAC's STREAMINFO. + A big ID3 tag or a picture block in the middle costs one more; the ID3v1 tag is no longer looked for (a request + for 128 bytes, 8 ms of a 128 kbps file). + - **Many files at once.** Files are only *noted* during the walk; the ones to read are read after it, + `LIST_AUDIO_INFO_THREADS` (16) at a time, so the round trips overlap. With 5 ms of simulated latency per + request, 16 workers read 35 times as many files a second as one. + - **No request at all for an unchanged file.** The cache is loaded into memory once and checked against the size + the directory listing already gave - no stat, no read - and written back once. + - **A time limit.** `LIST_AUDIO_INFO_MINUTES` (5; 0 = none) bounds the reading one rebuild does: past it no read is + started, the list publishes with what was read, the rest keep their size alone and the next rebuild reads them. + The dashboard shows *Reading length and quality: n of m files*, which also keeps the stall check fed. + The two live under *List rebuild* on the Settings page. +- **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index), keyed by the + row's folder and name. Each list prunes only its own rows, and only when its rebuild **publishes**; a rebuild that + fails or is stopped keeps what it read and forgets nothing. A file that could not be read is remembered as such + until its size changes, so a broken file is not re-read on every rebuild. The rebuild says how many it read, how + many were unchanged and how many are left. A cache that cannot be opened is said, and that list is written + size-only. (A cache from the first version is read as it is.) - **`@find` keeps the name first.** A result row goes through the line budget as before, but one that would be cut is sent without its audio tail first, so no letter of the name - the part people paste back - is spent on it. Search words still match the whole row, so `@find 320` narrows to 320 kbps copies. @@ -41,12 +59,15 @@ bots' read it unchanged: `!rar` rows stay exactly as they were. Stacked on #913, where *Your list* has room since #776 moved the rebuild limits out. -`tests/test_the_list_says_how_long_and_how_good.py` (31) builds every MP3 and FLAC byte by byte - CBR, both ID3 +`tests/test_the_list_says_how_long_and_how_good.py` (40) builds every MP3 and FLAC byte by byte - CBR, both ID3 tags, a tag bigger than the search window, a false sync, all four channel modes, Xing, Info, VBRI, MPEG-2, FLAC -mono / 6ch / 96 kHz / a 3 MB picture block, six kinds of broken file - plus the cache (reuse, change, prune, -per-list scope, cannot open), a real rebuild on and off, and `@find` at the exact length where the tail decides -whether the name is cut. Mutation-checked: removing the sync confirmation, the ID3v1 exclusion, the ID3v2 skip, -the per-list prune, the size/mtime check, the prune on publish or the search fallback each fails a test. +mono / 6ch / 96 kHz / a 3 MB picture block, six kinds of broken file - plus the reads each file costs (one, or two +with cover art), the cache (no request for an unchanged file, a changed size, prune, a stopped rebuild, per-list +scope, cannot open), reads that provably overlap (a barrier only four concurrent reads pass), the time limit, a +reader that raises, a real rebuild on, off and out of time, and `@find` at the exact length where the tail decides +whether the name is cut. Mutation-checked: removing the sync confirmation, the ID3v2 skip, the window's reuse, the +16 KB first read, the size check, the parallelism, the time limit, the save on a stopped rebuild, the per-list +prune, the prune on publish or the search fallback each fails a test. ### 🆕 The bot says when a new version of DCCore is out (#572) diff --git a/settings.conf.sample b/settings.conf.sample index bb914879..c537af14 100644 --- a/settings.conf.sample +++ b/settings.conf.sample @@ -491,8 +491,10 @@ #LIST_INDEX_FILE = ./data/list_index.db # Add each MP3 and FLAC file's length and quality after its size in your list, -# e.g. 10.3MB 4m31s 320/44.1/JS. The first rebuild with it on opens every -# audio file and takes longer; later ones only read new or changed files. +# e.g. 10.3MB 4m31s 320/44.1/JS. Every audio file has to be read once: on a +# large library, or one on a network drive, that takes several rebuilds, each +# limited by the reading time set under List rebuild. After that only new +# files are read. # # Duration and quality after the size on the list's MP3 and FLAC rows (#567): # "::INFO:: 10.3MB 4m31s 320/44.1/JS" - the spelling other servers' lists use. @@ -512,6 +514,28 @@ # the next rebuild reads every file again. #LIST_AUDIO_INFO_CACHE = ./data/audio_info.db +# How many audio files are read at once for their length and quality. On a +# network drive most of the time is waiting, so reading several at once is +# much faster. 1 to 64. +# +# How many audio files are read at once (#914). On a network mount (NFS, SMB) +# the time goes into round trips, which overlap: one at a time measured 9.8 +# files a second on a real NFS library. 1 to 64. +# Audio files read at once for length and quality +#LIST_AUDIO_INFO_THREADS = 16 + +# The longest one rebuild spends reading audio files it has not read before, +# since searches wait while it runs. Past it, the list is published and the +# rest are read by the next rebuild. 0 means no limit. +# +# The most time one rebuild spends reading audio files it has not read before +# (#914). A rebuild pauses searches and requests, and the first one with +# LIST_AUDIO_INFO on has the whole library to read: past this, the list +# publishes with what was read and the rest wait for the next rebuild. 0 = no +# limit. +# Minutes one rebuild may spend reading new audio files; 0 = no limit +#LIST_AUDIO_INFO_MINUTES = 5 + # Where the count of how often each file was sent is kept, for the Most # downloaded table. # diff --git a/settings_help.py b/settings_help.py index 2a4a2d0d..493fc684 100644 --- a/settings_help.py +++ b/settings_help.py @@ -275,7 +275,9 @@ def help_text(name): 'STATS_FILE': 'Where the lifetime totals, the speed record and the daily figures are saved.', 'KNOWN_BOTS_FILE': 'Where the bot remembers the other bots it has seen advertising.', 'FETCHED_BOT_LISTS_FILE': "Where the bot remembers which other bots' lists it holds.", - 'LIST_SHOW_AUDIO_INFO': 'Add each MP3 and FLAC file\'s length and quality after its size in your list, e.g. 10.3MB 4m31s 320/44.1/JS. The first rebuild with it on opens every audio file and takes longer; later ones only read new or changed files.', + 'LIST_SHOW_AUDIO_INFO': 'Add each MP3 and FLAC file\'s length and quality after its size in your list, e.g. 10.3MB 4m31s 320/44.1/JS. Every audio file has to be read once: on a large library, or one on a network drive, that takes several rebuilds, each limited by the reading time set under List rebuild. After that only new files are read.', + 'LIST_AUDIO_INFO_THREADS': 'How many audio files are read at once for their length and quality. On a network drive most of the time is waiting, so reading several at once is much faster. 1 to 64.', + 'LIST_AUDIO_INFO_MINUTES': 'The longest one rebuild spends reading audio files it has not read before, since searches wait while it runs. Past it, the list is published and the rest are read by the next rebuild. 0 means no limit.', 'LIST_AUDIO_INFO_CACHE': 'Where the length and quality read from your audio files are kept between rebuilds. Safe to delete; the next rebuild reads every file again.', 'LIST_INDEX_FILE': 'The search index over every list you have fetched from other bots. Can be large; safe to delete, it is rebuilt at the next fetch.', 'FETCH_HISTORY_FILE': 'Where finished downloads from other bots are recorded for the Downloads page.', diff --git a/tests/test_the_list_says_how_long_and_how_good.py b/tests/test_the_list_says_how_long_and_how_good.py index 43625cce..249b4497 100644 --- a/tests/test_the_list_says_how_long_and_how_good.py +++ b/tests/test_the_list_says_how_long_and_how_good.py @@ -67,11 +67,16 @@ def vbri_frame(frame_count, byte_count): return bytes(frame) -def flac(rate=44100, channels=2, bits=16, samples=44100 * 2, audio_bytes=278750, padding=100): +def flac(rate=44100, channels=2, bits=16, samples=44100 * 2, audio_bytes=278750, padding=100, + picture=0): + """A FLAC: STREAMINFO, then a PICTURE block of `picture` bytes if asked + for, then a last PADDING block, then the audio.""" word = (rate << 44) | ((channels - 1) << 41) | ((bits - 1) << 36) | samples streaminfo = (b"\x10\x00\x10\x00" + b"\x00" * 6 + word.to_bytes(8, "big") + b"\x00" * 16) - blocks = (bytes([0x00]) + len(streaminfo).to_bytes(3, "big") + streaminfo - + bytes([0x81]) + padding.to_bytes(3, "big") + b"\x00" * padding) + blocks = bytes([0x00]) + len(streaminfo).to_bytes(3, "big") + streaminfo + if picture: + blocks += bytes([0x06]) + picture.to_bytes(3, "big") + b"\x00" * picture + blocks += bytes([0x81]) + padding.to_bytes(3, "big") + b"\x00" * padding return b"fLaC" + blocks + b"\x55" * audio_bytes @@ -97,11 +102,11 @@ def test_cbr_from_its_bytes_and_bitrate(self): # 1000 frames of 417 bytes at 128 kbps: 417000 * 8 / 128000 = 26.06 s. self.assertEqual(self.described("cbr.mp3", frames(1000)), "0m26s 128/44.1/JS") - def test_tags_at_either_end_are_not_audio(self): - # 940 frames: 24.499 s of audio. Counting the ID3v1 tag's 128 bytes - # as audio would make it 24.507 s, and the row would say 0m25s. - data = id3v2() + frames(940) + id3v1() - self.assertEqual(self.described("tagged.mp3", data), "0m24s 128/44.1/JS") + def test_a_leading_tag_is_not_audio_and_a_trailing_one_changes_nothing(self): + # An ID3v1 tag at the end is no longer looked for (#914): a request of + # its own, for 128 bytes - 8 ms of a 128 kbps file. + data = id3v2() + frames(1000) + id3v1() + self.assertEqual(self.described("tagged.mp3", data), "0m26s 128/44.1/JS") def test_a_tag_bigger_than_the_search_window_is_skipped(self): """Embedded cover art makes ID3v2 tags of hundreds of KB; the frame @@ -182,6 +187,56 @@ def test_describe_of_nothing_is_empty(self): self.assertEqual(audio_info.describe(None), "") +class CountingOpen: + """audio_info's open(), counting the reads each file costs - on a network + mount every one is a round trip (#914).""" + + def __init__(self, case): + self.reads = [] + real = open + + class Counted: + def __init__(inner, path, *args, **kwargs): + inner.file = real(path, *args, **kwargs) + self.reads.append(0) + + def read(inner, *args): + self.reads[-1] += 1 + return inner.file.read(*args) + + def seek(inner, *args): + return inner.file.seek(*args) + + def __enter__(inner): + return inner + + def __exit__(inner, *exc): + inner.file.close() + + audio_info.open = Counted + case.addCleanup(delattr, audio_info, "open") + + +class FewRequestsPerFile(FileCase): + def test_an_ordinary_mp3_or_flac_is_one_read(self): + counted = CountingOpen(self) + self.described("cbr.mp3", id3v2() + frames(1000)) + self.described("vbr.mp3", xing_frame(b"Xing", 2297, 1837592) + frames(5)) + self.described("track.flac", flac()) + self.assertEqual(counted.reads, [1, 1, 1]) + + def test_cover_art_costs_one_more(self): + counted = CountingOpen(self) + self.assertEqual(self.described("art.mp3", id3v2(body_size=200 * 1024) + frames(1000)), + "0m26s 128/44.1/JS") + # The picture is stepped over: the block header after it is fetched + # where it is, not read up to. + self.assertEqual(self.described("art.flac", flac(picture=3 * 1024 * 1024)), "0m2s 1115/44.1/S") + # A big LAST block needs nothing after it: the audio starts there. + self.assertEqual(self.described("pad.flac", flac(padding=3 * 1024 * 1024)), "0m2s 1115/44.1/S") + self.assertEqual(counted.reads, [2, 2, 1]) + + class TheCache(FileCase): def counting(self): self.reads = [] @@ -196,61 +251,77 @@ def open(self, scope=""): self.addCleanup(cache.close) return cache + def build(self, scope="", keys=("k",), publish=True): + """One rebuild's worth: note, read, publish, close.""" + cache = self.open(scope) + for key in keys: + cache.note(key, self.path, os.path.getsize(self.path)) + cache.read_pending(workers=2) + if publish: + cache.publish() + cache.close() + return cache + def setUp(self): super().setUp() self.reader = self.counting() self.path = self.write("a.mp3", frames(1000)) - self.size = os.path.getsize(self.path) def test_an_unchanged_file_is_not_read_again(self): - first = self.open() - first.observe("k", self.path, self.size) - first.publish() - first.close() - second = self.open() - second.observe("k", self.path, self.size) + self.build() + second = self.build() self.assertEqual(self.reads, ["a.mp3"]) self.assertEqual(second.suffix("k"), "0m26s 128/44.1/JS") self.assertEqual((second.read_count, second.reused_count), (0, 1)) - def test_a_changed_file_is(self): - first = self.open() - first.observe("k", self.path, self.size) - first.close() + def test_note_makes_no_request_at_all(self): + """The whole point on a network mount: an unchanged file costs no + stat and no read - its size came from the directory listing.""" + self.build() + calls = [] + real_stat = os.stat + cache = self.open() + size = os.path.getsize(self.path) + + def counting_stat(*args, **kwargs): + calls.append(args[0]) + return real_stat(*args, **kwargs) + + os.stat = counting_stat + try: + cache.note("k", self.path, size) + finally: + os.stat = real_stat + self.assertEqual(calls, []) + self.assertEqual(cache.pending, []) + + def test_a_changed_size_is_read_again(self): + self.build() self.write("a.mp3", frames(2000)) - second = self.open() - second.observe("k", self.path, os.path.getsize(self.path)) + second = self.build() self.assertEqual(self.reads, ["a.mp3", "a.mp3"]) self.assertEqual(second.suffix("k"), "0m52s 128/44.1/JS") def test_a_published_rebuild_forgets_what_it_did_not_see(self): - first = self.open() - first.observe("gone", self.path, self.size) - first.publish() - first.close() - second = self.open() - second.publish() - second.close() - third = self.open() - third.observe("gone", self.path, self.size) - self.assertEqual(len(self.reads), 2, "the row was dropped, so it was read again") + self.build(keys=("gone", "kept")) + self.build(keys=("kept",)) + self.build(keys=("gone", "kept")) + self.assertEqual(self.reads.count("a.mp3"), 3, "gone and kept once, then gone again") + + def test_a_stopped_rebuild_keeps_what_it_read_and_forgets_nothing(self): + self.build(keys=("old",)) + self.build(keys=("new",), publish=False) + self.build(keys=("old", "new")) + self.assertEqual(len(self.reads), 2, "neither was read twice") def test_a_rebuild_that_did_not_see_a_file_has_no_suffix_for_it(self): - first = self.open() - first.observe("k", self.path, self.size) - first.close() + self.build() self.assertEqual(self.open().suffix("k"), "") def test_one_list_never_prunes_another(self): - other = self.open(scope="other") - other.observe("k", self.path, self.size) - other.publish() - other.close() - primary = self.open(scope="") - primary.publish() - primary.close() - again = self.open(scope="other") - again.observe("k", self.path, self.size) + self.build(scope="other") + self.build(scope="", keys=()) + self.build(scope="other") self.assertEqual(self.reads, ["a.mp3"], "the other list's row survived") def test_a_cache_that_cannot_open_says_so(self): @@ -262,6 +333,68 @@ def test_a_cache_that_cannot_open_says_so(self): self.assertIn("written with sizes only", said[0]) +class ReadingManyAtOnce(FileCase): + def pending(self, count): + cache = audio_info.Cache.open(reader=self.reader) + self.addCleanup(cache.close) + for n in range(count): + path = self.write(f"t{n}.mp3", frames(10)) + cache.note(f"k{n}", path, os.path.getsize(path)) + return cache + + def test_the_reads_really_overlap(self): + """Four reads that can only finish if four are in flight together: + a barrier of four, which a one-at-a-time reader never passes.""" + import threading + barrier = threading.Barrier(4, timeout=10) + + def reader(path, size=None): + barrier.wait() + return audio_info.read(path, size) + + self.reader = reader + cache = self.pending(4) + cache.read_pending(workers=4) + self.assertEqual(cache.read_count, 4) + self.assertTrue(all(cache.suffix(f"k{n}") for n in range(4))) + + def test_past_the_time_limit_no_read_is_started(self): + """A clock that says the budget ran out after two reads were started: + those finish, nothing else starts, and the rest are counted as left.""" + started = [] + ticks = iter([0, 0, 0, 999] + [999] * 50) + + def reader(path, size=None): + started.append(path) + return audio_info.read(path, size) + + self.reader = reader + cache = self.pending(6) + cache.read_pending(workers=1, budget=60, clock=lambda: next(ticks)) + self.assertEqual(len(started), 2) + self.assertEqual((cache.read_count, cache.left_count), (2, 4)) + self.assertEqual(cache.suffix("k5"), "") + + def test_a_reader_that_raises_costs_only_its_file(self): + def reader(path, size=None): + if path.endswith("t1.mp3"): + raise OSError("stale NFS handle") + return audio_info.read(path, size) + + self.reader = reader + cache = self.pending(3) + cache.read_pending(workers=2) + self.assertEqual(cache.suffix("k1"), "") + self.assertTrue(cache.suffix("k0") and cache.suffix("k2")) + + def test_progress_is_reported(self): + self.reader = audio_info.read + cache = self.pending(3) + seen = [] + cache.read_pending(workers=2, progress=lambda done, total: seen.append((done, total))) + self.assertEqual(seen[-1], (3, 3)) + + class TheList(FileCase): def setUp(self): super().setUp() @@ -272,11 +405,21 @@ def setUp(self): self.write("Front.jpg", b"\xff\xd8" + b"\x00" * 500, folder="Album") self.write("Broken.mp3", b"not audio at all", folder="Album") - def rows(self, **overrides): + def rows(self, _clock=None, **overrides): self.set_config(**overrides) buffer = io.StringIO() - with redirect_stdout(buffer): - built = update_list.generate_master_list() + real = audio_info.Cache.read_pending + if _clock is not None: + ticks = iter(_clock) + + def with_clock(cache, **kwargs): + return real(cache, clock=lambda: next(ticks), **kwargs) + audio_info.Cache.read_pending = with_clock + try: + with redirect_stdout(buffer): + built = update_list.generate_master_list() + finally: + audio_info.Cache.read_pending = real self.assertTrue(built, buffer.getvalue()) path = list_mod.find_latest_list() with open(path, encoding="utf-8") as handle: @@ -291,6 +434,7 @@ def test_on_the_audio_rows_carry_it_and_nothing_else_does(self): self.assertNotIn(" ", rows["Broken.mp3"], "unreadable: size only") # make_tree() ships a few audio files of its own; every one is read. self.assertRegex(said, r"\[LIST-GEN\] Audio info: [1-9]\d* file\(s\) read, 0 unchanged") + self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 16 at a time, for at most 5 minute\(s\)") def test_off_nothing_is_opened_and_the_rows_are_as_before(self): rows, said = self.rows(LIST_SHOW_AUDIO_INFO=False) @@ -315,6 +459,19 @@ def test_a_published_rebuild_forgets_a_removed_file(self): self.assertTrue(any(key.endswith("Opening.mp3") for key in keys), keys) self.assertFalse(any(key.endswith("Closing.flac") for key in keys), keys) + def test_a_rebuild_out_of_time_publishes_and_the_next_one_finishes(self): + rows, said = self.rows(LIST_SHOW_AUDIO_INFO=True, LIST_AUDIO_INFO_THREADS=1, + LIST_AUDIO_INFO_MINUTES=1, _clock=[0, 0] + [999] * 200) + # One read started before the clock ran out; which file it was is the + # walk's order, and make_tree() has audio files of its own. + self.assertIn("not read within LIST_AUDIO_INFO_MINUTES = 1", said) + self.assertRegex(said, r"Audio info: 1 file\(s\) read, 0 unchanged since the last " + r"rebuild, [1-9]\d* left for the next one\.") + self.assertLessEqual(len([tail for tail in rows.values() if " " in tail]), 1) + rows, said = self.rows(LIST_SHOW_AUDIO_INFO=True, LIST_AUDIO_INFO_MINUTES=0) + self.assertRegex(rows["Example Artist - 01 - Opening.mp3"], r" 0m26s 128/44\.1/JS$") + self.assertRegex(rows["Example Artist - 02 - Closing.flac"], r" 0m2s 1115/44\.1/S$") + def test_our_own_parser_reads_the_row_back(self): self.rows(LIST_SHOW_AUDIO_INFO=True) entries, _total = list_mod.find_matching_entries(["opening"]) diff --git a/update_list.py b/update_list.py index b9f02bca..590c64c7 100644 --- a/update_list.py +++ b/update_list.py @@ -1489,8 +1489,11 @@ def _on_walk_error(err): else: all_files_data.append((rel_dir, file, file_bytes)) if audio is not None and audio_info.is_audio(file): - audio.observe(audio_info.row_key(rel_dir, file), - os.path.join(root, file), file_bytes) + # No request here: an unchanged file is answered + # from the cache by its size, the rest are read + # after the walk, many at once (#914). + audio.note(audio_info.row_key(rel_dir, file), + os.path.join(root, file), file_bytes) if walk_errors: print(f"[LIST-GEN ERROR] {len(walk_errors)} part(s) of the library could not be " @@ -1499,6 +1502,25 @@ def _on_walk_error(err): audio.close() return False + # The audio files new or changed since the last rebuild (#567, #914). + # After the walk rather than inside it: read several at once, where on a + # network mount the time is round trips, and within LIST_AUDIO_INFO_MINUTES + # - the rebuild is pausing every search and request meanwhile. + if audio is not None and audio.pending: + workers = max(1, min(64, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 16) or 1))) + minutes = max(0, int(getattr(config, "LIST_AUDIO_INFO_MINUTES", 5) or 0)) + listed = len(all_files_data) + len(video_files_data) + print(f"[LIST-GEN] Reading the length and quality of {len(audio.pending):,} new or changed " + f"audio file(s), {workers} at a time" + f"{f', for at most {minutes} minute(s)' if minutes else ''}...") + audio.read_pending(workers=workers, budget=minutes * 60, + progress=lambda done, total: write_progress( + "audio", folder_index=done, folder_count=total, files=listed)) + if audio.left_count: + print(f"[LIST-GEN] {audio.left_count:,} audio file(s) not read within " + f"LIST_AUDIO_INFO_MINUTES = {minutes}: they show their size alone this " + f"time, and the next rebuild reads them.") + if denied_dirs: # Said once, with the count, because it is a standing condition rather # than an event: these folders will be missing from every list until @@ -2057,7 +2079,8 @@ def _every_listed_row(): # failed one may have seen half the library. audio.publish() print(f"[LIST-GEN] Audio info: {audio.read_count:,} file(s) read, " - f"{audio.reused_count:,} unchanged since the last rebuild.") + f"{audio.reused_count:,} unchanged since the last rebuild" + f"{f', {audio.left_count:,} left for the next one' if audio.left_count else ''}.") return True except Exception as e: diff --git a/web/app.js b/web/app.js index 9ef807ab..00ead672 100644 --- a/web/app.js +++ b/web/app.js @@ -3254,6 +3254,12 @@ var parts = []; if (progress.phase === "writing") { parts.push(t("tools.writingList")); + } else if (progress.phase === "audio") { + // #914: reading length and quality - folder_index/folder_count carry + // files read / files to read, so the bar below follows it too. + parts.push(t("tools.readingAudioInfo") + .replace("{done}", (progress.folder_index || 0).toLocaleString()) + .replace("{total}", (progress.folder_count || 0).toLocaleString())); } else if (progress.folder_count) { parts.push(t("tools.scanningFolder") .replace("{index}", progress.folder_index).replace("{total}", progress.folder_count)); @@ -3842,6 +3848,8 @@ LIST_INDEX_FILE: "settings.field.LIST_INDEX_FILE", LIST_AUDIO_INFO_CACHE: "settings.field.LIST_AUDIO_INFO_CACHE", LIST_SHOW_AUDIO_INFO: "settings.field.LIST_SHOW_AUDIO_INFO", + LIST_AUDIO_INFO_MINUTES: "settings.field.LIST_AUDIO_INFO_MINUTES", + LIST_AUDIO_INFO_THREADS: "settings.field.LIST_AUDIO_INFO_THREADS", DOWNLOAD_COUNTS_FILE: "settings.field.DOWNLOAD_COUNTS_FILE", FETCHED_BOT_LISTS_FILE: "settings.field.FETCHED_BOT_LISTS_FILE", FETCH_HISTORY_FILE: "settings.field.FETCH_HISTORY_FILE", diff --git a/web/lang/en.json b/web/lang/en.json index a5af3485..2c7a3c49 100644 --- a/web/lang/en.json +++ b/web/lang/en.json @@ -345,6 +345,7 @@ "tools.starting": "Starting…", "tools.rebuildingMasterList": "Rebuilding the master list…", "tools.writingList": "Writing the list…", + "tools.readingAudioInfo": "Reading length and quality: {done} of {total} files", "tools.scanningFolder": "Scanning folder {index} of {total}", "tools.scanningLibrary": "Scanning the library…", "tools.filesSoFar": "{count} files so far", @@ -450,6 +451,8 @@ "settings.field.LIST_INDEX_FILE": "Cross-list search index", "settings.field.LIST_AUDIO_INFO_CACHE": "Audio info cache", "settings.field.LIST_SHOW_AUDIO_INFO": "Length and quality in the list", + "settings.field.LIST_AUDIO_INFO_MINUTES": "Time limit for reading audio files", + "settings.field.LIST_AUDIO_INFO_THREADS": "Audio files read at once", "settings.field.DOWNLOAD_COUNTS_FILE": "Download counts file", "settings.field.FETCHED_BOT_LISTS_FILE": "Fetched bot lists file", "settings.field.FETCH_HISTORY_FILE": "Fetch history file", diff --git a/web/lang/es.json b/web/lang/es.json index 3b6a2566..017a9328 100644 --- a/web/lang/es.json +++ b/web/lang/es.json @@ -345,6 +345,7 @@ "tools.starting": "Iniciando…", "tools.rebuildingMasterList": "Reconstruyendo la lista maestra…", "tools.writingList": "Escribiendo la lista…", + "tools.readingAudioInfo": "Leyendo duración y calidad: {done} de {total} archivos", "tools.scanningFolder": "Analizando la carpeta {index} de {total}", "tools.scanningLibrary": "Analizando la biblioteca…", "tools.filesSoFar": "{count} archivos hasta ahora", @@ -450,6 +451,8 @@ "settings.field.LIST_INDEX_FILE": "Índice de búsqueda entre listas", "settings.field.LIST_AUDIO_INFO_CACHE": "Caché de información de audio", "settings.field.LIST_SHOW_AUDIO_INFO": "Duración y calidad en la lista", + "settings.field.LIST_AUDIO_INFO_MINUTES": "Tiempo máximo para leer archivos de audio", + "settings.field.LIST_AUDIO_INFO_THREADS": "Archivos de audio leídos a la vez", "settings.field.DOWNLOAD_COUNTS_FILE": "Archivo de contadores de descargas", "settings.field.FETCHED_BOT_LISTS_FILE": "Archivo de listas de bots obtenidas", "settings.field.FETCH_HISTORY_FILE": "Archivo de historial de obtenciones", @@ -574,7 +577,9 @@ "settings.field.KNOWN_BOTS_FILE.help": "Dónde recuerda el bot a los otros bots que ha visto anunciarse.", "settings.field.LIST_INDEX_FILE.help": "El índice de búsqueda sobre todas las listas descargadas de otros bots. Puede ser grande; se puede borrar sin problema, se reconstruye en la siguiente descarga.", "settings.field.LIST_AUDIO_INFO_CACHE.help": "Dónde se guardan entre reconstrucciones la duración y la calidad leídas de sus archivos de audio. Se puede borrar sin problema; la siguiente reconstrucción vuelve a leer todos los archivos.", - "settings.field.LIST_SHOW_AUDIO_INFO.help": "Añade tras el tamaño de cada archivo MP3 y FLAC de su lista su duración y su calidad, p. ej. 10.3MB 4m31s 320/44.1/JS. La primera reconstrucción con esta opción abre todos los archivos de audio y tarda más; las siguientes solo leen los archivos nuevos o modificados.", + "settings.field.LIST_SHOW_AUDIO_INFO.help": "Añade tras el tamaño de cada archivo MP3 y FLAC de su lista su duración y su calidad, p. ej. 10.3MB 4m31s 320/44.1/JS. Cada archivo de audio se lee una vez: en una biblioteca grande, o en una unidad de red, eso lleva varias reconstrucciones, cada una limitada por el tiempo de lectura fijado en Reconstrucción de la lista. Después solo se leen los archivos nuevos.", + "settings.field.LIST_AUDIO_INFO_MINUTES.help": "El tiempo máximo que una reconstrucción dedica a leer archivos de audio que aún no ha leído, ya que las búsquedas esperan mientras tanto. Al llegar a él, la lista se publica y el resto lo lee la siguiente reconstrucción. 0 significa sin límite.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Cuántos archivos de audio se leen a la vez para obtener su duración y calidad. En una unidad de red la mayor parte del tiempo es espera, así que leer varios a la vez es mucho más rápido. De 1 a 64.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Dónde se guarda el recuento de cuántas veces se envió cada archivo, para la tabla de los más descargados.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Dónde recuerda el bot qué listas de otros bots tiene.", "settings.field.FETCH_HISTORY_FILE.help": "Dónde se registran las descargas terminadas desde otros bots, para la página Descargas.", diff --git a/web/lang/fr.json b/web/lang/fr.json index 910feb41..b57e8dea 100644 --- a/web/lang/fr.json +++ b/web/lang/fr.json @@ -345,6 +345,7 @@ "tools.starting": "Démarrage…", "tools.rebuildingMasterList": "Reconstruction de la liste principale…", "tools.writingList": "Écriture de la liste…", + "tools.readingAudioInfo": "Lecture de la durée et de la qualité : {done} sur {total} fichiers", "tools.scanningFolder": "Analyse du dossier {index} sur {total}", "tools.scanningLibrary": "Analyse de la bibliothèque…", "tools.filesSoFar": "{count} fichiers jusqu'ici", @@ -450,6 +451,8 @@ "settings.field.LIST_INDEX_FILE": "Index de recherche multi-listes", "settings.field.LIST_AUDIO_INFO_CACHE": "Cache des infos audio", "settings.field.LIST_SHOW_AUDIO_INFO": "Durée et qualité dans la liste", + "settings.field.LIST_AUDIO_INFO_MINUTES": "Durée maximale de lecture des fichiers audio", + "settings.field.LIST_AUDIO_INFO_THREADS": "Fichiers audio lus en même temps", "settings.field.DOWNLOAD_COUNTS_FILE": "Fichier des compteurs de téléchargement", "settings.field.FETCHED_BOT_LISTS_FILE": "Fichier des listes de bots récupérées", "settings.field.FETCH_HISTORY_FILE": "Fichier d'historique des récupérations", @@ -574,7 +577,9 @@ "settings.field.KNOWN_BOTS_FILE.help": "Où le bot se souvient des autres bots qu'il a vus s'annoncer.", "settings.field.LIST_INDEX_FILE.help": "L'index de recherche sur toutes les listes récupérées d'autres bots. Peut être gros ; peut être supprimé sans risque, il est reconstruit à la prochaine récupération.", "settings.field.LIST_AUDIO_INFO_CACHE.help": "L'endroit où la durée et la qualité lues dans vos fichiers audio sont gardées entre deux reconstructions. Peut être supprimé sans risque ; la reconstruction suivante relit tous les fichiers.", - "settings.field.LIST_SHOW_AUDIO_INFO.help": "Ajoute après la taille de chaque fichier MP3 et FLAC de votre liste sa durée et sa qualité, par ex. 10.3MB 4m31s 320/44.1/JS. La première reconstruction avec cette option ouvre tous les fichiers audio et prend plus de temps ; les suivantes ne lisent que les fichiers nouveaux ou modifiés.", + "settings.field.LIST_SHOW_AUDIO_INFO.help": "Ajoute après la taille de chaque fichier MP3 et FLAC de votre liste sa durée et sa qualité, par ex. 10.3MB 4m31s 320/44.1/JS. Chaque fichier audio doit être lu une fois : sur une grande bibliothèque, ou sur un lecteur réseau, cela prend plusieurs reconstructions, chacune limitée par la durée de lecture réglée dans Reconstruction de la liste. Ensuite seuls les nouveaux fichiers sont lus.", + "settings.field.LIST_AUDIO_INFO_MINUTES.help": "Le temps maximal qu'une reconstruction consacre à lire des fichiers audio pas encore lus, puisque les recherches attendent pendant ce temps. Au-delà, la liste est publiée et la reconstruction suivante lit le reste. 0 signifie sans limite.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Combien de fichiers audio sont lus en même temps pour leur durée et leur qualité. Sur un lecteur réseau, l'essentiel du temps est de l'attente : en lire plusieurs à la fois est bien plus rapide. De 1 à 64.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Où est conservé le compte des envois de chaque fichier, pour le tableau des plus téléchargés.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Où le bot se souvient des listes d'autres bots qu'il détient.", "settings.field.FETCH_HISTORY_FILE.help": "Où sont enregistrés les téléchargements terminés depuis d'autres bots, pour la page Téléchargements.", diff --git a/webserver.py b/webserver.py index 1d4dd6ea..5bf80fd7 100644 --- a/webserver.py +++ b/webserver.py @@ -2393,7 +2393,9 @@ def start_list_update(): # the sixteen the grouping test allows before one becomes a dumping ground. ("list-rebuild", "List rebuild", ["LIST_REBUILD_SCHEDULE", "LIST_UPDATE_TIMEOUT", - "LIST_UPDATE_STALL_SECONDS"]), + "LIST_UPDATE_STALL_SECONDS", + "LIST_AUDIO_INFO_MINUTES", + "LIST_AUDIO_INFO_THREADS"]), ("fetching", "Fetching from bots", ["MAX_FETCH_SLOTS", "AUTO_REFETCH_LISTS", "AUTO_REFETCH_INTERVAL_HOURS", "AUTO_REFETCH_MAX_PER_RUN", @@ -2534,6 +2536,8 @@ def start_list_update(): "LIST_INDEX_FILE": "Cross-list search index", "LIST_AUDIO_INFO_CACHE": "Audio info cache", "LIST_SHOW_AUDIO_INFO": "Length and quality in the list", + "LIST_AUDIO_INFO_MINUTES": "Time limit for reading audio files", + "LIST_AUDIO_INFO_THREADS": "Audio files read at once", "DOWNLOAD_COUNTS_FILE": "Download counts file", "FETCHED_BOT_LISTS_FILE": "Fetched bot lists file", "FETCH_HISTORY_FILE": "Fetch history file", From d7710299a20a23f2ae925f4f3c68b5872ef5db74 Mon Sep 17 00:00:00 2001 From: chchatzop <35049131+chchatzop@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:00:48 +0300 Subject: [PATCH 3/6] The rebuild says how fast it read the audio files (#567) Neo's live re-test on the NFS library read at about 55-60 files a second with 16 workers, well below the synthetic estimate. On a network mount the ceiling is the server's; the rebuild's last line now ends "Read at N files a second, 16 at a time", so an operator can try another LIST_AUDIO_INFO_THREADS and compare. Measured on the real clock, not the budget's. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AP6LSxkr4n9dMFNSNMogmW --- audio_info.py | 15 +++++++++++++++ docs/INSTALL.md | 3 ++- docs/UPDATES.md | 7 ++++++- tests/test_the_list_says_how_long_and_how_good.py | 13 +++++++++++++ update_list.py | 4 +++- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/audio_info.py b/audio_info.py index 73273204..bf673e2e 100644 --- a/audio_info.py +++ b/audio_info.py @@ -326,6 +326,8 @@ def __init__(self, conn, reader=None, scope=""): self.read_count = 0 self.reused_count = 0 self.left_count = 0 + self.workers = 0 + self.read_seconds = 0.0 self.published = False @classmethod @@ -373,6 +375,10 @@ def read_pending(self, workers=16, budget=None, clock=time.monotonic, progress=N if not total: return workers = max(1, int(workers)) + self.workers = workers + # Wall time on the real clock, not `clock` - that one is the budget's, + # and a test drives it by hand. + began = time.monotonic() deadline = None if not budget else clock() + budget queue = iter(self.pending) running = {} @@ -408,6 +414,15 @@ def top_up(): self.read_count = done self.left_count = total - done + self.read_seconds = time.monotonic() - began + + def rate(self): + """Files read per second of reading, or None when nothing was read. + What an operator compares to choose LIST_AUDIO_INFO_THREADS: on a + network mount the ceiling is the server's, not the setting's.""" + if not self.read_count or self.read_seconds <= 0: + return None + return self.read_count / self.read_seconds self.pending = [] def suffix(self, key): diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 7e84f78d..bb83682f 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -257,7 +257,8 @@ spends at most `LIST_AUDIO_INFO_MINUTES` (5) on it, since searches wait while a or one on a network drive, the first few rebuilds each publish with part of the library read and the rest showing its size alone, until everything has been read once. After that only new files are read, and a rebuild costs what it did without the setting. What was read is kept in `data/audio_info.db`; deleting it is safe - the files are -read again. +read again. The rebuild's last line says how fast the files were read; on a network drive, try a higher +`LIST_AUDIO_INFO_THREADS` once and compare - past the server's own limit it stops helping. ### If your users queue with AutoQ diff --git a/docs/UPDATES.md b/docs/UPDATES.md index 76699d41..cabca223 100644 --- a/docs/UPDATES.md +++ b/docs/UPDATES.md @@ -45,6 +45,11 @@ bots' read it unchanged: - **A time limit.** `LIST_AUDIO_INFO_MINUTES` (5; 0 = none) bounds the reading one rebuild does: past it no read is started, the list publishes with what was read, the rest keep their size alone and the next rebuild reads them. The dashboard shows *Reading length and quality: n of m files*, which also keeps the stall check fed. + - **The rate is said.** The rebuild's last line ends *Read at N files a second, 16 at a time* - on a network + mount the ceiling is the server's, so that is what an operator compares when trying another thread count. + Measured live on the NFS library (Neo, #914): three ordinary rebuilds of 5-6 minutes each read 62,657 of + the 62,699 audio files (42 unreadable) - 17 minutes in total, where the first version needed two hours in one - at about 55-60 files a + second with 16. The two live under *List rebuild* on the Settings page. - **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index), keyed by the row's folder and name. Each list prunes only its own rows, and only when its rebuild **publishes**; a rebuild that @@ -59,7 +64,7 @@ bots' read it unchanged: `!rar` rows stay exactly as they were. Stacked on #913, where *Your list* has room since #776 moved the rebuild limits out. -`tests/test_the_list_says_how_long_and_how_good.py` (40) builds every MP3 and FLAC byte by byte - CBR, both ID3 +`tests/test_the_list_says_how_long_and_how_good.py` (41) builds every MP3 and FLAC byte by byte - CBR, both ID3 tags, a tag bigger than the search window, a false sync, all four channel modes, Xing, Info, VBRI, MPEG-2, FLAC mono / 6ch / 96 kHz / a 3 MB picture block, six kinds of broken file - plus the reads each file costs (one, or two with cover art), the cache (no request for an unchanged file, a changed size, prune, a stopped rebuild, per-list diff --git a/tests/test_the_list_says_how_long_and_how_good.py b/tests/test_the_list_says_how_long_and_how_good.py index 249b4497..2ca2ec9b 100644 --- a/tests/test_the_list_says_how_long_and_how_good.py +++ b/tests/test_the_list_says_how_long_and_how_good.py @@ -387,6 +387,17 @@ def reader(path, size=None): self.assertEqual(cache.suffix("k1"), "") self.assertTrue(cache.suffix("k0") and cache.suffix("k2")) + def test_the_rate_is_measured_on_the_real_clock(self): + """Files per second of reading - what an operator compares to pick + LIST_AUDIO_INFO_THREADS - and not taken from the budget's clock, + which a test (or a frozen clock) can stop.""" + self.reader = audio_info.read + cache = self.pending(4) + self.assertIsNone(cache.rate(), "nothing read yet") + cache.read_pending(workers=2, clock=lambda: 0) + self.assertGreater(cache.rate(), 0) + self.assertEqual(cache.workers, 2) + def test_progress_is_reported(self): self.reader = audio_info.read cache = self.pending(3) @@ -434,6 +445,7 @@ def test_on_the_audio_rows_carry_it_and_nothing_else_does(self): self.assertNotIn(" ", rows["Broken.mp3"], "unreadable: size only") # make_tree() ships a few audio files of its own; every one is read. self.assertRegex(said, r"\[LIST-GEN\] Audio info: [1-9]\d* file\(s\) read, 0 unchanged") + self.assertRegex(said, r"Read at [\d,]+ files a second, 16 at a time\.") self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 16 at a time, for at most 5 minute\(s\)") def test_off_nothing_is_opened_and_the_rows_are_as_before(self): @@ -446,6 +458,7 @@ def test_the_second_rebuild_reads_nothing(self): self.rows(LIST_SHOW_AUDIO_INFO=True) rows, said = self.rows(LIST_SHOW_AUDIO_INFO=True) self.assertRegex(said, r"\[LIST-GEN\] Audio info: 0 file\(s\) read, [1-9]\d* unchanged") + self.assertNotIn("files a second", said, "no rate when nothing was read") self.assertRegex(rows["Example Artist - 01 - Opening.mp3"], r" 0m26s 128/44\.1/JS$") def test_a_published_rebuild_forgets_a_removed_file(self): diff --git a/update_list.py b/update_list.py index 590c64c7..f3037fd4 100644 --- a/update_list.py +++ b/update_list.py @@ -2078,9 +2078,11 @@ def _every_listed_row(): # Only a PUBLISHED rebuild forgets the files it did not see; a # failed one may have seen half the library. audio.publish() + rate = audio.rate() print(f"[LIST-GEN] Audio info: {audio.read_count:,} file(s) read, " f"{audio.reused_count:,} unchanged since the last rebuild" - f"{f', {audio.left_count:,} left for the next one' if audio.left_count else ''}.") + f"{f', {audio.left_count:,} left for the next one' if audio.left_count else ''}." + f"{f' Read at {rate:,.0f} files a second, {audio.workers} at a time.' if rate else ''}") return True except Exception as e: From 586b9d185094ecbb5a6634b6985a497e8f0e7832 Mon Sep 17 00:00:00 2001 From: chchatzop <35049131+chchatzop@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:11:02 +0300 Subject: [PATCH 4/6] Time the audio reads with perf_counter, not monotonic (#567) On Windows before Python 3.13, time.monotonic() ticks every ~15.6 ms. A short batch read inside one tick measured 0 s, so no rate was printed - the rate tests failed on the Windows 3.10 and 3.12 runners. perf_counter is high-resolution everywhere. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AP6LSxkr4n9dMFNSNMogmW --- audio_info.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/audio_info.py b/audio_info.py index bf673e2e..d302092f 100644 --- a/audio_info.py +++ b/audio_info.py @@ -377,8 +377,10 @@ def read_pending(self, workers=16, budget=None, clock=time.monotonic, progress=N workers = max(1, int(workers)) self.workers = workers # Wall time on the real clock, not `clock` - that one is the budget's, - # and a test drives it by hand. - began = time.monotonic() + # and a test drives it by hand. perf_counter, not monotonic: on + # Windows before 3.13 monotonic ticks every ~15.6 ms, and a short + # batch read inside one tick measured 0 s and no rate at all. + began = time.perf_counter() deadline = None if not budget else clock() + budget queue = iter(self.pending) running = {} @@ -414,7 +416,7 @@ def top_up(): self.read_count = done self.left_count = total - done - self.read_seconds = time.monotonic() - began + self.read_seconds = time.perf_counter() - began def rate(self): """Files read per second of reading, or None when nothing was read. From 37cbde9b03d5d4a2129ec5ce6898d72235d94cc7 Mon Sep 17 00:00:00 2001 From: chchatzop <35049131+chchatzop@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:25:08 +0300 Subject: [PATCH 5/6] Read 32 audio files at once by default, up to 128 (#567) Neo's cold-cache test on the real NFS library: 16 threads read about 73 files a second, 64 about 236 - partly on a server cache warmed by the run before, so not fully isolated. The default goes to 32, harmless on a local disk; the range to 1-128; and the help says a network drive should try 64, comparing the rate the rebuild reports. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01AP6LSxkr4n9dMFNSNMogmW --- defaults.py | 9 ++++++--- docs/INSTALL.md | 4 ++-- docs/UPDATES.md | 8 +++++--- settings.conf.sample | 13 ++++++++----- settings_help.py | 2 +- tests/test_the_list_says_how_long_and_how_good.py | 13 +++++++++++-- update_list.py | 2 +- web/lang/es.json | 2 +- web/lang/fr.json | 2 +- 9 files changed, 36 insertions(+), 19 deletions(-) diff --git a/defaults.py b/defaults.py index d44023a4..1248eba3 100644 --- a/defaults.py +++ b/defaults.py @@ -354,9 +354,12 @@ # the next rebuild reads every file again. LIST_AUDIO_INFO_CACHE: str = "./data/audio_info.db" # How many audio files are read at once (#914). On a network mount (NFS, SMB) -# the time goes into round trips, which overlap: one at a time measured 9.8 -# files a second on a real NFS library. 1 to 64. -LIST_AUDIO_INFO_THREADS: int = 16 # Audio files read at once for length and quality +# the time goes into round trips, which overlap. Measured on a real 64,136-file +# NFS library: one at a time 9.8 files a second, 16 about 73, 64 about 236 (the +# last partly on a cache warmed by the run before). 32 by default - harmless on +# a local disk; a network drive may want 64 or more. The rebuild's last line +# says the rate it got, to compare. 1 to 128. +LIST_AUDIO_INFO_THREADS: int = 32 # Audio files read at once for length and quality # The most time one rebuild spends reading audio files it has not read before # (#914). A rebuild pauses searches and requests, and the first one with # LIST_AUDIO_INFO on has the whole library to read: past this, the list diff --git a/docs/INSTALL.md b/docs/INSTALL.md index bb83682f..599729fe 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -252,13 +252,13 @@ Two settings decide how the result is split up: **`LIST_SHOW_AUDIO_INFO`** (off by default) adds each MP3 and FLAC file's length and quality after its size - `::INFO:: 10.3MB 4m31s 320/44.1/JS`, the way other servers' lists show it (`~245` is a VBR average). Every audio -file has to be read once. The files are read several at a time (`LIST_AUDIO_INFO_THREADS`, 16), and each rebuild +file has to be read once. The files are read several at a time (`LIST_AUDIO_INFO_THREADS`, 32), and each rebuild spends at most `LIST_AUDIO_INFO_MINUTES` (5) on it, since searches wait while a rebuild runs: on a large library, or one on a network drive, the first few rebuilds each publish with part of the library read and the rest showing its size alone, until everything has been read once. After that only new files are read, and a rebuild costs what it did without the setting. What was read is kept in `data/audio_info.db`; deleting it is safe - the files are read again. The rebuild's last line says how fast the files were read; on a network drive, try a higher -`LIST_AUDIO_INFO_THREADS` once and compare - past the server's own limit it stops helping. +`LIST_AUDIO_INFO_THREADS` (64, say) once and compare - past the server's own limit it stops helping. ### If your users queue with AutoQ diff --git a/docs/UPDATES.md b/docs/UPDATES.md index cabca223..cabb3814 100644 --- a/docs/UPDATES.md +++ b/docs/UPDATES.md @@ -38,7 +38,7 @@ bots' read it unchanged: A big ID3 tag or a picture block in the middle costs one more; the ID3v1 tag is no longer looked for (a request for 128 bytes, 8 ms of a 128 kbps file). - **Many files at once.** Files are only *noted* during the walk; the ones to read are read after it, - `LIST_AUDIO_INFO_THREADS` (16) at a time, so the round trips overlap. With 5 ms of simulated latency per + `LIST_AUDIO_INFO_THREADS` (32) at a time, so the round trips overlap. With 5 ms of simulated latency per request, 16 workers read 35 times as many files a second as one. - **No request at all for an unchanged file.** The cache is loaded into memory once and checked against the size the directory listing already gave - no stat, no read - and written back once. @@ -49,7 +49,9 @@ bots' read it unchanged: mount the ceiling is the server's, so that is what an operator compares when trying another thread count. Measured live on the NFS library (Neo, #914): three ordinary rebuilds of 5-6 minutes each read 62,657 of the 62,699 audio files (42 unreadable) - 17 minutes in total, where the first version needed two hours in one - at about 55-60 files a - second with 16. + second with 16. From an empty cache again, 16 read about 73 files a second and 64 about 236 - three to four + times as fast, partly on a server cache warmed by the run before. So the default is 32, the range 1 to 128, + and the help says a network drive should try 64. The two live under *List rebuild* on the Settings page. - **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index), keyed by the row's folder and name. Each list prunes only its own rows, and only when its rebuild **publishes**; a rebuild that @@ -64,7 +66,7 @@ bots' read it unchanged: `!rar` rows stay exactly as they were. Stacked on #913, where *Your list* has room since #776 moved the rebuild limits out. -`tests/test_the_list_says_how_long_and_how_good.py` (41) builds every MP3 and FLAC byte by byte - CBR, both ID3 +`tests/test_the_list_says_how_long_and_how_good.py` (42) builds every MP3 and FLAC byte by byte - CBR, both ID3 tags, a tag bigger than the search window, a false sync, all four channel modes, Xing, Info, VBRI, MPEG-2, FLAC mono / 6ch / 96 kHz / a 3 MB picture block, six kinds of broken file - plus the reads each file costs (one, or two with cover art), the cache (no request for an unchanged file, a changed size, prune, a stopped rebuild, per-list diff --git a/settings.conf.sample b/settings.conf.sample index c537af14..c49ba9d5 100644 --- a/settings.conf.sample +++ b/settings.conf.sample @@ -515,14 +515,17 @@ #LIST_AUDIO_INFO_CACHE = ./data/audio_info.db # How many audio files are read at once for their length and quality. On a -# network drive most of the time is waiting, so reading several at once is -# much faster. 1 to 64. +# network drive most of the time is waiting, so more at once is faster: try 64 +# there. The rebuild says the rate it got, to compare. 1 to 128. # # How many audio files are read at once (#914). On a network mount (NFS, SMB) -# the time goes into round trips, which overlap: one at a time measured 9.8 -# files a second on a real NFS library. 1 to 64. +# the time goes into round trips, which overlap. Measured on a real 64,136-file +# NFS library: one at a time 9.8 files a second, 16 about 73, 64 about 236 (the +# last partly on a cache warmed by the run before). 32 by default - harmless on +# a local disk; a network drive may want 64 or more. The rebuild's last line +# says the rate it got, to compare. 1 to 128. # Audio files read at once for length and quality -#LIST_AUDIO_INFO_THREADS = 16 +#LIST_AUDIO_INFO_THREADS = 32 # The longest one rebuild spends reading audio files it has not read before, # since searches wait while it runs. Past it, the list is published and the diff --git a/settings_help.py b/settings_help.py index 493fc684..be2a8a58 100644 --- a/settings_help.py +++ b/settings_help.py @@ -276,7 +276,7 @@ def help_text(name): 'KNOWN_BOTS_FILE': 'Where the bot remembers the other bots it has seen advertising.', 'FETCHED_BOT_LISTS_FILE': "Where the bot remembers which other bots' lists it holds.", 'LIST_SHOW_AUDIO_INFO': 'Add each MP3 and FLAC file\'s length and quality after its size in your list, e.g. 10.3MB 4m31s 320/44.1/JS. Every audio file has to be read once: on a large library, or one on a network drive, that takes several rebuilds, each limited by the reading time set under List rebuild. After that only new files are read.', - 'LIST_AUDIO_INFO_THREADS': 'How many audio files are read at once for their length and quality. On a network drive most of the time is waiting, so reading several at once is much faster. 1 to 64.', + 'LIST_AUDIO_INFO_THREADS': 'How many audio files are read at once for their length and quality. On a network drive most of the time is waiting, so more at once is faster: try 64 there. The rebuild says the rate it got, to compare. 1 to 128.', 'LIST_AUDIO_INFO_MINUTES': 'The longest one rebuild spends reading audio files it has not read before, since searches wait while it runs. Past it, the list is published and the rest are read by the next rebuild. 0 means no limit.', 'LIST_AUDIO_INFO_CACHE': 'Where the length and quality read from your audio files are kept between rebuilds. Safe to delete; the next rebuild reads every file again.', 'LIST_INDEX_FILE': 'The search index over every list you have fetched from other bots. Can be large; safe to delete, it is rebuilt at the next fetch.', diff --git a/tests/test_the_list_says_how_long_and_how_good.py b/tests/test_the_list_says_how_long_and_how_good.py index 2ca2ec9b..668a8add 100644 --- a/tests/test_the_list_says_how_long_and_how_good.py +++ b/tests/test_the_list_says_how_long_and_how_good.py @@ -398,6 +398,15 @@ def test_the_rate_is_measured_on_the_real_clock(self): self.assertGreater(cache.rate(), 0) self.assertEqual(cache.workers, 2) + def test_the_thread_count_is_clamped_to_1_to_128(self): + """What update_list hands read_pending(): the setting, held to a range + a typo cannot turn into ten thousand threads or none.""" + with io.open(os.path.join(REPO_ROOT, "update_list.py"), encoding="utf-8") as handle: + code = handle.read() + self.assertIn('workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 32) or 1)))', code) + import defaults + self.assertEqual(defaults.LIST_AUDIO_INFO_THREADS, 32) + def test_progress_is_reported(self): self.reader = audio_info.read cache = self.pending(3) @@ -445,8 +454,8 @@ def test_on_the_audio_rows_carry_it_and_nothing_else_does(self): self.assertNotIn(" ", rows["Broken.mp3"], "unreadable: size only") # make_tree() ships a few audio files of its own; every one is read. self.assertRegex(said, r"\[LIST-GEN\] Audio info: [1-9]\d* file\(s\) read, 0 unchanged") - self.assertRegex(said, r"Read at [\d,]+ files a second, 16 at a time\.") - self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 16 at a time, for at most 5 minute\(s\)") + self.assertRegex(said, r"Read at [\d,]+ files a second, 32 at a time\.") + self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 32 at a time, for at most 5 minute\(s\)") def test_off_nothing_is_opened_and_the_rows_are_as_before(self): rows, said = self.rows(LIST_SHOW_AUDIO_INFO=False) diff --git a/update_list.py b/update_list.py index f3037fd4..3e3834a8 100644 --- a/update_list.py +++ b/update_list.py @@ -1507,7 +1507,7 @@ def _on_walk_error(err): # network mount the time is round trips, and within LIST_AUDIO_INFO_MINUTES # - the rebuild is pausing every search and request meanwhile. if audio is not None and audio.pending: - workers = max(1, min(64, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 16) or 1))) + workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 32) or 1))) minutes = max(0, int(getattr(config, "LIST_AUDIO_INFO_MINUTES", 5) or 0)) listed = len(all_files_data) + len(video_files_data) print(f"[LIST-GEN] Reading the length and quality of {len(audio.pending):,} new or changed " diff --git a/web/lang/es.json b/web/lang/es.json index 017a9328..42205dcd 100644 --- a/web/lang/es.json +++ b/web/lang/es.json @@ -579,7 +579,7 @@ "settings.field.LIST_AUDIO_INFO_CACHE.help": "Dónde se guardan entre reconstrucciones la duración y la calidad leídas de sus archivos de audio. Se puede borrar sin problema; la siguiente reconstrucción vuelve a leer todos los archivos.", "settings.field.LIST_SHOW_AUDIO_INFO.help": "Añade tras el tamaño de cada archivo MP3 y FLAC de su lista su duración y su calidad, p. ej. 10.3MB 4m31s 320/44.1/JS. Cada archivo de audio se lee una vez: en una biblioteca grande, o en una unidad de red, eso lleva varias reconstrucciones, cada una limitada por el tiempo de lectura fijado en Reconstrucción de la lista. Después solo se leen los archivos nuevos.", "settings.field.LIST_AUDIO_INFO_MINUTES.help": "El tiempo máximo que una reconstrucción dedica a leer archivos de audio que aún no ha leído, ya que las búsquedas esperan mientras tanto. Al llegar a él, la lista se publica y el resto lo lee la siguiente reconstrucción. 0 significa sin límite.", - "settings.field.LIST_AUDIO_INFO_THREADS.help": "Cuántos archivos de audio se leen a la vez para obtener su duración y calidad. En una unidad de red la mayor parte del tiempo es espera, así que leer varios a la vez es mucho más rápido. De 1 a 64.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Cuántos archivos de audio se leen a la vez para obtener su duración y calidad. En una unidad de red la mayor parte del tiempo es espera, así que más a la vez es más rápido: pruebe 64. La reconstrucción indica la velocidad obtenida, para comparar. De 1 a 128.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Dónde se guarda el recuento de cuántas veces se envió cada archivo, para la tabla de los más descargados.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Dónde recuerda el bot qué listas de otros bots tiene.", "settings.field.FETCH_HISTORY_FILE.help": "Dónde se registran las descargas terminadas desde otros bots, para la página Descargas.", diff --git a/web/lang/fr.json b/web/lang/fr.json index b57e8dea..37e4216b 100644 --- a/web/lang/fr.json +++ b/web/lang/fr.json @@ -579,7 +579,7 @@ "settings.field.LIST_AUDIO_INFO_CACHE.help": "L'endroit où la durée et la qualité lues dans vos fichiers audio sont gardées entre deux reconstructions. Peut être supprimé sans risque ; la reconstruction suivante relit tous les fichiers.", "settings.field.LIST_SHOW_AUDIO_INFO.help": "Ajoute après la taille de chaque fichier MP3 et FLAC de votre liste sa durée et sa qualité, par ex. 10.3MB 4m31s 320/44.1/JS. Chaque fichier audio doit être lu une fois : sur une grande bibliothèque, ou sur un lecteur réseau, cela prend plusieurs reconstructions, chacune limitée par la durée de lecture réglée dans Reconstruction de la liste. Ensuite seuls les nouveaux fichiers sont lus.", "settings.field.LIST_AUDIO_INFO_MINUTES.help": "Le temps maximal qu'une reconstruction consacre à lire des fichiers audio pas encore lus, puisque les recherches attendent pendant ce temps. Au-delà, la liste est publiée et la reconstruction suivante lit le reste. 0 signifie sans limite.", - "settings.field.LIST_AUDIO_INFO_THREADS.help": "Combien de fichiers audio sont lus en même temps pour leur durée et leur qualité. Sur un lecteur réseau, l'essentiel du temps est de l'attente : en lire plusieurs à la fois est bien plus rapide. De 1 à 64.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Combien de fichiers audio sont lus en même temps pour leur durée et leur qualité. Sur un lecteur réseau, l'essentiel du temps est de l'attente : en lire davantage à la fois est plus rapide, essayez 64. La reconstruction indique la vitesse obtenue, pour comparer. De 1 à 128.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Où est conservé le compte des envois de chaque fichier, pour le tableau des plus téléchargés.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Où le bot se souvient des listes d'autres bots qu'il détient.", "settings.field.FETCH_HISTORY_FILE.help": "Où sont enregistrés les téléchargements terminés depuis d'autres bots, pour la page Téléchargements.", From 4842704db2fb8b1b0a146463896af1cd10314ef5 Mon Sep 17 00:00:00 2001 From: Ninja-FSE <16465468+Ninja-FSE@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:21:52 +0200 Subject: [PATCH 6/6] Set LIST_AUDIO_INFO_THREADS default to 64, up from 32 (#914) Ninja-FSE's own call, made with the measured numbers from live testing in hand: this repository's proposal (37cbde9) shipped 32 as a cautious middle ground between the old default (16) and the measured 64-thread result, since that result was partly on a warmed cache and not fully isolated from the thread-count change alone. Asked directly, the operator preferred shipping the number actually measured rather than splitting the difference - a plain disk answers 64 concurrent requests as readily as 16, so there is little reason to leave the higher concurrency unused by default. Same value everywhere it appears: defaults.py, update_list.py's getattr() fallback (kept in sync with the real default, matching this codebase's own convention elsewhere), settings.conf.sample (regenerated in an isolated clone, not this checkout - it holds this operator's own live, gitignored settings), settings_help.py, docs/INSTALL.md, docs/UPDATES.md, and the es/fr translations. The two tests that pinned 32 (the clamp-range test's literal defaults.py/update_list.py check, and the two log-line assertions naming the thread count) now pin 64. Co-Authored-By: Claude Sonnet 5 --- defaults.py | 9 +++++---- docs/INSTALL.md | 6 +++--- docs/UPDATES.md | 9 +++++---- settings.conf.sample | 14 ++++++++------ settings_help.py | 2 +- tests/test_the_list_says_how_long_and_how_good.py | 8 ++++---- update_list.py | 2 +- web/lang/es.json | 2 +- web/lang/fr.json | 2 +- 9 files changed, 29 insertions(+), 25 deletions(-) diff --git a/defaults.py b/defaults.py index 1248eba3..172b7194 100644 --- a/defaults.py +++ b/defaults.py @@ -356,10 +356,11 @@ # How many audio files are read at once (#914). On a network mount (NFS, SMB) # the time goes into round trips, which overlap. Measured on a real 64,136-file # NFS library: one at a time 9.8 files a second, 16 about 73, 64 about 236 (the -# last partly on a cache warmed by the run before). 32 by default - harmless on -# a local disk; a network drive may want 64 or more. The rebuild's last line -# says the rate it got, to compare. 1 to 128. -LIST_AUDIO_INFO_THREADS: int = 32 # Audio files read at once for length and quality +# last partly on a cache warmed by the run before). 64 by default - a plain +# disk answers 64 requests as readily as it answers 16; on a very old drive +# or a very small library, lower it. The rebuild's last line says the rate it +# got, to compare. 1 to 128. +LIST_AUDIO_INFO_THREADS: int = 64 # Audio files read at once for length and quality # The most time one rebuild spends reading audio files it has not read before # (#914). A rebuild pauses searches and requests, and the first one with # LIST_AUDIO_INFO on has the whole library to read: past this, the list diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 599729fe..e16043ec 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -252,13 +252,13 @@ Two settings decide how the result is split up: **`LIST_SHOW_AUDIO_INFO`** (off by default) adds each MP3 and FLAC file's length and quality after its size - `::INFO:: 10.3MB 4m31s 320/44.1/JS`, the way other servers' lists show it (`~245` is a VBR average). Every audio -file has to be read once. The files are read several at a time (`LIST_AUDIO_INFO_THREADS`, 32), and each rebuild +file has to be read once. The files are read several at a time (`LIST_AUDIO_INFO_THREADS`, 64), and each rebuild spends at most `LIST_AUDIO_INFO_MINUTES` (5) on it, since searches wait while a rebuild runs: on a large library, or one on a network drive, the first few rebuilds each publish with part of the library read and the rest showing its size alone, until everything has been read once. After that only new files are read, and a rebuild costs what it did without the setting. What was read is kept in `data/audio_info.db`; deleting it is safe - the files are -read again. The rebuild's last line says how fast the files were read; on a network drive, try a higher -`LIST_AUDIO_INFO_THREADS` (64, say) once and compare - past the server's own limit it stops helping. +read again. The rebuild's last line says how fast the files were read; if raising `LIST_AUDIO_INFO_THREADS` +further does not raise that number, you have found the server's own limit rather than the setting's. ### If your users queue with AutoQ diff --git a/docs/UPDATES.md b/docs/UPDATES.md index cabb3814..acfa9f08 100644 --- a/docs/UPDATES.md +++ b/docs/UPDATES.md @@ -38,20 +38,21 @@ bots' read it unchanged: A big ID3 tag or a picture block in the middle costs one more; the ID3v1 tag is no longer looked for (a request for 128 bytes, 8 ms of a 128 kbps file). - **Many files at once.** Files are only *noted* during the walk; the ones to read are read after it, - `LIST_AUDIO_INFO_THREADS` (32) at a time, so the round trips overlap. With 5 ms of simulated latency per + `LIST_AUDIO_INFO_THREADS` (64) at a time, so the round trips overlap. With 5 ms of simulated latency per request, 16 workers read 35 times as many files a second as one. - **No request at all for an unchanged file.** The cache is loaded into memory once and checked against the size the directory listing already gave - no stat, no read - and written back once. - **A time limit.** `LIST_AUDIO_INFO_MINUTES` (5; 0 = none) bounds the reading one rebuild does: past it no read is started, the list publishes with what was read, the rest keep their size alone and the next rebuild reads them. The dashboard shows *Reading length and quality: n of m files*, which also keeps the stall check fed. - - **The rate is said.** The rebuild's last line ends *Read at N files a second, 16 at a time* - on a network + - **The rate is said.** The rebuild's last line ends *Read at N files a second, 64 at a time* - on a network mount the ceiling is the server's, so that is what an operator compares when trying another thread count. Measured live on the NFS library (Neo, #914): three ordinary rebuilds of 5-6 minutes each read 62,657 of the 62,699 audio files (42 unreadable) - 17 minutes in total, where the first version needed two hours in one - at about 55-60 files a second with 16. From an empty cache again, 16 read about 73 files a second and 64 about 236 - three to four - times as fast, partly on a server cache warmed by the run before. So the default is 32, the range 1 to 128, - and the help says a network drive should try 64. + times as fast, partly on a server cache warmed by the run before. The proposal here had shipped 32 as the + cautious middle of that result; asked directly, the operator preferred shipping the number actually measured + - 64 by default, range 1 to 128 - since a plain disk answers 64 requests as readily as 16. The two live under *List rebuild* on the Settings page. - **The cache**, SQLite at `LIST_AUDIO_INFO_CACHE` (`./data/audio_info.db`, beside the list index), keyed by the row's folder and name. Each list prunes only its own rows, and only when its rebuild **publishes**; a rebuild that diff --git a/settings.conf.sample b/settings.conf.sample index c49ba9d5..561ce505 100644 --- a/settings.conf.sample +++ b/settings.conf.sample @@ -515,17 +515,19 @@ #LIST_AUDIO_INFO_CACHE = ./data/audio_info.db # How many audio files are read at once for their length and quality. On a -# network drive most of the time is waiting, so more at once is faster: try 64 -# there. The rebuild says the rate it got, to compare. 1 to 128. +# network drive most of the time is waiting, so this is 64 by default; lower +# it only if a slow or small setup does not benefit. The rebuild says the rate +# it got, to compare. 1 to 128. # # How many audio files are read at once (#914). On a network mount (NFS, SMB) # the time goes into round trips, which overlap. Measured on a real 64,136-file # NFS library: one at a time 9.8 files a second, 16 about 73, 64 about 236 (the -# last partly on a cache warmed by the run before). 32 by default - harmless on -# a local disk; a network drive may want 64 or more. The rebuild's last line -# says the rate it got, to compare. 1 to 128. +# last partly on a cache warmed by the run before). 64 by default - a plain +# disk answers 64 requests as readily as it answers 16; on a very old drive +# or a very small library, lower it. The rebuild's last line says the rate it +# got, to compare. 1 to 128. # Audio files read at once for length and quality -#LIST_AUDIO_INFO_THREADS = 32 +#LIST_AUDIO_INFO_THREADS = 64 # The longest one rebuild spends reading audio files it has not read before, # since searches wait while it runs. Past it, the list is published and the diff --git a/settings_help.py b/settings_help.py index be2a8a58..67e68cb4 100644 --- a/settings_help.py +++ b/settings_help.py @@ -276,7 +276,7 @@ def help_text(name): 'KNOWN_BOTS_FILE': 'Where the bot remembers the other bots it has seen advertising.', 'FETCHED_BOT_LISTS_FILE': "Where the bot remembers which other bots' lists it holds.", 'LIST_SHOW_AUDIO_INFO': 'Add each MP3 and FLAC file\'s length and quality after its size in your list, e.g. 10.3MB 4m31s 320/44.1/JS. Every audio file has to be read once: on a large library, or one on a network drive, that takes several rebuilds, each limited by the reading time set under List rebuild. After that only new files are read.', - 'LIST_AUDIO_INFO_THREADS': 'How many audio files are read at once for their length and quality. On a network drive most of the time is waiting, so more at once is faster: try 64 there. The rebuild says the rate it got, to compare. 1 to 128.', + 'LIST_AUDIO_INFO_THREADS': 'How many audio files are read at once for their length and quality. On a network drive most of the time is waiting, so this is 64 by default; lower it only if a slow or small setup does not benefit. The rebuild says the rate it got, to compare. 1 to 128.', 'LIST_AUDIO_INFO_MINUTES': 'The longest one rebuild spends reading audio files it has not read before, since searches wait while it runs. Past it, the list is published and the rest are read by the next rebuild. 0 means no limit.', 'LIST_AUDIO_INFO_CACHE': 'Where the length and quality read from your audio files are kept between rebuilds. Safe to delete; the next rebuild reads every file again.', 'LIST_INDEX_FILE': 'The search index over every list you have fetched from other bots. Can be large; safe to delete, it is rebuilt at the next fetch.', diff --git a/tests/test_the_list_says_how_long_and_how_good.py b/tests/test_the_list_says_how_long_and_how_good.py index 668a8add..ae259894 100644 --- a/tests/test_the_list_says_how_long_and_how_good.py +++ b/tests/test_the_list_says_how_long_and_how_good.py @@ -403,9 +403,9 @@ def test_the_thread_count_is_clamped_to_1_to_128(self): a typo cannot turn into ten thousand threads or none.""" with io.open(os.path.join(REPO_ROOT, "update_list.py"), encoding="utf-8") as handle: code = handle.read() - self.assertIn('workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 32) or 1)))', code) + self.assertIn('workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 64) or 1)))', code) import defaults - self.assertEqual(defaults.LIST_AUDIO_INFO_THREADS, 32) + self.assertEqual(defaults.LIST_AUDIO_INFO_THREADS, 64) def test_progress_is_reported(self): self.reader = audio_info.read @@ -454,8 +454,8 @@ def test_on_the_audio_rows_carry_it_and_nothing_else_does(self): self.assertNotIn(" ", rows["Broken.mp3"], "unreadable: size only") # make_tree() ships a few audio files of its own; every one is read. self.assertRegex(said, r"\[LIST-GEN\] Audio info: [1-9]\d* file\(s\) read, 0 unchanged") - self.assertRegex(said, r"Read at [\d,]+ files a second, 32 at a time\.") - self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 32 at a time, for at most 5 minute\(s\)") + self.assertRegex(said, r"Read at [\d,]+ files a second, 64 at a time\.") + self.assertRegex(said, r"Reading the length and quality of [1-9]\d* new or changed audio file\(s\), 64 at a time, for at most 5 minute\(s\)") def test_off_nothing_is_opened_and_the_rows_are_as_before(self): rows, said = self.rows(LIST_SHOW_AUDIO_INFO=False) diff --git a/update_list.py b/update_list.py index 3e3834a8..f92b3103 100644 --- a/update_list.py +++ b/update_list.py @@ -1507,7 +1507,7 @@ def _on_walk_error(err): # network mount the time is round trips, and within LIST_AUDIO_INFO_MINUTES # - the rebuild is pausing every search and request meanwhile. if audio is not None and audio.pending: - workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 32) or 1))) + workers = max(1, min(128, int(getattr(config, "LIST_AUDIO_INFO_THREADS", 64) or 1))) minutes = max(0, int(getattr(config, "LIST_AUDIO_INFO_MINUTES", 5) or 0)) listed = len(all_files_data) + len(video_files_data) print(f"[LIST-GEN] Reading the length and quality of {len(audio.pending):,} new or changed " diff --git a/web/lang/es.json b/web/lang/es.json index 42205dcd..bd6ea24b 100644 --- a/web/lang/es.json +++ b/web/lang/es.json @@ -579,7 +579,7 @@ "settings.field.LIST_AUDIO_INFO_CACHE.help": "Dónde se guardan entre reconstrucciones la duración y la calidad leídas de sus archivos de audio. Se puede borrar sin problema; la siguiente reconstrucción vuelve a leer todos los archivos.", "settings.field.LIST_SHOW_AUDIO_INFO.help": "Añade tras el tamaño de cada archivo MP3 y FLAC de su lista su duración y su calidad, p. ej. 10.3MB 4m31s 320/44.1/JS. Cada archivo de audio se lee una vez: en una biblioteca grande, o en una unidad de red, eso lleva varias reconstrucciones, cada una limitada por el tiempo de lectura fijado en Reconstrucción de la lista. Después solo se leen los archivos nuevos.", "settings.field.LIST_AUDIO_INFO_MINUTES.help": "El tiempo máximo que una reconstrucción dedica a leer archivos de audio que aún no ha leído, ya que las búsquedas esperan mientras tanto. Al llegar a él, la lista se publica y el resto lo lee la siguiente reconstrucción. 0 significa sin límite.", - "settings.field.LIST_AUDIO_INFO_THREADS.help": "Cuántos archivos de audio se leen a la vez para obtener su duración y calidad. En una unidad de red la mayor parte del tiempo es espera, así que más a la vez es más rápido: pruebe 64. La reconstrucción indica la velocidad obtenida, para comparar. De 1 a 128.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Cuántos archivos de audio se leen a la vez para obtener su duración y calidad. En una unidad de red la mayor parte del tiempo es espera, así que el valor por defecto es 64; redúzcalo solo si un equipo lento o pequeño no se beneficia. La reconstrucción indica la velocidad obtenida, para comparar. De 1 a 128.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Dónde se guarda el recuento de cuántas veces se envió cada archivo, para la tabla de los más descargados.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Dónde recuerda el bot qué listas de otros bots tiene.", "settings.field.FETCH_HISTORY_FILE.help": "Dónde se registran las descargas terminadas desde otros bots, para la página Descargas.", diff --git a/web/lang/fr.json b/web/lang/fr.json index 37e4216b..8c39483c 100644 --- a/web/lang/fr.json +++ b/web/lang/fr.json @@ -579,7 +579,7 @@ "settings.field.LIST_AUDIO_INFO_CACHE.help": "L'endroit où la durée et la qualité lues dans vos fichiers audio sont gardées entre deux reconstructions. Peut être supprimé sans risque ; la reconstruction suivante relit tous les fichiers.", "settings.field.LIST_SHOW_AUDIO_INFO.help": "Ajoute après la taille de chaque fichier MP3 et FLAC de votre liste sa durée et sa qualité, par ex. 10.3MB 4m31s 320/44.1/JS. Chaque fichier audio doit être lu une fois : sur une grande bibliothèque, ou sur un lecteur réseau, cela prend plusieurs reconstructions, chacune limitée par la durée de lecture réglée dans Reconstruction de la liste. Ensuite seuls les nouveaux fichiers sont lus.", "settings.field.LIST_AUDIO_INFO_MINUTES.help": "Le temps maximal qu'une reconstruction consacre à lire des fichiers audio pas encore lus, puisque les recherches attendent pendant ce temps. Au-delà, la liste est publiée et la reconstruction suivante lit le reste. 0 signifie sans limite.", - "settings.field.LIST_AUDIO_INFO_THREADS.help": "Combien de fichiers audio sont lus en même temps pour leur durée et leur qualité. Sur un lecteur réseau, l'essentiel du temps est de l'attente : en lire davantage à la fois est plus rapide, essayez 64. La reconstruction indique la vitesse obtenue, pour comparer. De 1 à 128.", + "settings.field.LIST_AUDIO_INFO_THREADS.help": "Combien de fichiers audio sont lus en même temps pour leur durée et leur qualité. Sur un lecteur réseau, l'essentiel du temps est de l'attente, d'où la valeur par défaut de 64 ; ne la réduisez que si une configuration lente ou modeste n'en profite pas. La reconstruction indique la vitesse obtenue, pour comparer. De 1 à 128.", "settings.field.DOWNLOAD_COUNTS_FILE.help": "Où est conservé le compte des envois de chaque fichier, pour le tableau des plus téléchargés.", "settings.field.FETCHED_BOT_LISTS_FILE.help": "Où le bot se souvient des listes d'autres bots qu'il détient.", "settings.field.FETCH_HISTORY_FILE.help": "Où sont enregistrés les téléchargements terminés depuis d'autres bots, pour la page Téléchargements.",