From 278ea8b0c68771887c1d8c807de7387b13935869 Mon Sep 17 00:00:00 2001 From: thedancingdeveloper <306930456+thedancingdeveloper@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:27:17 +0000 Subject: [PATCH] sync: nzb-core from rustnzb monorepo c6d859f59ac2b (v0.2.23) --- Cargo.toml | 8 +- src/config.rs | 558 +++++++++- src/config_backup.rs | 1002 ++++++++++++++++++ src/config_registry.rs | 1317 ++++++++++++++++++++++++ src/config_store.rs | 575 +++++++++++ src/damage_ledger.rs | 75 ++ src/db.rs | 2200 ++++++++++++++++++++++++++++++++++++++-- src/error.rs | 3 + src/lib.rs | 10 +- src/models.rs | 131 ++- src/nzb_parser.rs | 2 + src/overrides.rs | 230 +++++ src/par2_match.rs | 145 +++ src/sabnzbd_import.rs | 7 +- src/schedule.rs | 537 ++++++++++ tests/groups_db.rs | 268 +++++ 16 files changed, 6914 insertions(+), 154 deletions(-) create mode 100644 src/config_backup.rs create mode 100644 src/config_registry.rs create mode 100644 src/config_store.rs create mode 100644 src/overrides.rs create mode 100644 src/par2_match.rs create mode 100644 src/schedule.rs create mode 100644 tests/groups_db.rs diff --git a/Cargo.toml b/Cargo.toml index 658b872..6ae5a1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nzb-core" -version = "0.2.19" +version = "0.2.23" edition = "2024" description = "Shared models, config, NZB parser, and SQLite database for NZB clients" license = "MIT" @@ -12,10 +12,12 @@ default = [] groups-db = [] [dependencies] -nzb-nntp = { version = "0.2.26" } +nzb-nntp = { version = "0.2.27" } serde = { version = "1", features = ["derive"] } +utoipa = { version = "5", features = ["axum_extras", "chrono"] } serde_json = "1" toml = "1.1" +toml_edit = "0.25" rusqlite = { version = "0.32", features = ["bundled"] } chrono = { version = "0.4", features = ["serde"] } uuid = { version = "1", features = ["v4", "serde"] } @@ -24,7 +26,9 @@ anyhow = "1" tracing = "0.1" parking_lot = "0.12" quick-xml = "0.41" +regex = "1" unicode-normalization = "0.1" +sha2 = "0.11" [dev-dependencies] tempfile = "3" diff --git a/src/config.rs b/src/config.rs index e36d831..80716bf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,11 +2,19 @@ use std::path::PathBuf; use anyhow::Context; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; /// Top-level application configuration. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] +/// +/// Unknown keys are rejected at parse time (`deny_unknown_fields`), so a typo +/// in `config.toml` fails loudly instead of silently keeping the default. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] pub struct AppConfig { + /// Schema version of the file, stamped on every save. A file without it + /// is version 0 and is migrated on load (see `config_store::migrate`). + #[serde(default)] + pub config_version: u32, pub general: GeneralConfig, pub servers: Vec, pub categories: Vec, @@ -16,23 +24,95 @@ pub struct AppConfig { pub rss_feeds: Vec, #[serde(default)] pub dav: DavConfig, + #[serde(default)] + pub notifications: NotificationConfig, + #[serde(default)] + pub par2: Par2Config, + #[serde(default)] + pub schedule: crate::schedule::ScheduleConfig, } impl Default for AppConfig { fn default() -> Self { Self { + config_version: crate::config_store::CONFIG_VERSION, general: GeneralConfig::default(), servers: Vec::new(), categories: vec![CategoryConfig::default()], otel: OtelConfig::default(), rss_feeds: Vec::new(), dav: DavConfig::default(), + notifications: NotificationConfig::default(), + par2: Par2Config::default(), + schedule: crate::schedule::ScheduleConfig::default(), } } } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] +/// PAR2 handling during download. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] +pub struct Par2Config { + /// Hold PAR2 recovery volumes until the in-flight slice map shows + /// damage, then fetch only as many as the damage needs. Off fetches every + /// volume up front. + pub defer_volumes: bool, + /// Accept a PAR2 set as verified from the in-flight slice map when it + /// covers every file, skipping the disk verify pass. Off always verifies + /// from disk. + pub trust_inflight_verify: bool, +} + +impl Default for Par2Config { + fn default() -> Self { + Self { + defer_volumes: true, + trust_inflight_verify: true, + } + } +} + +/// What admission does when an incoming NZB matches a queued or history job +/// by name or by content digest. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "lowercase")] +pub enum DuplicatePolicy { + /// Admit the NZB without checking for a match. + Allow, + /// Admit the NZB and record the match in the log (default). + #[default] + Warn, + /// Refuse the NZB; the existing job is left as it is. + Skip, +} + +impl std::fmt::Display for DuplicatePolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Allow => "allow", + Self::Warn => "warn", + Self::Skip => "skip", + }) + } +} + +impl std::str::FromStr for DuplicatePolicy { + type Err = String; + + fn from_str(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "allow" => Ok(Self::Allow), + "warn" => Ok(Self::Warn), + "skip" => Ok(Self::Skip), + other => Err(format!( + "duplicate_policy must be allow, warn or skip, not `{other}`" + )), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] pub struct GeneralConfig { /// HTTP API listen address pub listen_addr: String, @@ -41,10 +121,13 @@ pub struct GeneralConfig { /// API key for authentication pub api_key: Option, /// Directory for incomplete downloads + #[schema(value_type = String)] pub incomplete_dir: PathBuf, /// Directory for completed downloads + #[schema(value_type = String)] pub complete_dir: PathBuf, /// Directory for application data (DB, logs) + #[schema(value_type = String)] pub data_dir: PathBuf, /// Download speed limit in bytes/sec (0 = unlimited) pub speed_limit_bps: u64, @@ -52,11 +135,20 @@ pub struct GeneralConfig { pub cache_size: u64, /// Log level pub log_level: String, + /// The level the debug-logging toggle replaced, restored when it is + /// turned off again. Unset while the toggle is off. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub log_level_before_debug: Option, /// Log file path (None = stdout only) + #[schema(value_type = Option)] pub log_file: Option, /// History retention: how many NZBs to keep in history. /// `None` or `Some(0)` both mean keep all; see [`normalize_history_retention`]. pub history_retention: Option, + /// Keep the History row when an *arr removes a job through the SABnzbd + /// API (the files are still deleted as asked). Off restores SABnzbd's + /// behaviour of dropping the row too. + pub keep_history_on_arr_delete: bool, /// Max number of NZBs downloading simultaneously (default 1). /// `0` means unlimited: every queued job may download at once. pub max_active_downloads: usize, @@ -73,6 +165,7 @@ pub struct GeneralConfig { #[serde(default = "default_min_free_space")] pub min_free_space_bytes: u64, /// Directory to watch for new .nzb files to auto-enqueue + #[schema(value_type = Option)] pub watch_dir: Option, /// RSS feed history limit: how many feed items to keep (None = keep all, default 500) #[serde(default = "default_rss_history_limit")] @@ -115,14 +208,31 @@ pub struct GeneralConfig { /// Remove downloaded RSS records after this many days. None keeps them. #[serde(default)] pub rss_downloaded_item_expiry_days: Option, + /// What to do when an incoming NZB matches a queued or history job by + /// name or content digest: `allow`, `warn` (default) or `skip`. + #[serde(default)] + pub duplicate_policy: DuplicatePolicy, + /// Directory that SABnzbd `addlocalfile` may read NZBs from. Unset refuses + /// the mode; a file outside this directory (after resolving symlinks and + /// `..`) is refused too. + #[serde(default)] + #[schema(value_type = Option)] + pub local_import_dir: Option, + /// Keep the SABnzbd `change_complete_action` after it has run once. When + /// false (SABnzbd's default) the action is reset after each run. + #[serde(default)] + pub queue_complete_pers: bool, /// Optional directory containing post-processing scripts. #[serde(default)] + #[schema(value_type = Option)] pub scripts_dir: Option, /// Script run after a successful post-processing job. #[serde(default)] + #[schema(value_type = Option)] pub script_success: Option, /// Script run after a failed post-processing job. #[serde(default)] + #[schema(value_type = Option)] pub script_failure: Option, /// Maximum runtime for a post-processing script. #[serde(default = "default_script_timeout_secs")] @@ -148,6 +258,48 @@ pub struct GeneralConfig { /// oldest are pruned first. 0 keeps all fetched headers. Default: 100000. #[serde(default = "default_group_max_headers")] pub group_max_headers: usize, + /// Seconds to wait on shutdown for open HTTP connections and in-flight + /// downloads to finish before the process exits anyway. 0 waits forever. + /// Default: 30. + #[serde(default = "default_shutdown_timeout_secs")] + pub shutdown_timeout_secs: u64, + /// Reverse proxies whose `X-Forwarded-For` header is believed, as IP + /// addresses or CIDR blocks. Wrong web-login passwords are delayed per + /// client address; behind a proxy every request comes from the proxy, so + /// list it here to count the real clients separately. Empty (the default) + /// ignores `X-Forwarded-For`. + #[serde(default)] + pub trusted_proxies: Vec, + /// How long a confirmed password authorises further sensitive writes, + /// in seconds. Default 300 (five minutes). `0` means every sensitive + /// write must present the password again. + #[serde(default = "default_step_up_ttl_secs")] + pub step_up_ttl_secs: u64, +} + +/// Parse `addr` or `addr/prefix` (IPv4 or IPv6, optionally in brackets) into +/// a network address and prefix length. `None` if it is not one. +pub fn parse_ip_network(entry: &str) -> Option<(std::net::IpAddr, u8)> { + let (addr, prefix) = match entry.split_once('/') { + Some((addr, prefix)) => (addr, Some(prefix)), + None => (entry, None), + }; + let addr = addr.trim_start_matches('[').trim_end_matches(']'); + let ip: std::net::IpAddr = addr.parse().ok()?; + let max = if ip.is_ipv4() { 32 } else { 128 }; + let prefix = match prefix { + Some(p) => p.parse::().ok().filter(|p| *p <= max)?, + None => max, + }; + Some((ip, prefix)) +} + +fn default_shutdown_timeout_secs() -> u64 { + 30 +} + +fn default_step_up_ttl_secs() -> u64 { + 300 } fn default_group_max_headers() -> usize { @@ -196,6 +348,19 @@ pub fn normalize_history_retention(limit: Option) -> Option { limit.filter(|max| *max > 0) } +/// Upper bound for `rss_downloaded_item_expiry_days`: 100 years. +/// +/// The value is subtracted from `Utc::now()` when downloaded RSS items are +/// pruned; a large enough span overflows chrono's `DateTime` range (about +/// ±262 000 years) and panics. A hundred years is already far past any useful +/// retention. +pub const MAX_RSS_EXPIRY_DAYS: u64 = 36_500; + +/// Clamp a configured RSS expiry so the cutoff can always be computed. +pub fn normalize_rss_expiry_days(days: Option) -> Option { + days.map(|days| days.min(MAX_RSS_EXPIRY_DAYS)) +} + fn default_script_timeout_secs() -> u64 { 300 } @@ -209,6 +374,7 @@ impl Default for GeneralConfig { Self { listen_addr: "0.0.0.0".into(), port: 9090, + step_up_ttl_secs: default_step_up_ttl_secs(), api_key: None, incomplete_dir: PathBuf::from("/downloads/incomplete"), complete_dir: PathBuf::from("/downloads/complete"), @@ -216,8 +382,10 @@ impl Default for GeneralConfig { speed_limit_bps: 0, cache_size: 500 * 1024 * 1024, // 500 MB log_level: "info".into(), + log_level_before_debug: None, log_file: None, history_retention: None, // keep all + keep_history_on_arr_delete: true, max_active_downloads: 1, max_post_processing_jobs: default_max_post_processing_jobs(), max_repair_workers: default_max_repair_workers(), @@ -232,7 +400,10 @@ impl Default for GeneralConfig { required_completion_pct: default_required_completion_pct(), article_timeout_secs: default_article_timeout_secs(), auto_sort_remaining_pct: false, + duplicate_policy: DuplicatePolicy::Warn, rss_downloaded_item_expiry_days: None, + local_import_dir: None, + queue_complete_pers: false, scripts_dir: None, script_success: None, script_failure: None, @@ -241,13 +412,15 @@ impl Default for GeneralConfig { fetch_allow_private: false, fetch_allowed_hosts: Vec::new(), group_max_headers: default_group_max_headers(), + shutdown_timeout_secs: default_shutdown_timeout_secs(), + trusted_proxies: Vec::new(), } } } /// OpenTelemetry configuration. All values can be overridden via env vars. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] pub struct OtelConfig { /// Enable both OTEL logs and metrics. Retained as a shared fallback for /// existing configs; prefer the per-signal fields below for new setups. @@ -303,11 +476,12 @@ impl OtelConfig { pub use nzb_nntp::ServerConfig; /// Category configuration for organizing downloads. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct CategoryConfig { /// Category name pub name: String, /// Output directory override (relative to complete_dir) + #[schema(value_type = Option)] pub output_dir: Option, /// Post-processing level: 0=none, 1=repair, 2=unpack, 3=repair+unpack pub post_processing: u8, @@ -356,7 +530,7 @@ impl CategoryConfig { } /// RSS feed configuration for automatic NZB downloading. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct RssFeedConfig { /// Display name for the feed pub name: String, @@ -393,9 +567,45 @@ fn default_true() -> bool { true } -/// DAV pipeline auto-send configuration. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +/// Outbound notifications for completed, failed and disk-full events. +/// +/// Each target is off until its URL or topic is set. Targets are read at +/// startup, so a change takes effect after a restart. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] #[serde(default)] +pub struct NotificationConfig { + /// Generic webhook: each event is POSTed as a JSON document. + pub webhook_url: String, + /// Apprise API notify endpoint, for example `http://apprise:8000/notify` + /// or `http://apprise:8000/notify/`. Empty disables Apprise. + pub apprise_api_url: String, + /// Apprise notification URLs (comma-separated) sent with each message. + /// Leave empty when the endpoint already carries a configuration key. + pub apprise_urls: String, + /// ntfy server base URL. + pub ntfy_server: String, + /// ntfy topic to publish to. Empty disables ntfy. + pub ntfy_topic: String, + /// Optional ntfy access token, sent as a bearer token. + pub ntfy_token: String, +} + +impl Default for NotificationConfig { + fn default() -> Self { + Self { + webhook_url: String::new(), + apprise_api_url: String::new(), + apprise_urls: String::new(), + ntfy_server: "https://ntfy.sh".to_string(), + ntfy_topic: String::new(), + ntfy_token: String::new(), + } + } +} + +/// DAV pipeline auto-send configuration. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] pub struct DavConfig { /// Whether the WebDAV media library is mounted at runtime. pub enabled: bool, @@ -417,16 +627,132 @@ pub struct DavConfig { /// don't support Basic but accept a custom header). #[serde(default)] pub api_key: Option, + /// Percent of each server's `connections` set aside for WebDAV playback + /// and import while the library is enabled (rounded up; the downloader + /// always keeps at least one). The downloader and WebDAV share one + /// allowance per server, so this is what keeps playback responsive + /// during a download. Playback may borrow more while downloads are idle. + pub playback_reserve_pct: u8, +} + +/// Default for [`DavConfig::playback_reserve_pct`]. +pub const DEFAULT_DAV_PLAYBACK_RESERVE_PCT: u8 = 25; + +impl Default for DavConfig { + fn default() -> Self { + Self { + enabled: false, + auto_send_all: false, + category_rules: Vec::new(), + username: None, + password: None, + api_key: None, + playback_reserve_pct: DEFAULT_DAV_PLAYBACK_RESERVE_PCT, + } + } } impl AppConfig { + /// The config path used when none is given: an absolute path under the + /// user's config directory (`$XDG_CONFIG_HOME/rustnzb/config.toml`, else + /// `~/.config/rustnzb/config.toml`). Falls back to `config.toml` in the + /// current directory, made absolute, when neither variable is usable. + pub fn default_path() -> PathBuf { + let xdg = std::env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .filter(|dir| dir.is_absolute()); + let home = std::env::var_os("HOME") + .map(PathBuf::from) + .filter(|dir| dir.is_absolute()) + .map(|dir| dir.join(".config")); + match xdg.or(home) { + Some(dir) => dir.join("rustnzb").join("config.toml"), + None => std::path::absolute("config.toml").unwrap_or_else(|_| "config.toml".into()), + } + } + + /// Read and parse an existing config file. Unknown keys and malformed + /// values fail here, and the error names the file. Does not create the + /// file and does not validate values; see [`AppConfig::validate`]. + pub fn read(path: &std::path::Path) -> anyhow::Result { + let contents = std::fs::read_to_string(path) + .with_context(|| format!("Failed to read config file: {}", path.display()))?; + let mut config: AppConfig = toml::from_str(&contents) + .with_context(|| format!("Failed to parse config file: {}", path.display()))?; + config.assign_missing_server_ids(); + config.general.rss_downloaded_item_expiry_days = + normalize_rss_expiry_days(config.general.rss_downloaded_item_expiry_days); + Ok(config) + } + + /// Check values that the parser cannot: each returned string is one + /// problem, phrased with the setting's path. An empty list means the + /// config can run. Category rules are checked separately, by + /// [`CategoryConfig::validate`], because startup only warns about them. + pub fn validate(&self) -> Vec { + let mut problems = Vec::new(); + if self.general.port == 0 { + problems.push("general.port must be between 1 and 65535".to_string()); + } + if self.general.listen_addr.trim().is_empty() { + problems.push("general.listen_addr must not be empty".to_string()); + } + for (i, entry) in self.general.trusted_proxies.iter().enumerate() { + if parse_ip_network(entry.trim()).is_none() { + problems.push(format!( + "general.trusted_proxies[{i}] ({entry}) must be an IP address or CIDR block" + )); + } + } + let mut seen_ids = std::collections::HashSet::new(); + for (i, server) in self.servers.iter().enumerate() { + let label = format!("servers[{i}] ({})", server.id); + if server.host.trim().is_empty() { + problems.push(format!("{label}: host must not be empty")); + } + if server.port == 0 { + problems.push(format!("{label}: port must be between 1 and 65535")); + } + if server.connections == 0 { + problems.push(format!("{label}: connections must be at least 1")); + } + if !seen_ids.insert(server.id.as_str()) { + problems.push(format!("{label}: duplicate server id")); + } + } + if self.dav.playback_reserve_pct > 100 { + problems.push("dav.playback_reserve_pct must be between 0 and 100".to_string()); + } + for (i, rule) in self.schedule.rules.iter().enumerate() { + if let Some(problem) = crate::schedule::rule_problem(rule) { + problems.push(format!("schedule.rules[{i}] ({}): {problem}", rule.id)); + } + } + for (i, feed) in self.rss_feeds.iter().enumerate() { + if let Some(pattern) = feed.filter_regex.as_deref() + && let Err(e) = regex::Regex::new(pattern) + { + problems.push(format!( + "rss_feeds[{i}] ({}): filter_regex does not compile: {e}", + feed.name + )); + } + } + problems + } + /// Load config from a TOML file, creating default if it doesn't exist. pub fn load(path: &std::path::Path) -> anyhow::Result { if path.exists() { - let contents = std::fs::read_to_string(path) - .with_context(|| format!("Failed to read config file: {}", path.display()))?; - let mut config: AppConfig = toml::from_str(&contents)?; - config.assign_missing_server_ids(); + let mut config = Self::read(path)?; + if config.config_version < crate::config_store::CONFIG_VERSION { + // Saving backs up the old file first. A read-only file still + // loads; the upgrade is then applied on the next save. + crate::config_store::migrate(&mut config); + if let Err(e) = config.save(path) { + tracing::warn!(error = %e, "could not write upgraded config file"); + } + } Ok(config) } else { let config = AppConfig::default(); @@ -460,17 +786,13 @@ impl AppConfig { } } - /// Save config to a TOML file. + /// Save this config as the file layer of `path`. + /// + /// Comments and formatting already in the file are kept, the write is + /// atomic and private, and the previous file is backed up. See + /// [`crate::config_store::write_file_layer`]. pub fn save(&self, path: &std::path::Path) -> anyhow::Result<()> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).with_context(|| { - format!("Failed to create config directory: {}", parent.display()) - })?; - } - let contents = toml::to_string_pretty(self)?; - std::fs::write(path, &contents) - .with_context(|| format!("Failed to write config file: {}", path.display()))?; - Ok(()) + crate::config_store::write_file_layer(self, path) } /// Find a category by name. @@ -497,6 +819,14 @@ impl AppConfig { #[cfg(test)] mod tests { + #[test] + fn par2_trust_inflight_verify_defaults_on_and_can_be_disabled() { + assert!(Par2Config::default().trust_inflight_verify); + let off: Par2Config = toml::from_str("trust_inflight_verify = false").unwrap(); + assert!(!off.trust_inflight_verify); + assert!(off.defer_volumes); + } + use super::*; #[test] @@ -572,6 +902,56 @@ ssl = false assert_eq!(cfg.max_nested_archive_depth, 5); } + #[test] + fn duplicate_policy_defaults_to_warn_and_parses_known_values() { + assert_eq!( + GeneralConfig::default().duplicate_policy, + DuplicatePolicy::Warn + ); + assert_eq!("Skip".parse::(), Ok(DuplicatePolicy::Skip)); + assert_eq!( + " allow ".parse::(), + Ok(DuplicatePolicy::Allow) + ); + assert!("ignore".parse::().is_err()); + + let general: GeneralConfig = toml::from_str("duplicate_policy = \"skip\"").unwrap(); + assert_eq!(general.duplicate_policy, DuplicatePolicy::Skip); + let omitted: GeneralConfig = toml::from_str("").unwrap(); + assert_eq!(omitted.duplicate_policy, DuplicatePolicy::Warn); + } + + #[test] + fn rss_expiry_is_clamped_to_a_computable_cutoff() { + assert_eq!(normalize_rss_expiry_days(None), None); + assert_eq!(normalize_rss_expiry_days(Some(0)), Some(0)); + assert_eq!(normalize_rss_expiry_days(Some(30)), Some(30)); + assert_eq!( + normalize_rss_expiry_days(Some(u64::MAX)), + Some(MAX_RSS_EXPIRY_DAYS) + ); + } + + #[test] + fn read_clamps_an_absurd_rss_expiry() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write( + &path, + format!( + "[general]\nrss_downloaded_item_expiry_days = {}\n", + u64::MAX + ), + ) + .unwrap(); + + let config = AppConfig::read(&path).unwrap(); + assert_eq!( + config.general.rss_downloaded_item_expiry_days, + Some(MAX_RSS_EXPIRY_DAYS) + ); + } + #[test] fn zero_history_retention_normalizes_to_keep_all() { assert_eq!(normalize_history_retention(Some(0)), None); @@ -778,6 +1158,10 @@ ssl = false assert!(cfg.dav.username.is_none()); assert!(cfg.dav.password.is_none()); assert!(cfg.dav.api_key.is_none()); + assert_eq!( + cfg.dav.playback_reserve_pct, + DEFAULT_DAV_PLAYBACK_RESERVE_PCT + ); } #[test] @@ -811,6 +1195,132 @@ ssl = false assert!(dav.username.is_none()); assert!(dav.password.is_none()); assert!(dav.api_key.is_none()); + assert_eq!(dav.playback_reserve_pct, DEFAULT_DAV_PLAYBACK_RESERVE_PCT); + } + + #[test] + fn read_rejects_unknown_keys_and_names_the_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, "[general]\nprot = 9090\n").unwrap(); + + let err = format!("{:#}", AppConfig::read(&path).unwrap_err()); + assert!(err.contains(&path.display().to_string()), "{err}"); + assert!(err.contains("unknown field `prot`"), "{err}"); + } + + #[test] + fn read_does_not_create_a_missing_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("absent.toml"); + + assert!(AppConfig::read(&path).is_err()); + assert!(!path.exists()); + } + + #[test] + fn default_config_and_example_config_validate_and_parse_strictly() { + assert!(AppConfig::default().validate().is_empty()); + + let example = include_str!("../../../apps/rustnzb/config.example.toml"); + let config: AppConfig = toml::from_str(example).unwrap(); + assert!(config.validate().is_empty(), "{:?}", config.validate()); + } + + #[test] + fn validate_checks_each_trusted_proxy_entry() { + let mut cfg = AppConfig::default(); + cfg.general.trusted_proxies = vec![ + "172.18.0.5".into(), + "10.0.0.0/8".into(), + "fd00::/8".into(), + "caddy".into(), + "10.0.0.0/33".into(), + ]; + assert_eq!( + cfg.validate(), + vec![ + "general.trusted_proxies[3] (caddy) must be an IP address or CIDR block", + "general.trusted_proxies[4] (10.0.0.0/33) must be an IP address or CIDR block", + ] + ); + } + + #[test] + fn validate_reports_each_bad_value_with_its_path() { + let mut cfg = AppConfig::default(); + cfg.general.port = 0; + cfg.general.listen_addr = " ".into(); + let mut first = ServerConfig::new("dup", "news.example.com"); + first.connections = 0; + let mut second = ServerConfig::new("dup", ""); + second.port = 0; + cfg.servers = vec![first, second]; + cfg.rss_feeds.push(RssFeedConfig { + name: "Bad".into(), + url: "https://example.com/rss".into(), + poll_interval_secs: 900, + category: None, + filter_regex: Some("(unclosed".into()), + enabled: true, + auto_download: false, + max_age_days: None, + }); + + let problems = cfg.validate(); + for expected in [ + "general.port", + "general.listen_addr", + "servers[0] (dup): connections", + "servers[1] (dup): host", + "servers[1] (dup): port", + "servers[1] (dup): duplicate server id", + "rss_feeds[0] (Bad): filter_regex", + ] { + assert!( + problems.iter().any(|p| p.contains(expected)), + "missing {expected:?} in {problems:?}" + ); + } + } + + #[test] + fn validate_names_a_bad_schedule_rule_by_index_and_id() { + let mut cfg = AppConfig::default(); + cfg.schedule.rules.push(crate::schedule::ScheduleRule { + id: "quiet".into(), + days: vec!["funday".into()], + ..Default::default() + }); + + let problems = cfg.validate(); + assert!( + problems + .iter() + .any(|p| p.starts_with("schedule.rules[0] (quiet):") && p.contains("funday")), + "{problems:?}" + ); + } + + #[test] + fn default_path_is_absolute() { + assert!(AppConfig::default_path().is_absolute()); + } + + #[cfg(unix)] + #[test] + fn save_is_owner_only_even_over_a_looser_existing_file() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + std::fs::write(&path, "").unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + AppConfig::default().save(&path).unwrap(); + + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); } #[test] diff --git a/src/config_backup.rs b/src/config_backup.rs new file mode 100644 index 0000000..4e737d5 --- /dev/null +++ b/src/config_backup.rs @@ -0,0 +1,1002 @@ +//! Export and restore of the configuration as one TOML document. +//! +//! A backup is a `config.toml` under a `[config]` table, with a `[backup]` +//! header describing the file: +//! +//! ```toml +//! [backup] +//! backup_version = 1 +//! created_at = "2026-10-20T09:00:00Z" +//! app_version = "1.6.0" +//! secrets_included = false +//! +//! [config] +//! config_version = 1 +//! +//! [config.general] +//! ... +//! ``` +//! +//! # Secrets +//! +//! The default export replaces every stored secret with [`SECRET_MASK`] +//! (`********`) and sets `secrets_included = false`. That file can be shared, +//! committed or attached to an issue without exposing a credential. An export +//! with real secrets is opt-in, and the document then says so. +//! +//! On restore the mask is a *keep token*: a masked secret keeps the value the +//! target already holds, so restoring a masked backup onto a fresh install +//! leaves those credentials unset instead of writing the literal mask. A +//! secret that is absent means absent — restoring a real backup therefore +//! clears a secret that was not in it, which is what a restore means. A mask +//! with nothing to keep is reported as a warning, never written through. + +use anyhow::Context; +use serde::{Deserialize, Serialize}; + +use crate::config::{ + AppConfig, CategoryConfig, DavConfig, NotificationConfig, OtelConfig, ServerConfig, + normalize_history_retention, normalize_rss_expiry_days, +}; + +/// Schema version written as `backup_version` in every backup file. +pub const BACKUP_VERSION: u32 = 1; + +/// Placeholder the API and a masked backup use in place of a stored secret. +pub const SECRET_MASK: &str = "********"; + +/// The `[backup]` header of a backup document. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct BackupMeta { + /// Schema version of the backup envelope, independent of the config's own + /// `config_version`. A file from a newer build is refused, not guessed at. + /// The serde default is 0, so a document that omits it is refused rather + /// than silently read as the current version. + #[serde(default)] + pub backup_version: u32, + /// When the file was written, RFC 3339 in UTC. + pub created_at: String, + /// The rustnzb build that wrote it. + pub app_version: String, + /// Whether real credentials are in the file or [`SECRET_MASK`] is. + pub secrets_included: bool, +} + +/// One backup file: the header plus the configuration it carries. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackupDocument { + pub backup: BackupMeta, + pub config: AppConfig, +} + +/// What a restore did, reported to the caller. +#[derive(Debug, Clone, Serialize)] +pub struct RestoreSummary { + pub server_count: usize, + pub category_count: usize, + pub rss_feed_count: usize, + /// Stored secrets the masked file kept in place of its masks. + pub secrets_kept: usize, + /// Things the operator should know about the restored file. + pub warnings: Vec, +} + +/// Render `config` as a backup file. With `include_secrets` false every stored +/// secret is replaced by [`SECRET_MASK`]. +pub fn export( + config: &AppConfig, + app_version: &str, + created_at: &str, + include_secrets: bool, +) -> anyhow::Result { + let body = if include_secrets { + config.clone() + } else { + mask_secrets(config) + }; + let mut body = body; + // A backup is a portable document, not this process: drop the remembered + // pre-debug level, which only means something alongside the process that + // toggled it. The saved level itself is kept. + body.general.log_level_before_debug = None; + let document = BackupDocument { + backup: BackupMeta { + backup_version: BACKUP_VERSION, + created_at: created_at.to_string(), + app_version: app_version.to_string(), + secrets_included: include_secrets, + }, + config: body, + }; + toml::to_string_pretty(&document).context("serialize the config backup") +} + +/// Parse a backup file. Refuses a newer envelope, a newer config schema, and +/// any key the running build does not know. +pub fn parse(text: &str) -> Result { + let document: BackupDocument = toml::from_str(text).map_err(|e| parse_error(text, &e))?; + validate_document(&document)?; + Ok(document) +} + +/// Describe a parse failure without echoing the offending source line: a +/// syntax error next to a password in a secrets-included file must not be +/// copied into the response. The message and its position are enough. +fn parse_error(text: &str, error: &toml::de::Error) -> String { + let location = error.span().and_then(|span| { + let prefix = text.get(..span.start)?; + let line = prefix.matches('\n').count() + 1; + let column = prefix + .rsplit('\n') + .next() + .map_or(1, |s| s.chars().count() + 1); + Some(format!(" at line {line}, column {column}")) + }); + match location { + Some(location) => format!("not a valid rustnzb backup{location}: {}", error.message()), + None => format!("not a valid rustnzb backup: {}", error.message()), + } +} + +fn validate_document(document: &BackupDocument) -> Result<(), String> { + let version = document.backup.backup_version; + if version == 0 { + return Err("backup_version is missing; this is not a rustnzb backup".into()); + } + if version > BACKUP_VERSION { + return Err(format!( + "backup_version {version} was written by a newer rustnzb; this build reads up to {BACKUP_VERSION}" + )); + } + if document.config.config_version > crate::config_store::CONFIG_VERSION { + return Err(format!( + "config_version {} is newer than this build supports ({}); upgrade before restoring", + document.config.config_version, + crate::config_store::CONFIG_VERSION + )); + } + Ok(()) +} + +/// Normalize a restored configuration the same way `AppConfig::read` does, so +/// a hand-written backup behaves exactly like a hand-written `config.toml`. +pub fn normalize(config: &mut AppConfig) { + config.assign_missing_server_ids(); + config.general.history_retention = + normalize_history_retention(config.general.history_retention); + config.general.rss_downloaded_item_expiry_days = + normalize_rss_expiry_days(config.general.rss_downloaded_item_expiry_days); + crate::config_store::migrate(config); +} + +/// Every problem that would stop the restored configuration from running. +/// The messages are phrased with the setting's path, as `AppConfig::validate` +/// does, and nothing is written when the list is non-empty. +pub fn validate(config: &AppConfig) -> Vec { + let mut problems = config.validate(); + for (i, category) in config.categories.iter().enumerate() { + if let Err(reason) = category.validate() { + problems.push(format!("categories[{i}] ({}): {reason}", category.name)); + } + } + problems.extend(check_masks(config)); + problems +} + +/// Refuse a document that still carries a mask after the merge. Simple +/// secrets are cleared or kept by then; a mask left in a URL means this +/// install had no stored value to keep, and writing `********` into a feed or +/// collector URL would only move the breakage into the saved configuration. +fn check_masks(config: &AppConfig) -> Vec { + let mut problems = Vec::new(); + { + let mut exact = |field: &str, value: Option<&str>| { + if value == Some(SECRET_MASK) { + problems.push(format!( + "{field} still contains the mask; this install has no stored value to keep" + )); + } + }; + exact("general.api_key", config.general.api_key.as_deref()); + exact("dav.password", config.dav.password.as_deref()); + exact("dav.api_key", config.dav.api_key.as_deref()); + exact( + "notifications.webhook_url", + Some(config.notifications.webhook_url.as_str()), + ); + exact( + "notifications.apprise_api_url", + Some(config.notifications.apprise_api_url.as_str()), + ); + exact( + "notifications.apprise_urls", + Some(config.notifications.apprise_urls.as_str()), + ); + exact( + "notifications.ntfy_token", + Some(config.notifications.ntfy_token.as_str()), + ); + exact( + "notifications.ntfy_topic", + Some(config.notifications.ntfy_topic.as_str()), + ); + for (i, server) in config.servers.iter().enumerate() { + exact( + &format!("servers[{i}].password"), + server.password.as_deref(), + ); + } + } + for (i, server) in config.servers.iter().enumerate() { + if server + .proxy_url + .as_deref() + .is_some_and(|url| url.contains(&format!("{SECRET_MASK}@"))) + { + problems.push(format!( + "servers[{i}].proxy_url still contains the mask; this install has no matching proxy to keep" + )); + } + } + for (i, feed) in config.rss_feeds.iter().enumerate() { + if feed.url.contains(SECRET_MASK) { + problems.push(format!( + "rss_feeds[{i}] ({}): the URL still contains the mask and this install has no matching feed URL to keep", + feed.name + )); + } + } + for (field, value) in [ + ("otel.endpoint", Some(config.otel.endpoint.as_str())), + ("otel.logs_endpoint", config.otel.logs_endpoint.as_deref()), + ( + "otel.metrics_endpoint", + config.otel.metrics_endpoint.as_deref(), + ), + ] { + if value.is_some_and(|url| url.contains(&format!("{SECRET_MASK}@"))) { + problems.push(format!( + "{field} still contains the mask; this install has no matching endpoint to keep" + )); + } + } + problems +} + +/// Replace every stored secret in `config` with [`SECRET_MASK`]. +/// +/// Covers the SABnzbd API key, DAV credentials, server passwords, the +/// `user:pass@` part of a server proxy URL, notification URLs and the ntfy +/// token and topic, RSS feed URLs (whose query string normally carries the +/// indexer key), and the userinfo of OTLP endpoints. +pub fn mask_secrets(config: &AppConfig) -> AppConfig { + let mut config = config.clone(); + if config.general.api_key.is_some() { + config.general.api_key = Some(SECRET_MASK.to_string()); + } + mask_dav(&mut config.dav); + mask_notifications(&mut config.notifications); + mask_otel(&mut config.otel); + for server in &mut config.servers { + if server.password.is_some() { + server.password = Some(SECRET_MASK.to_string()); + } + if let Some(url) = server.proxy_url.as_deref() { + server.proxy_url = Some(mask_proxy_userinfo(url)); + } + } + for feed in &mut config.rss_feeds { + feed.url = mask_url_secrets(&feed.url); + } + config +} + +fn mask_dav(dav: &mut DavConfig) { + if dav.password.is_some() { + dav.password = Some(SECRET_MASK.to_string()); + } + if dav.api_key.is_some() { + dav.api_key = Some(SECRET_MASK.to_string()); + } +} + +fn mask_notifications(notifications: &mut NotificationConfig) { + for secret in [ + &mut notifications.webhook_url, + &mut notifications.apprise_api_url, + &mut notifications.apprise_urls, + &mut notifications.ntfy_token, + &mut notifications.ntfy_topic, + ] { + if !secret.is_empty() { + *secret = SECRET_MASK.to_string(); + } + } +} + +fn mask_otel(otel: &mut OtelConfig) { + otel.endpoint = mask_proxy_userinfo(&otel.endpoint); + if let Some(endpoint) = otel.logs_endpoint.clone() { + otel.logs_endpoint = Some(mask_proxy_userinfo(&endpoint)); + } + if let Some(endpoint) = otel.metrics_endpoint.clone() { + otel.metrics_endpoint = Some(mask_proxy_userinfo(&endpoint)); + } +} + +/// Replace the `user:pass@` part and the query string of a URL with the mask. +/// Scheme, host and path survive, so a file still says which feed it is. +/// A URL without userinfo or a query is returned unchanged. +pub fn mask_url_secrets(url: &str) -> String { + let masked = mask_proxy_userinfo(url); + let (before_fragment, fragment) = match masked.split_once('#') { + Some((head, tail)) => (head, Some(tail)), + None => (masked.as_str(), None), + }; + let mut out = match before_fragment.find('?') { + Some(query_start) => format!("{}?{SECRET_MASK}", &before_fragment[..query_start]), + None => before_fragment.to_string(), + }; + if let Some(fragment) = fragment { + out.push('#'); + out.push_str(fragment); + } + out +} + +/// Replace the `user:pass@` part of a proxy URL with the mask. A URL without +/// userinfo, or without a scheme, is returned unchanged. +pub fn mask_proxy_userinfo(url: &str) -> String { + let Some(scheme_end) = url.find("://").map(|i| i + 3) else { + return url.to_string(); + }; + let rest = &url[scheme_end..]; + let authority_end = rest.find('/').unwrap_or(rest.len()); + let Some(at) = rest[..authority_end].rfind('@') else { + return url.to_string(); + }; + format!("{}{SECRET_MASK}@{}", &url[..scheme_end], &rest[at + 1..]) +} + +/// Whether a restored value is the mask, meaning "keep the stored secret". +fn is_mask(value: Option<&str>) -> bool { + value == Some(SECRET_MASK) +} + +/// Keep the target's stored secrets wherever `incoming` carries a mask. +/// +/// The returned configuration is what will be saved; `summary.secrets_kept` +/// counts the masks that found something to keep, and each mask that found +/// nothing becomes a warning rather than being written through as a literal +/// `********` password. +pub fn merge_secrets(incoming: &mut AppConfig, stored: &AppConfig, summary: &mut RestoreSummary) { + if is_mask(incoming.general.api_key.as_deref()) { + match &stored.general.api_key { + Some(existing) => { + incoming.general.api_key = Some(existing.clone()); + summary.secrets_kept += 1; + } + None => { + incoming.general.api_key = None; + summary + .warnings + .push("general.api_key was masked and this install has no key to keep; it stays unset".into()); + } + } + } + if is_mask(incoming.dav.password.as_deref()) { + match &stored.dav.password { + Some(existing) => { + incoming.dav.password = Some(existing.clone()); + summary.secrets_kept += 1; + } + None => { + incoming.dav.password = None; + summary.warnings.push( + "dav.password was masked and this install has no password to keep; it stays unset" + .into(), + ); + } + } + } + if is_mask(incoming.dav.api_key.as_deref()) { + match &stored.dav.api_key { + Some(existing) => { + incoming.dav.api_key = Some(existing.clone()); + summary.secrets_kept += 1; + } + None => { + incoming.dav.api_key = None; + summary.warnings.push( + "dav.api_key was masked and this install has no key to keep; it stays unset" + .into(), + ); + } + } + } + for (field, incoming_value, stored_value) in [ + ( + "notifications.webhook_url", + &mut incoming.notifications.webhook_url, + &stored.notifications.webhook_url, + ), + ( + "notifications.apprise_api_url", + &mut incoming.notifications.apprise_api_url, + &stored.notifications.apprise_api_url, + ), + ( + "notifications.apprise_urls", + &mut incoming.notifications.apprise_urls, + &stored.notifications.apprise_urls, + ), + ( + "notifications.ntfy_token", + &mut incoming.notifications.ntfy_token, + &stored.notifications.ntfy_token, + ), + ( + "notifications.ntfy_topic", + &mut incoming.notifications.ntfy_topic, + &stored.notifications.ntfy_topic, + ), + ] { + if incoming_value == SECRET_MASK { + if stored_value.is_empty() { + incoming_value.clear(); + summary.warnings.push(format!( + "{field} was masked and this install has no value to keep; it stays empty" + )); + } else { + *incoming_value = stored_value.clone(); + summary.secrets_kept += 1; + } + } + } + if incoming.otel.endpoint.contains(SECRET_MASK) { + if incoming.otel.endpoint == mask_proxy_userinfo(&stored.otel.endpoint) { + incoming.otel.endpoint = stored.otel.endpoint.clone(); + summary.secrets_kept += 1; + } else { + incoming.otel.endpoint = OtelConfig::default().endpoint; + summary.warnings.push( + "otel.endpoint is masked and this install has no matching endpoint to keep; it returns to the default" + .into(), + ); + } + } + for (field, incoming_endpoint, stored_endpoint) in [ + ( + "otel.logs_endpoint", + &mut incoming.otel.logs_endpoint, + stored.otel.logs_endpoint.as_deref(), + ), + ( + "otel.metrics_endpoint", + &mut incoming.otel.metrics_endpoint, + stored.otel.metrics_endpoint.as_deref(), + ), + ] { + let Some(current) = incoming_endpoint.as_deref() else { + continue; + }; + if !current.contains(SECRET_MASK) { + continue; + } + match stored_endpoint { + Some(stored_url) if current == mask_proxy_userinfo(stored_url) => { + *incoming_endpoint = Some(stored_url.to_string()); + summary.secrets_kept += 1; + } + _ => { + *incoming_endpoint = None; + summary + .warnings + .push(format!("{field} is masked and this install has no matching endpoint to keep; it stays unset")); + } + } + } + // A feed whose URL carries a masked key keeps the stored URL when the + // masked form names the same feed. Without one there is nothing to restore + // and an empty URL would only break polling, so the feed is dropped rather + // than saved half-configured; the warning says so. + let mut kept_feeds = Vec::with_capacity(incoming.rss_feeds.len()); + for mut feed in std::mem::take(&mut incoming.rss_feeds) { + if !feed.url.contains(SECRET_MASK) { + kept_feeds.push(feed); + continue; + } + match stored.rss_feeds.iter().find(|stored_feed| { + stored_feed.name == feed.name && mask_url_secrets(&stored_feed.url) == feed.url + }) { + Some(stored_feed) => { + feed.url = stored_feed.url.clone(); + summary.secrets_kept += 1; + kept_feeds.push(feed); + } + None => summary.warnings.push(format!( + "RSS feed `{}` has a masked URL and this install has no matching feed; the feed was dropped, so add it again with its URL", + feed.name + )), + } + } + incoming.rss_feeds = kept_feeds; + for server in &mut incoming.servers { + let Some(existing) = stored.servers.iter().find(|s| s.id == server.id) else { + if is_mask(server.password.as_deref()) { + server.password = None; + summary.warnings.push(format!( + "server `{}` was masked and is not configured here; its password stays unset", + server_label(server) + )); + } + if server + .proxy_url + .as_deref() + .is_some_and(|url| url.contains(&format!("{SECRET_MASK}@"))) + { + server.proxy_url = None; + summary.warnings.push(format!( + "server `{}` has a masked proxy URL and is not configured here; the proxy stays unset", + server_label(server) + )); + } + continue; + }; + if is_mask(server.password.as_deref()) { + match &existing.password { + Some(password) => { + server.password = Some(password.clone()); + summary.secrets_kept += 1; + } + None => { + server.password = None; + summary.warnings.push(format!( + "server `{}` was masked and has no stored password to keep", + server_label(server) + )); + } + } + } + if let Some(url) = server.proxy_url.as_deref() { + if Some(url) + == existing + .proxy_url + .as_deref() + .map(mask_proxy_userinfo) + .as_deref() + { + server.proxy_url = existing.proxy_url.clone(); + summary.secrets_kept += 1; + } else if url.contains(&format!("{SECRET_MASK}@")) { + server.proxy_url = None; + summary.warnings.push(format!( + "server `{}` has a masked proxy URL that does not match this install's; the proxy stays unset", + server_label(server) + )); + } + } + } +} + +fn server_label(server: &ServerConfig) -> String { + if server.name.trim().is_empty() { + server.id.clone() + } else { + format!("{} ({})", server.name, server.id) + } +} + +/// Build the summary for a configuration that is about to be restored or, +/// with `dry_run`, merely previewed. +pub fn summarize(config: &AppConfig, warnings: Vec) -> RestoreSummary { + RestoreSummary { + server_count: config.servers.len(), + category_count: config.categories.len(), + rss_feed_count: config.rss_feeds.len(), + secrets_kept: 0, + warnings, + } +} + +/// Whether a parsed category list is one the queue manager can use: at least +/// one entry, and `AppConfig`'s invariant is that a job always resolves to a +/// category. An empty list is refused instead of being silently repaired, +/// because the next enqueue would otherwise panic. +pub fn check_categories(categories: &[CategoryConfig]) -> Result<(), String> { + if categories.is_empty() { + return Err("categories must contain at least one entry (the default `Default`)".into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const API_KEY: &str = "canary-api-key-111"; + const SERVER_PASSWORD: &str = "canary-server-password-222"; + const DAV_PASSWORD: &str = "canary-dav-password-333"; + const DAV_KEY: &str = "canary-dav-key-444"; + const PROXY_SECRET: &str = "canary-proxy-secret-555"; + const WEBHOOK: &str = "https://hooks.example/canary-webhook-666"; + const NTFY: &str = "canary-ntfy-token-777"; + const NTFY_TOPIC: &str = "canary-ntfy-topic-788"; + const RSS_KEY: &str = "canary-rss-key-888"; + const OTEL_SECRET: &str = "canary-otel-secret-999"; + + fn seeded() -> AppConfig { + let mut config = AppConfig::default(); + config.general.api_key = Some(API_KEY.into()); + config.general.log_level_before_debug = Some("info".into()); + let mut server = ServerConfig::new("srv-1", "news.example.com"); + server.name = "Primary".into(); + server.password = Some(SERVER_PASSWORD.into()); + server.proxy_url = Some(format!( + "socks5://user:{PROXY_SECRET}@proxy.example.net:1080" + )); + config.servers.push(server); + config.categories.push(CategoryConfig { + name: "TV".into(), + output_dir: Some("tv".into()), + ..CategoryConfig::default() + }); + config.dav.password = Some(DAV_PASSWORD.into()); + config.dav.api_key = Some(DAV_KEY.into()); + config.notifications.webhook_url = WEBHOOK.into(); + config.notifications.ntfy_token = NTFY.into(); + config.notifications.ntfy_topic = NTFY_TOPIC.into(); + config.rss_feeds.push(crate::config::RssFeedConfig { + name: "daily".into(), + url: format!("https://indexer.example/rss?t=2000&apikey={RSS_KEY}&i=42"), + poll_interval_secs: 900, + category: None, + filter_regex: None, + enabled: true, + auto_download: false, + max_age_days: None, + }); + config.otel.endpoint = format!("https://collector:{OTEL_SECRET}@otel.example:4317"); + config + } + + fn assert_no_canary(text: &str) { + // Name the field that leaked, never the value: a failing assertion must + // not put a credential in the test log. + for (field, secret) in [ + ("api_key", API_KEY), + ("server_password", SERVER_PASSWORD), + ("dav_password", DAV_PASSWORD), + ("dav_api_key", DAV_KEY), + ("proxy_secret", PROXY_SECRET), + ("webhook_url", WEBHOOK), + ("ntfy_token", NTFY), + ("ntfy_topic", NTFY_TOPIC), + ("rss_key", RSS_KEY), + ("otel_secret", OTEL_SECRET), + ] { + assert!(!text.contains(secret), "a masked backup leaked {field}"); + } + } + + #[test] + fn masked_export_carries_no_secret_and_says_so() { + let text = export(&seeded(), "1.6.0", "2026-10-20T00:00:00Z", false).unwrap(); + assert_no_canary(&text); + assert!(text.contains("secrets_included = false")); + assert!(text.contains(SECRET_MASK)); + + let document = parse(&text).unwrap(); + assert!(!document.backup.secrets_included); + assert_eq!(document.backup.backup_version, BACKUP_VERSION); + assert_eq!(document.backup.app_version, "1.6.0"); + assert_eq!(document.backup.created_at, "2026-10-20T00:00:00Z"); + // Non-secret settings survive. + assert_eq!(document.config.general.port, 9090); + assert_eq!(document.config.servers[0].host, "news.example.com"); + assert_eq!(document.config.categories.len(), 2); + // The process-local pre-debug level is not part of a backup. + assert!(document.config.general.log_level_before_debug.is_none()); + } + + #[test] + fn export_with_secrets_round_trips_every_value() { + let original = seeded(); + let text = export(&original, "1.6.0", "2026-10-20T00:00:00Z", true).unwrap(); + assert!(text.contains("secrets_included = true")); + let restored = parse(&text).unwrap().config; + assert_eq!(restored.general.api_key.as_deref(), Some(API_KEY)); + assert_eq!( + restored.servers[0].password.as_deref(), + Some(SERVER_PASSWORD) + ); + assert_eq!( + restored.servers[0].proxy_url.as_deref(), + original.servers[0].proxy_url.as_deref() + ); + assert_eq!(restored.dav.password.as_deref(), Some(DAV_PASSWORD)); + assert_eq!(restored.notifications.webhook_url, WEBHOOK); + assert_eq!(restored.notifications.ntfy_topic, NTFY_TOPIC); + assert_eq!(restored.rss_feeds[0].url, original.rss_feeds[0].url); + assert_eq!(restored.otel.endpoint, original.otel.endpoint); + } + + #[test] + fn masked_export_masks_feed_keys_and_keeps_the_feed_recognisable() { + let text = export(&seeded(), "1.6.0", "", false).unwrap(); + assert_no_canary(&text); + let document = parse(&text).unwrap(); + let url = &document.config.rss_feeds[0].url; + assert!( + url.starts_with("https://indexer.example/rss?********"), + "{url}" + ); + assert!(!url.contains("t=2000"), "the whole query is masked: {url}"); + assert_eq!( + document.config.otel.endpoint, + "https://********@otel.example:4317" + ); + assert_eq!(document.config.notifications.ntfy_topic, SECRET_MASK); + } + + #[test] + fn a_masked_feed_url_keeps_the_stored_url_only_for_the_same_feed() { + let stored = seeded(); + let mut incoming = parse(&export(&stored, "1.6.0", "", false).unwrap()) + .unwrap() + .config; + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + assert_eq!(incoming.rss_feeds[0].url, stored.rss_feeds[0].url); + assert!(!incoming.rss_feeds[0].url.contains(SECRET_MASK)); + assert_eq!(incoming.otel.endpoint, stored.otel.endpoint); + assert_eq!(incoming.notifications.ntfy_topic, NTFY_TOPIC); + assert!(summary.warnings.is_empty(), "{summary:?}"); + + // A masked feed the target does not know is dropped, not saved with a + // literal mask that would break every poll. + let mut incoming = parse(&export(&stored, "1.6.0", "", false).unwrap()) + .unwrap() + .config; + incoming.rss_feeds[0].name = "other".into(); + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &AppConfig::default(), &mut summary); + assert!(incoming.rss_feeds.is_empty()); + assert!( + summary.warnings.iter().any(|w| w.contains("RSS feed")), + "{summary:?}" + ); + assert!(validate(&incoming).is_empty(), "{:?}", validate(&incoming)); + } + + #[test] + fn a_masked_url_with_no_stored_value_is_refused_rather_than_saved() { + let mut config = AppConfig::default(); + config.rss_feeds.push(crate::config::RssFeedConfig { + name: "daily".into(), + url: "https://indexer.example/rss?********".into(), + poll_interval_secs: 900, + category: None, + filter_regex: None, + enabled: true, + auto_download: false, + max_age_days: None, + }); + config.notifications.ntfy_topic = SECRET_MASK.into(); + let problems = validate(&config); + assert!( + problems.iter().any(|p| p.contains("rss_feeds[0]")), + "{problems:?}" + ); + assert!( + problems + .iter() + .any(|p| p.contains("notifications.ntfy_topic")), + "{problems:?}" + ); + } + + #[test] + fn a_masked_backup_keeps_the_secrets_the_target_already_has() { + let stored = seeded(); + let mut incoming = parse(&export(&stored, "1.6.0", "", false).unwrap()) + .unwrap() + .config; + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + + assert_eq!(incoming.general.api_key.as_deref(), Some(API_KEY)); + assert_eq!( + incoming.servers[0].password.as_deref(), + Some(SERVER_PASSWORD) + ); + assert_eq!( + incoming.servers[0].proxy_url.as_deref(), + stored.servers[0].proxy_url.as_deref() + ); + assert_eq!(incoming.dav.password.as_deref(), Some(DAV_PASSWORD)); + assert_eq!(incoming.notifications.webhook_url, WEBHOOK); + assert_eq!(incoming.notifications.ntfy_token, NTFY); + assert_eq!(incoming.notifications.ntfy_topic, NTFY_TOPIC); + assert_eq!(summary.secrets_kept, 10, "{summary:?}"); + assert!(summary.warnings.is_empty(), "{summary:?}"); + assert!( + !incoming.servers[0] + .proxy_url + .as_deref() + .unwrap() + .contains(SECRET_MASK) + ); + } + + #[test] + fn a_masked_backup_onto_a_fresh_install_warns_and_leaves_secrets_unset() { + let stored = AppConfig::default(); + let mut incoming = parse(&export(&seeded(), "1.6.0", "", false).unwrap()) + .unwrap() + .config; + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + + assert!(incoming.general.api_key.is_none()); + assert!(incoming.servers[0].password.is_none()); + assert!(incoming.servers[0].proxy_url.is_none()); + assert!(incoming.notifications.webhook_url.is_empty()); + assert_eq!(summary.secrets_kept, 0); + // One warning per mask with nothing behind it. + assert_eq!(summary.warnings.len(), 10, "{summary:?}"); + assert!(summary.warnings.iter().all(|w| w.contains("masked"))); + // The literal mask is never written through. + for server in &incoming.servers { + assert_ne!(server.password.as_deref(), Some(SECRET_MASK)); + } + } + + #[test] + fn secrets_absent_from_a_real_backup_are_cleared() { + let stored = seeded(); + let mut incoming = AppConfig::default(); + incoming + .servers + .push(ServerConfig::new("srv-1", "news.example.com")); + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + assert!(incoming.general.api_key.is_none()); + assert!(incoming.servers[0].password.is_none()); + assert_eq!(summary.secrets_kept, 0); + // Nothing was masked, so nothing is reported. + assert!(summary.warnings.is_empty(), "{summary:?}"); + } + + #[test] + fn a_masked_server_that_does_not_exist_here_is_not_given_another_servers_password() { + let stored = seeded(); + let mut incoming = stored.clone(); + incoming.servers[0].id = "other-id".into(); + incoming.servers[0].password = Some(SECRET_MASK.into()); + incoming.servers[0].proxy_url = + Some(format!("socks5://{SECRET_MASK}@proxy.example.net:1080")); + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + assert!(incoming.servers[0].password.is_none()); + assert!(incoming.servers[0].proxy_url.is_none()); + assert_eq!(summary.warnings.len(), 2, "{summary:?}"); + } + + #[test] + fn a_masked_proxy_that_does_not_match_is_cleared_not_written_through() { + let stored = seeded(); + let mut incoming = parse(&export(&stored, "1.6.0", "", false).unwrap()) + .unwrap() + .config; + incoming.servers[0].proxy_url = + Some(format!("socks5://{SECRET_MASK}@elsewhere.example:1080")); + let mut summary = summarize(&incoming, Vec::new()); + merge_secrets(&mut incoming, &stored, &mut summary); + assert!(incoming.servers[0].proxy_url.is_none()); + assert!( + summary.warnings.iter().any(|w| w.contains("proxy")), + "{summary:?}" + ); + } + + #[test] + fn envelope_requires_a_version_and_refuses_unknown_header_keys() { + let text = export(&AppConfig::default(), "1.6.0", "", false).unwrap(); + let without_version = text.replace("backup_version = 1\n", ""); + let error = parse(&without_version).unwrap_err(); + assert!(error.contains("backup_version is missing"), "{error}"); + + let typo = text.replace( + "secrets_included = false", + "secrets_included = false\ntypo = 1", + ); + let error = parse(&typo).unwrap_err(); + assert!(error.contains("not a valid rustnzb backup"), "{error}"); + } + + #[test] + fn a_parse_error_does_not_echo_the_offending_line() { + let text = + format!("[backup]\nbackup_version = 1\n\n[config.general]\napi_key = \"{API_KEY}\n"); + let error = parse(&text).unwrap_err(); + assert!(error.contains("not a valid rustnzb backup"), "{error}"); + assert!( + !error.contains(API_KEY), + "the source line was echoed: {error}" + ); + } + + #[test] + fn parse_refuses_a_newer_envelope_or_config_schema() { + let mut text = export(&AppConfig::default(), "1.6.0", "", false).unwrap(); + text = text.replace("backup_version = 1", "backup_version = 99"); + let error = parse(&text).unwrap_err(); + assert!(error.contains("newer rustnzb"), "{error}"); + + let mut text = export(&AppConfig::default(), "1.6.0", "", false).unwrap(); + text = text.replace("config_version = 1", "config_version = 99"); + let error = parse(&text).unwrap_err(); + assert!(error.contains("config_version"), "{error}"); + + let error = parse("[general]\nport = 9090\n").unwrap_err(); + assert!(error.contains("not a valid rustnzb backup"), "{error}"); + } + + #[test] + fn parse_refuses_an_unknown_key_instead_of_dropping_it() { + let text = export(&AppConfig::default(), "1.6.0", "", false) + .unwrap() + .replace("port = 9090", "port = 9090\ntypo_setting = true"); + let error = parse(&text).unwrap_err(); + assert!(error.contains("typo_setting"), "{error}"); + } + + #[test] + fn validate_reports_bad_values_and_empty_categories() { + let mut config = AppConfig::default(); + config.general.port = 0; + config.servers.push(ServerConfig::new("", "")); + config.categories.clear(); + let problems = validate(&config); + assert!( + problems.iter().any(|p| p.contains("general.port")), + "{problems:?}" + ); + assert!( + problems + .iter() + .any(|p| p.contains("host must not be empty")), + "{problems:?}" + ); + assert!(check_categories(&config.categories).is_err()); + } + + #[test] + fn normalize_derives_missing_server_ids_and_folds_zero_retention() { + let mut config = AppConfig::default(); + config + .servers + .push(ServerConfig::new("", "news.example.com")); + config.general.history_retention = Some(0); + normalize(&mut config); + assert_eq!(config.servers[0].id, "news.example.com-563"); + assert!(config.general.history_retention.is_none()); + assert_eq!(config.config_version, crate::config_store::CONFIG_VERSION); + } + + #[test] + fn proxy_masking_only_touches_userinfo() { + assert_eq!( + mask_proxy_userinfo("socks5://user:pass@proxy.example.net:1080"), + "socks5://********@proxy.example.net:1080" + ); + assert_eq!( + mask_proxy_userinfo("socks5://proxy.example.net:1080"), + "socks5://proxy.example.net:1080" + ); + assert_eq!(mask_proxy_userinfo("not a url"), "not a url"); + } +} diff --git a/src/config_registry.rs b/src/config_registry.rs new file mode 100644 index 0000000..5be94db --- /dev/null +++ b/src/config_registry.rs @@ -0,0 +1,1317 @@ +//! Static settings registry. +//! +//! One entry per leaf of [`crate::config::AppConfig`] that the GUI schema can +//! describe. Servers, categories, RSS feeds and schedule rules stay on their +//! own editors and are not leaves here. See +//! `docs/gui-config-design-2026-10-16.md` §5.3 and +//! `docs/config-schema.md` for which keys the granular endpoints already write. + +use serde::Serialize; + +use crate::config::AppConfig; + +/// How a changed value takes effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Apply { + /// The running process picks the new value up without a restart. + Live, + /// The saved value differs from the value the process started with until + /// the next start. + Restart, +} + +/// Who may change the setting, matching `nzb-web::auth_class::AuthClass`. +/// +/// Defined here so the registry (and `GET /api/config/schema`) does not depend +/// on the web crate. `class_of` in `nzb-web` stays the gate for writes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuthClass { + /// Session, HTTP Basic, or `X-Api-Key`. + Normal, + /// Interactive session plus a recent password confirmation. `X-Api-Key` + /// and the SABnzbd API must not write it. + Sensitive, +} + +/// Value shape the GUI and `validate` use. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Kind { + Bool, + U64 { + min: u64, + max: u64, + }, + Usize { + min: u64, + max: u64, + }, + U16 { + min: u16, + max: u16, + }, + U8 { + min: u8, + max: u8, + }, + F64 { + min: f64, + max: f64, + }, + /// Filesystem path. `optional` is `Option`. + Path { + optional: bool, + }, + String { + optional: bool, + /// `None` means unbounded. + max_len: Option, + }, + /// A secret string. The schema never includes the current value. + Secret { + optional: bool, + }, + Enum { + values: &'static [&'static str], + }, + /// A list of strings (hosts, CIDRs, categories). + StringList, +} + +/// One leaf setting. +#[derive(Debug, Clone, Serialize)] +pub struct Setting { + pub key: &'static str, + pub kind: Kind, + /// JSON value of the serde default, taken from `AppConfig::default()`. + pub default: serde_json::Value, + pub apply: Apply, + pub class: AuthClass, + /// Environment names, primary first, then legacy aliases. + pub env: &'static [&'static str], + pub cli: Option<&'static str>, + pub group: &'static str, + pub label: &'static str, + /// This key is already written by a granular endpoint from WI-2079 + /// (paths, post-processing, network). The schema still lists it so the + /// GUI can render one form; writers must keep using that endpoint. + pub granular: bool, +} + +const U64_MAX: u64 = u64::MAX; + +struct Draft { + key: &'static str, + kind: Kind, + default: serde_json::Value, + apply: Apply, + class: AuthClass, + env: &'static [&'static str], + cli: Option<&'static str>, + group: &'static str, + label: &'static str, + granular: bool, +} + +fn setting(draft: Draft) -> Setting { + Setting { + key: draft.key, + kind: draft.kind, + default: draft.default, + apply: draft.apply, + class: draft.class, + env: draft.env, + cli: draft.cli, + group: draft.group, + label: draft.label, + granular: draft.granular, + } +} + +fn path_default(path: &std::path::Path) -> serde_json::Value { + serde_json::Value::String(path.display().to_string()) +} + +/// Every leaf the schema publishes, in GUI-section order. +pub fn settings() -> Vec { + use Apply::{Live, Restart}; + use AuthClass::{Normal, Sensitive}; + let defaults = AppConfig::default(); + let general = &defaults.general; + let otel = &defaults.otel; + let dav = &defaults.dav; + let note = &defaults.notifications; + let par2 = &defaults.par2; + vec![ + // --- general: already covered by WI-2079 granular endpoints --- + setting(Draft { + key: "general.listen_addr", + kind: Kind::String { + optional: false, + max_len: Some(256), + }, + default: serde_json::json!(general.listen_addr), + apply: Restart, + class: Sensitive, + env: &["RUSTNZB_LISTEN_ADDR"], + cli: Some("--listen-addr"), + group: "api", + label: "Listen address", + granular: true, + }), + setting(Draft { + key: "general.port", + kind: Kind::U16 { min: 1, max: 65535 }, + default: serde_json::json!(general.port), + apply: Restart, + class: Sensitive, + env: &["RUSTNZB_PORT"], + cli: Some("--port"), + group: "api", + label: "HTTP port", + granular: true, + }), + setting(Draft { + key: "general.api_key", + kind: Kind::Secret { optional: true }, + default: serde_json::Value::Null, + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "api", + label: "API key", + granular: true, + }), + setting(Draft { + key: "general.incomplete_dir", + kind: Kind::Path { optional: false }, + default: path_default(&general.incomplete_dir), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "paths", + label: "Incomplete directory", + granular: true, + }), + setting(Draft { + key: "general.complete_dir", + kind: Kind::Path { optional: false }, + default: path_default(&general.complete_dir), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "paths", + label: "Complete directory", + granular: true, + }), + setting(Draft { + key: "general.data_dir", + kind: Kind::Path { optional: false }, + default: path_default(&general.data_dir), + apply: Restart, + class: Sensitive, + env: &["RUSTNZB_DATA_DIR"], + cli: Some("--data-dir"), + group: "paths", + label: "Data directory", + granular: true, + }), + setting(Draft { + key: "general.speed_limit_bps", + kind: Kind::U64 { + min: 0, + max: U64_MAX, + }, + default: serde_json::json!(general.speed_limit_bps), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Speed limit (bytes/sec, 0 = unlimited)", + granular: true, + }), + setting(Draft { + key: "general.log_level", + kind: Kind::Enum { + values: &["error", "warn", "info", "debug", "trace"], + }, + default: serde_json::json!(general.log_level), + apply: Restart, + class: Normal, + env: &["RUSTNZB_LOG_LEVEL", "RUST_LOG"], + cli: Some("--log-level"), + group: "general", + label: "Log level", + granular: true, + }), + setting(Draft { + key: "general.history_retention", + kind: Kind::Usize { + min: 0, + max: U64_MAX, + }, + default: serde_json::json!(0), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "History retention (0 = keep all)", + granular: true, + }), + setting(Draft { + key: "general.keep_history_on_arr_delete", + kind: Kind::Bool, + default: serde_json::json!(general.keep_history_on_arr_delete), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Keep history when an arr client deletes a job", + granular: true, + }), + setting(Draft { + key: "general.max_active_downloads", + kind: Kind::Usize { min: 1, max: 100 }, + default: serde_json::json!(general.max_active_downloads), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Maximum active downloads", + granular: true, + }), + setting(Draft { + key: "general.min_free_space_bytes", + kind: Kind::U64 { + min: 0, + max: U64_MAX, + }, + default: serde_json::json!(general.min_free_space_bytes), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "disk", + label: "Minimum free space (bytes)", + granular: true, + }), + setting(Draft { + key: "general.watch_dir", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "paths", + label: "Watched folder", + granular: true, + }), + setting(Draft { + key: "general.direct_unpack", + kind: Kind::Bool, + default: serde_json::json!(general.direct_unpack), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Direct unpack", + granular: true, + }), + setting(Draft { + key: "general.max_nested_archive_depth", + kind: Kind::U8 { min: 1, max: 20 }, + default: serde_json::json!(general.max_nested_archive_depth), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Maximum nested archive depth", + granular: true, + }), + setting(Draft { + key: "general.scripts_dir", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "post", + label: "Scripts directory", + granular: true, + }), + setting(Draft { + key: "general.script_success", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "post", + label: "Success script", + granular: true, + }), + setting(Draft { + key: "general.script_failure", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "post", + label: "Failure script", + granular: true, + }), + setting(Draft { + key: "general.script_timeout_secs", + kind: Kind::U64 { + min: 1, + max: 86_400, + }, + default: serde_json::json!(general.script_timeout_secs), + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "post", + label: "Script timeout (seconds)", + granular: true, + }), + setting(Draft { + key: "general.script_max_output_bytes", + kind: Kind::Usize { + min: 1024, + max: 16 * 1024 * 1024, + }, + default: serde_json::json!(general.script_max_output_bytes), + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "post", + label: "Script output cap (bytes)", + granular: true, + }), + setting(Draft { + key: "general.fetch_allow_private", + kind: Kind::Bool, + default: serde_json::json!(general.fetch_allow_private), + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "network", + label: "Allow fetches to private addresses", + granular: true, + }), + setting(Draft { + key: "general.fetch_allowed_hosts", + kind: Kind::StringList, + default: serde_json::json!(general.fetch_allowed_hosts), + apply: Live, + class: Sensitive, + env: &[], + cli: None, + group: "network", + label: "Extra hosts server-side fetch may reach", + granular: true, + }), + setting(Draft { + key: "general.group_max_headers", + kind: Kind::Usize { + min: 1, + max: 10_000_000, + }, + default: serde_json::json!(general.group_max_headers), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "network", + label: "Maximum headers fetched per group", + granular: true, + }), + // --- not written by a WI-2079 endpoint --- + setting(Draft { + key: "config_version", + kind: Kind::U64 { min: 0, max: 1_000 }, + default: serde_json::json!(defaults.config_version), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "internal", + label: "Config schema version", + granular: false, + }), + setting(Draft { + key: "general.log_level_before_debug", + kind: Kind::String { + optional: true, + max_len: Some(16), + }, + default: serde_json::Value::Null, + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "internal", + label: "Log level saved by the debug toggle", + granular: false, + }), + setting(Draft { + key: "general.cache_size", + kind: Kind::U64 { + min: 0, + max: U64_MAX, + }, + default: serde_json::json!(general.cache_size), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Article cache size (bytes)", + granular: false, + }), + setting(Draft { + key: "general.log_file", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Restart, + class: Normal, + env: &["RUSTNZB_LOG_FILE"], + cli: Some("--log-file"), + group: "general", + label: "Log file", + granular: false, + }), + setting(Draft { + key: "general.max_post_processing_jobs", + kind: Kind::Usize { min: 1, max: 32 }, + default: serde_json::json!(general.max_post_processing_jobs), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Maximum post-processing jobs", + granular: false, + }), + setting(Draft { + key: "general.max_repair_workers", + kind: Kind::Usize { min: 1, max: 32 }, + default: serde_json::json!(general.max_repair_workers), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Maximum PAR2 repair workers", + granular: false, + }), + setting(Draft { + key: "general.max_extract_workers", + kind: Kind::Usize { min: 1, max: 32 }, + default: serde_json::json!(general.max_extract_workers), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Maximum archive extract workers", + granular: false, + }), + setting(Draft { + key: "general.rss_history_limit", + kind: Kind::Usize { + min: 0, + max: 1_000_000, + }, + default: serde_json::json!(general.rss_history_limit.unwrap_or(0)), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "rss", + label: "RSS history rows kept (0 = default)", + granular: false, + }), + setting(Draft { + key: "general.abort_hopeless", + kind: Kind::Bool, + default: serde_json::json!(general.abort_hopeless), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Abort downloads that cannot complete", + granular: false, + }), + setting(Draft { + key: "general.early_failure_check", + kind: Kind::Bool, + default: serde_json::json!(general.early_failure_check), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Fail early when required articles are missing", + granular: false, + }), + setting(Draft { + key: "general.required_completion_pct", + kind: Kind::F64 { + min: 0.0, + max: 200.0, + }, + default: serde_json::json!(general.required_completion_pct), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Required completion percent", + granular: false, + }), + setting(Draft { + key: "general.article_timeout_secs", + kind: Kind::U64 { min: 1, max: 3_600 }, + default: serde_json::json!(general.article_timeout_secs), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "network", + label: "Article timeout (seconds)", + granular: false, + }), + setting(Draft { + key: "general.auto_sort_remaining_pct", + kind: Kind::Bool, + default: serde_json::json!(general.auto_sort_remaining_pct), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Sort the queue by remaining percent", + granular: false, + }), + setting(Draft { + key: "general.rss_downloaded_item_expiry_days", + kind: Kind::U64 { + min: 0, + max: 36_500, + }, + default: serde_json::json!(0), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "rss", + label: "Forget downloaded RSS items after (days, 0 = never)", + granular: false, + }), + setting(Draft { + key: "general.duplicate_policy", + kind: Kind::Enum { + values: &["allow", "warn", "skip"], + }, + default: serde_json::json!(general.duplicate_policy.to_string()), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Duplicate NZB policy", + granular: false, + }), + setting(Draft { + key: "general.local_import_dir", + kind: Kind::Path { optional: true }, + default: serde_json::Value::Null, + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "paths", + label: "Local NZB import directory", + granular: false, + }), + setting(Draft { + key: "general.queue_complete_pers", + kind: Kind::Bool, + default: serde_json::json!(general.queue_complete_pers), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "general", + label: "Persist queue-complete actions for SABnzbd clients", + granular: false, + }), + setting(Draft { + key: "general.shutdown_timeout_secs", + kind: Kind::U64 { min: 1, max: 3_600 }, + default: serde_json::json!(general.shutdown_timeout_secs), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "api", + label: "Shutdown timeout (seconds)", + granular: false, + }), + setting(Draft { + key: "general.trusted_proxies", + kind: Kind::StringList, + default: serde_json::json!(general.trusted_proxies), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "network", + label: "Trusted reverse proxies", + granular: false, + }), + setting(Draft { + key: "general.step_up_ttl_secs", + kind: Kind::U64 { + min: 0, + max: 86_400, + }, + default: serde_json::json!(general.step_up_ttl_secs), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "api", + label: "Password confirmation lifetime (seconds)", + granular: false, + }), + setting(Draft { + key: "otel.enabled", + kind: Kind::Bool, + default: serde_json::json!(otel.enabled), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "Enable OpenTelemetry", + granular: false, + }), + setting(Draft { + key: "otel.endpoint", + kind: Kind::String { + optional: false, + max_len: Some(512), + }, + default: serde_json::json!(otel.endpoint), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "OTLP endpoint", + granular: false, + }), + setting(Draft { + key: "otel.logs_enabled", + kind: Kind::Bool, + default: serde_json::json!(false), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "Enable OTLP logs (unset = follow enabled)", + granular: false, + }), + setting(Draft { + key: "otel.logs_endpoint", + kind: Kind::String { + optional: true, + max_len: Some(512), + }, + default: serde_json::Value::Null, + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "OTLP logs endpoint", + granular: false, + }), + setting(Draft { + key: "otel.metrics_enabled", + kind: Kind::Bool, + default: serde_json::json!(false), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "Enable OTLP metrics (unset = follow enabled)", + granular: false, + }), + setting(Draft { + key: "otel.metrics_endpoint", + kind: Kind::String { + optional: true, + max_len: Some(512), + }, + default: serde_json::Value::Null, + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "OTLP metrics endpoint", + granular: false, + }), + setting(Draft { + key: "otel.service_name", + kind: Kind::String { + optional: false, + max_len: Some(128), + }, + default: serde_json::json!(otel.service_name), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "telemetry", + label: "Service name", + granular: false, + }), + setting(Draft { + key: "notifications.webhook_url", + kind: Kind::Secret { optional: false }, + default: serde_json::json!(note.webhook_url), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "notifications", + label: "Webhook URL", + granular: false, + }), + setting(Draft { + key: "notifications.apprise_api_url", + kind: Kind::Secret { optional: false }, + default: serde_json::json!(note.apprise_api_url), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "notifications", + label: "Apprise API URL", + granular: false, + }), + setting(Draft { + key: "notifications.apprise_urls", + kind: Kind::Secret { optional: false }, + default: serde_json::json!(""), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "notifications", + label: "Apprise notification URLs", + granular: false, + }), + setting(Draft { + key: "notifications.ntfy_server", + kind: Kind::String { + optional: false, + max_len: Some(512), + }, + default: serde_json::json!(note.ntfy_server), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "notifications", + label: "ntfy server", + granular: false, + }), + setting(Draft { + key: "notifications.ntfy_topic", + kind: Kind::String { + optional: false, + max_len: Some(256), + }, + default: serde_json::json!(note.ntfy_topic), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "notifications", + label: "ntfy topic", + granular: false, + }), + setting(Draft { + key: "notifications.ntfy_token", + kind: Kind::Secret { optional: false }, + default: serde_json::json!(""), + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "notifications", + label: "ntfy access token", + granular: false, + }), + setting(Draft { + key: "dav.enabled", + kind: Kind::Bool, + default: serde_json::json!(dav.enabled), + apply: Restart, + class: Normal, + env: &["RUSTNZB_DAV_ENABLED"], + cli: None, + group: "dav", + label: "Mount the WebDAV library", + granular: false, + }), + setting(Draft { + key: "dav.auto_send_all", + kind: Kind::Bool, + default: serde_json::json!(dav.auto_send_all), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "dav", + label: "Send every completed download to DAV", + granular: false, + }), + setting(Draft { + key: "dav.category_rules", + kind: Kind::StringList, + default: serde_json::json!(dav.category_rules), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "dav", + label: "Categories sent to DAV", + granular: false, + }), + setting(Draft { + key: "dav.username", + kind: Kind::String { + optional: true, + max_len: Some(256), + }, + default: serde_json::Value::Null, + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "dav", + label: "DAV username", + granular: false, + }), + setting(Draft { + key: "dav.password", + kind: Kind::Secret { optional: true }, + default: serde_json::Value::Null, + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "dav", + label: "DAV password", + granular: false, + }), + setting(Draft { + key: "dav.api_key", + kind: Kind::Secret { optional: true }, + default: serde_json::Value::Null, + apply: Restart, + class: Sensitive, + env: &[], + cli: None, + group: "dav", + label: "DAV API key", + granular: false, + }), + setting(Draft { + key: "dav.playback_reserve_pct", + kind: Kind::U8 { min: 0, max: 90 }, + default: serde_json::json!(dav.playback_reserve_pct), + apply: Restart, + class: Normal, + env: &[], + cli: None, + group: "dav", + label: "Playback connection reserve (percent)", + granular: false, + }), + setting(Draft { + key: "par2.defer_volumes", + kind: Kind::Bool, + default: serde_json::json!(par2.defer_volumes), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Defer PAR2 volumes until damage is known", + granular: false, + }), + setting(Draft { + key: "par2.trust_inflight_verify", + kind: Kind::Bool, + default: serde_json::json!(par2.trust_inflight_verify), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "post", + label: "Trust the in-flight PAR2 verify", + granular: false, + }), + setting(Draft { + key: "schedule.enabled", + kind: Kind::Bool, + default: serde_json::json!(defaults.schedule.enabled), + apply: Live, + class: Normal, + env: &[], + cli: None, + group: "schedule", + label: "Enable the schedule", + granular: false, + }), + ] +} + +/// Look up one key. +pub fn get(key: &str) -> Option { + settings().into_iter().find(|setting| setting.key == key) +} + +/// A config with every `Option` set, so TOML keeps the key. +/// +/// `None` is omitted on serialise, which would hide real fields from the +/// leaf walk. +fn filled() -> AppConfig { + let mut config = AppConfig::default(); + let path = |name: &str| std::path::PathBuf::from(name); + config.general.api_key = Some("key".into()); + config.general.log_level_before_debug = Some("info".into()); + config.general.log_file = Some(path("/tmp/rustnzb.log")); + config.general.history_retention = Some(1); + config.general.watch_dir = Some(path("/tmp/watch")); + config.general.rss_history_limit = Some(1); + config.general.rss_downloaded_item_expiry_days = Some(1); + config.general.local_import_dir = Some(path("/tmp/import")); + config.general.scripts_dir = Some(path("/tmp/scripts")); + config.general.script_success = Some(path("/tmp/scripts/ok.sh")); + config.general.script_failure = Some(path("/tmp/scripts/bad.sh")); + config.otel.logs_enabled = Some(false); + config.otel.logs_endpoint = Some("http://127.0.0.1:4317".into()); + config.otel.metrics_enabled = Some(false); + config.otel.metrics_endpoint = Some("http://127.0.0.1:4317".into()); + config.dav.username = Some("user".into()); + config.dav.password = Some("secret".into()); + config.dav.api_key = Some("key".into()); + config +} + +/// `true` when `key` is a dotted path that exists on [`AppConfig`]. +pub fn config_has_leaf(key: &str) -> bool { + let value = toml::Value::try_from(filled()).expect("AppConfig serialises"); + let mut current = &value; + for segment in key.split('.') { + match current { + toml::Value::Table(table) => { + let Some(next) = table.get(segment) else { + return false; + }; + current = next; + } + _ => return false, + } + } + !matches!(current, toml::Value::Table(_)) +} + +/// Keys that name no real field. +pub fn unknown_keys() -> Vec<&'static str> { + settings() + .into_iter() + .filter(|setting| !config_has_leaf(setting.key)) + .map(|setting| setting.key) + .collect() +} + +/// Leaf keys of the default config that the registry does not name. +/// +/// Lists (`servers`, `categories`, `rss_feeds`, `schedule.rules`) are +/// collections with their own editors, so their contents are not leaves. +pub fn uncovered_leaves() -> Vec { + let value = toml::Value::try_from(filled()).expect("AppConfig serialises"); + let known: std::collections::BTreeSet<&str> = + settings().iter().map(|setting| setting.key).collect(); + let mut found = Vec::new(); + walk(&value, "", &mut found); + found.retain(|key| !known.contains(key.as_str())); + found +} + +fn walk(value: &toml::Value, prefix: &str, out: &mut Vec) { + // Lists have their own editors (servers, categories, feeds, rules). + if matches!(value, toml::Value::Array(_)) { + return; + } + let toml::Value::Table(table) = value else { + if !prefix.is_empty() { + out.push(prefix.to_string()); + } + return; + }; + for (name, child) in table { + let key = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}.{name}") + }; + walk(child, &key, out); + } +} + +/// Restart-required flag, for callers that only need the boolean. +pub fn restart_required(setting: &Setting) -> bool { + setting.apply == Apply::Restart +} + +/// Check one JSON value against a setting's [`Kind`]. +/// +/// On success the value is the shape `AppConfig` deserializes: numbers stay +/// numbers, paths and secrets are strings, and a missing optional value is +/// JSON null. The error names the constraint, not the key. +pub fn validate_value(kind: &Kind, value: &serde_json::Value) -> Result { + match kind { + Kind::Bool => value + .as_bool() + .map(serde_json::Value::Bool) + .ok_or_else(|| "expected a boolean".to_string()), + Kind::U64 { min, max } => bounded_u64(value, *min, *max), + Kind::Usize { min, max } => bounded_u64(value, *min, *max), + Kind::U16 { min, max } => bounded_u64(value, u64::from(*min), u64::from(*max)), + Kind::U8 { min, max } => bounded_u64(value, u64::from(*min), u64::from(*max)), + Kind::F64 { min, max } => { + let number = value + .as_f64() + .ok_or_else(|| "expected a number".to_string())?; + if !number.is_finite() || number < *min || number > *max { + return Err(format!("must be between {min} and {max}")); + } + Ok(serde_json::json!(number)) + } + Kind::Path { optional } | Kind::String { optional, .. } | Kind::Secret { optional } => { + text_value(kind, value, *optional) + } + Kind::Enum { values } => { + let text = value + .as_str() + .ok_or_else(|| "expected a string".to_string())?; + if values.contains(&text) { + Ok(serde_json::Value::String(text.to_string())) + } else { + Err(format!("must be one of {}", values.join(", "))) + } + } + Kind::StringList => { + let list = value + .as_array() + .ok_or_else(|| "expected a list of strings".to_string())?; + let mut out = Vec::with_capacity(list.len()); + for (index, item) in list.iter().enumerate() { + let text = item + .as_str() + .ok_or_else(|| format!("item {index} expected a string"))?; + out.push(serde_json::Value::String(text.to_string())); + } + Ok(serde_json::Value::Array(out)) + } + } +} + +fn bounded_u64(value: &serde_json::Value, min: u64, max: u64) -> Result { + let number = value + .as_u64() + .ok_or_else(|| "expected an integer".to_string())?; + if number < min || number > max { + return Err(format!("must be between {min} and {max}")); + } + Ok(serde_json::json!(number)) +} + +fn text_value( + kind: &Kind, + value: &serde_json::Value, + optional: bool, +) -> Result { + if value.is_null() { + return if optional { + Ok(serde_json::Value::Null) + } else { + Err("must not be null".to_string()) + }; + } + let text = value + .as_str() + .ok_or_else(|| "expected a string".to_string())?; + if text.is_empty() { + return if optional { + Ok(serde_json::Value::Null) + } else { + Err("must not be empty".to_string()) + }; + } + if let Kind::String { + max_len: Some(max), .. + } = kind + && text.chars().count() > *max + { + return Err(format!("must be at most {max} characters")); + } + Ok(serde_json::Value::String(text.to_string())) +} + +/// Replace one dotted leaf inside a JSON object. Missing objects along the +/// path are created. Returns the previous value, or null when the leaf was +/// absent. +pub fn set_leaf( + root: &mut serde_json::Value, + dotted: &str, + value: serde_json::Value, +) -> Result { + let mut parts = dotted.split('.').peekable(); + let mut current = root; + while let Some(part) = parts.next() { + if parts.peek().is_none() { + let object = current + .as_object_mut() + .ok_or_else(|| format!("{dotted} is not an object path"))?; + let previous = object + .insert(part.to_string(), value) + .unwrap_or(serde_json::Value::Null); + return Ok(previous); + } + let next = current + .as_object_mut() + .ok_or_else(|| format!("{dotted} is not an object path"))? + .entry(part) + .or_insert_with(|| serde_json::json!({})); + if !next.is_object() { + return Err(format!("{dotted} is not an object path")); + } + current = next; + } + Err("empty key".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_registry_key_is_a_real_config_field() { + let unknown = unknown_keys(); + assert!( + unknown.is_empty(), + "registry keys missing from AppConfig: {unknown:?}" + ); + } + + #[test] + fn every_config_leaf_is_in_the_registry() { + let missing = uncovered_leaves(); + assert!( + missing.is_empty(), + "AppConfig leaves missing from the registry: {missing:?}" + ); + } + + #[test] + fn schema_serialises_and_auth_class_matches_the_sensitive_list() { + let body = serde_json::to_value(settings()).expect("schema json"); + let entries = body.as_array().expect("array"); + assert!(!entries.is_empty()); + let sensitive = [ + "general.api_key", + "dav.password", + "dav.api_key", + "general.fetch_allow_private", + "general.fetch_allowed_hosts", + "general.scripts_dir", + "general.script_success", + "general.script_failure", + "general.script_timeout_secs", + "general.script_max_output_bytes", + "general.listen_addr", + "general.port", + "general.data_dir", + ]; + for key in sensitive { + let entry = entries + .iter() + .find(|entry| entry["key"] == key) + .unwrap_or_else(|| panic!("{key} missing")); + assert_eq!(entry["class"], "sensitive", "{key}"); + } + assert_eq!( + entries + .iter() + .find(|entry| entry["key"] == "general.log_level") + .unwrap()["class"], + "normal" + ); + assert!(entries.iter().any(|entry| entry["granular"] == false)); + assert!(entries.iter().any(|entry| entry["apply"] == "restart")); + } + + #[test] + fn validate_value_enforces_type_and_range() { + let port = Kind::U16 { min: 1, max: 65535 }; + assert!(validate_value(&port, &serde_json::json!(8085)).is_ok()); + assert!(validate_value(&port, &serde_json::json!(0)).is_err()); + assert!(validate_value(&port, &serde_json::json!("nope")).is_err()); + let flag = Kind::Bool; + assert!(validate_value(&flag, &serde_json::json!(true)).is_ok()); + let name = Kind::String { + optional: false, + max_len: Some(4), + }; + assert!(validate_value(&name, &serde_json::json!("abcd")).is_ok()); + assert!(validate_value(&name, &serde_json::json!("abcde")).is_err()); + let secret = Kind::Secret { optional: true }; + assert_eq!( + validate_value(&secret, &serde_json::Value::Null).unwrap(), + serde_json::Value::Null + ); + } +} diff --git a/src/config_store.rs b/src/config_store.rs new file mode 100644 index 0000000..40041fc --- /dev/null +++ b/src/config_store.rs @@ -0,0 +1,575 @@ +//! Crash-safe persistence of the file layer of `config.toml`. +//! +//! A save merges the new configuration into the document already on disk +//! with `toml_edit`, so the user's comments, key order and formatting survive +//! and only changed values are rewritten. The result is written to +//! `config.toml.tmp` (mode 0600), fsynced, renamed over `config.toml`, and the +//! directory is fsynced. Before the rename the previous file is copied into +//! `config-backups/`, and that directory is pruned to the newest +//! [`MAX_BACKUPS`] plus one per day for [`BACKUP_DAYS`] days. + +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::Context; +use chrono::{NaiveDate, Utc}; +use toml_edit::{DocumentMut, Item, Table}; + +use crate::config::AppConfig; + +/// Schema version written as `config_version` in every saved file. +pub const CONFIG_VERSION: u32 = 1; + +/// Newest backups kept regardless of age. +pub const MAX_BACKUPS: usize = 20; + +/// Days for which the newest backup of each day is also kept. +pub const BACKUP_DAYS: i64 = 7; + +const BACKUP_DIR: &str = "config-backups"; +const BACKUP_PREFIX: &str = "config-"; +const BACKUP_TIME_FORMAT: &str = "%Y%m%dT%H%M%S%3fZ"; + +/// Bring a configuration loaded from an older schema up to +/// [`CONFIG_VERSION`]. Version 0 files (no `config_version` key) only need +/// the stamp; there are no renamed keys yet. +pub fn migrate(config: &mut AppConfig) { + if config.config_version < CONFIG_VERSION { + config.config_version = CONFIG_VERSION; + } +} + +/// Resolve the config path given on the command line or in `RUSTNZB_CONFIG`. +/// +/// Call this once at startup, before any path is derived from it. The path is +/// operator input only: no API request can change it, so this is the single +/// point where it is checked. It must not contain `..`. Its directory is +/// created and canonicalised, so the temp file and `config-backups/` are always +/// built under a resolved base directory. +pub fn resolve_operator_config_path(path: &Path) -> anyhow::Result { + if path.components().any(|c| c == Component::ParentDir) { + anyhow::bail!("config path must not contain '..': {}", path.display()); + } + let file_name = path + .file_name() + .with_context(|| format!("config path has no file name: {}", path.display()))?; + let dir = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(dir) + .with_context(|| format!("Failed to create config directory: {}", dir.display()))?; + let dir = dir + .canonicalize() + .with_context(|| format!("Failed to resolve config directory: {}", dir.display()))?; + Ok(dir.join(file_name)) +} + +/// Write `config` to `path` as the file layer. +/// +/// Values equal to the file's current contents are left alone, and the +/// previous file is kept as a backup before it is replaced. +pub fn write_file_layer(config: &AppConfig, path: &Path) -> anyhow::Result<()> { + let dir = path.parent().filter(|p| !p.as_os_str().is_empty()); + let dir = dir.unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(dir) + .with_context(|| format!("Failed to create config directory: {}", dir.display()))?; + + let existing = match fs::read(path) { + Ok(bytes) => Some(bytes), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => { + return Err(e) + .with_context(|| format!("Failed to read config file: {}", path.display())); + } + }; + + let mut stamped = config.clone(); + stamped.config_version = CONFIG_VERSION; + let existing_text = existing + .as_deref() + .map(|bytes| { + std::str::from_utf8(bytes) + .with_context(|| format!("config file is not UTF-8: {}", path.display())) + }) + .transpose()?; + let contents = merged_document(existing_text, &stamped)?; + + if let Some(previous) = existing.as_deref() { + if previous == contents.as_bytes() { + // Nothing to save. Replacing the file would cost a durable write + // and a directory fsync for byte-identical content; on a contended + // disk that is measurable start-up time. Permissions are still + // tightened if an older save (or the user) left them open. + if !needs_permission_tightening(path) { + return Ok(()); + } + } else { + backup(dir, previous)?; + } + } + write_atomic(path, contents.as_bytes()) +} + +/// Whether a save still has to replace `path` for its mode alone. +fn needs_permission_tightening(path: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path) + .map(|m| m.permissions().mode() & 0o777 != 0o600) + .unwrap_or(true) + } + #[cfg(not(unix))] + { + let _ = path; + false + } +} + +/// Render `config` as TOML, starting from the document already on disk so +/// that comments and formatting are kept. +fn merged_document(existing: Option<&str>, config: &AppConfig) -> anyhow::Result { + let fresh = toml::to_string_pretty(config)?; + let Some(existing) = existing else { + return Ok(fresh); + }; + let mut doc: DocumentMut = existing + .parse() + .context("existing config file is not valid TOML; refusing to overwrite it")?; + let new: DocumentMut = fresh + .parse() + .context("serialised config is not valid TOML")?; + merge_table(doc.as_table_mut(), new.as_table()); + Ok(doc.to_string()) +} + +fn merge_table(existing: &mut Table, new: &Table) { + let stale: Vec = existing + .iter() + .map(|(key, _)| key.to_string()) + .filter(|key| !new.contains_key(key)) + .collect(); + for key in stale { + existing.remove(&key); + } + for (key, new_item) in new.iter() { + match existing.get_mut(key) { + Some(item) => merge_item(item, new_item), + None => { + existing.insert(key, new_item.clone()); + } + } + } +} + +fn merge_item(existing: &mut Item, new: &Item) { + match (existing, new) { + (Item::Table(e), Item::Table(n)) => merge_table(e, n), + (Item::ArrayOfTables(e), Item::ArrayOfTables(n)) if e.len() == n.len() => { + for (et, nt) in e.iter_mut().zip(n.iter()) { + merge_table(et, nt); + } + } + (Item::Value(e), Item::Value(n)) => { + // Compare without the surrounding whitespace and comments, so an + // unchanged value keeps the user's formatting around it. + if e.clone().decorated("", "").to_string() != n.clone().decorated("", "").to_string() { + let decor = e.decor().clone(); + *e = n.clone(); + *e.decor_mut() = decor; + } + } + (existing, new) => *existing = new.clone(), + } +} + +/// Write `contents` to a temporary file beside `path`, fsync it, rename it +/// over `path`, and fsync the directory. A crash leaves either the old file or +/// the new one, plus at most a stale `*.tmp` that loading ignores. +fn write_atomic(path: &Path, contents: &[u8]) -> anyhow::Result<()> { + let tmp = tmp_path(path); + { + let mut file = open_private(&tmp, false) + .with_context(|| format!("Failed to create {}", tmp.display()))?; + file.write_all(contents) + .with_context(|| format!("Failed to write {}", tmp.display()))?; + file.sync_all() + .with_context(|| format!("Failed to sync {}", tmp.display()))?; + } + fs::rename(&tmp, path) + .with_context(|| format!("Failed to write config file: {}", path.display()))?; + if let Some(dir) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + sync_dir(dir)?; + } + Ok(()) +} + +/// A temporary name no other write in this process or another one uses, so +/// concurrent saves never share, or rename away, each other's temp file. +fn tmp_path(path: &Path) -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + let unique = NEXT.fetch_add(1, Ordering::Relaxed); + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(format!(".{}-{unique}.tmp", std::process::id())); + path.with_file_name(name) +} + +/// Create a file readable and writable by its owner only. With `create_new` +/// the call fails if the file exists; otherwise an existing file is truncated. +fn open_private(path: &Path, create_new: bool) -> std::io::Result { + let mut options = OpenOptions::new(); + options.write(true); + if create_new { + options.create_new(true); + } else { + options.create(true).truncate(true); + } + #[cfg(unix)] + { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + options.mode(0o600); + let file = options.open(path)?; + // `mode` only applies when the file is created; a leftover temp file + // keeps its old permissions, so set them explicitly. + file.set_permissions(fs::Permissions::from_mode(0o600))?; + Ok(file) + } + #[cfg(not(unix))] + { + options.open(path) + } +} + +fn sync_dir(dir: &Path) -> anyhow::Result<()> { + #[cfg(unix)] + { + File::open(dir) + .and_then(|d| d.sync_all()) + .with_context(|| format!("Failed to sync directory {}", dir.display()))?; + } + #[cfg(not(unix))] + let _ = dir; + Ok(()) +} + +/// Copy the previous file into `config-backups/` and prune old backups. +fn backup(config_dir: &Path, previous: &[u8]) -> anyhow::Result<()> { + let dir = config_dir.join(BACKUP_DIR); + fs::create_dir_all(&dir).with_context(|| format!("Failed to create {}", dir.display()))?; + let stamp = Utc::now().format(BACKUP_TIME_FORMAT).to_string(); + + // Two saves in the same millisecond must not share a name. Each candidate + // is created exclusively, so no other save can take the name in between. + let mut n = 0; + let (target, mut file) = loop { + let name = if n == 0 { + format!("{BACKUP_PREFIX}{stamp}.toml") + } else { + format!("{BACKUP_PREFIX}{stamp}-{n}.toml") + }; + let candidate = dir.join(name); + match open_private(&candidate, true) { + Ok(file) => break (candidate, file), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => n += 1, + Err(e) => { + return Err(e) + .with_context(|| format!("Failed to create backup {}", candidate.display())); + } + } + }; + file.write_all(previous) + .and_then(|()| file.sync_all()) + .with_context(|| format!("Failed to write backup {}", target.display()))?; + prune_backups(&dir, Utc::now().date_naive()) +} + +/// Keep the newest [`MAX_BACKUPS`] backups, and the newest backup of each of +/// the last [`BACKUP_DAYS`] days. Everything else is deleted. +fn prune_backups(dir: &Path, today: NaiveDate) -> anyhow::Result<()> { + let mut names: Vec = fs::read_dir(dir) + .with_context(|| format!("Failed to list {}", dir.display()))? + .filter_map(|entry| entry.ok()?.file_name().into_string().ok()) + .filter(|name| name.starts_with(BACKUP_PREFIX) && name.ends_with(".toml")) + .collect(); + // The timestamp is fixed-width and UTC, so name order is time order. + names.sort_unstable_by(|a, b| b.cmp(a)); + + let oldest_kept_day = today - chrono::Duration::days(BACKUP_DAYS - 1); + let mut kept_days = std::collections::HashSet::new(); + for (index, name) in names.iter().enumerate() { + let day = backup_day(name); + let newest_of_recent_day = day.is_some_and(|d| d >= oldest_kept_day && kept_days.insert(d)); + if index < MAX_BACKUPS || newest_of_recent_day { + continue; + } + if let Err(e) = fs::remove_file(dir.join(name)) { + tracing::warn!(file = %name, error = %e, "could not prune config backup"); + } + } + Ok(()) +} + +/// The UTC day encoded in a backup file name. +fn backup_day(name: &str) -> Option { + let digits = name.strip_prefix(BACKUP_PREFIX)?.get(..8)?; + NaiveDate::parse_from_str(digits, "%Y%m%d").ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn backups(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir.join(BACKUP_DIR)) + .map(|rd| { + rd.filter_map(|e| e.ok()?.file_name().into_string().ok()) + .collect() + }) + .unwrap_or_default(); + names.sort(); + names + } + + #[test] + fn save_stamps_version_and_keeps_comments() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + fs::write( + &path, + "# my hand-written file\n[general]\n# keep this comment\nport = 9090\n", + ) + .unwrap(); + + let mut config = AppConfig::load(&path).unwrap(); + config.general.speed_limit_bps = 1234; + config.save(&path).unwrap(); + + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("# my hand-written file\n"), "{text}"); + assert!(text.contains("# keep this comment"), "{text}"); + let reloaded: AppConfig = toml::from_str(&text).unwrap(); + assert_eq!(reloaded.config_version, CONFIG_VERSION); + assert_eq!(reloaded.general.speed_limit_bps, 1234); + assert_eq!(reloaded.general.port, 9090); + } + + #[test] + fn unchanged_values_keep_their_formatting() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + fs::write( + &path, + "config_version = 1\n[general]\nport = 9090 # the port\nlog_level = \"debug\"\n", + ) + .unwrap(); + + let config = AppConfig::load(&path).unwrap(); + config.save(&path).unwrap(); + + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("port = 9090 # the port"), "{text}"); + } + + #[test] + fn save_refuses_to_overwrite_unparseable_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + fs::write(&path, "this is = = not toml [").unwrap(); + + let err = AppConfig::default().save(&path).unwrap_err(); + assert!( + format!("{err:#}").contains("refusing to overwrite"), + "{err:#}" + ); + assert_eq!(fs::read_to_string(&path).unwrap(), "this is = = not toml ["); + } + + #[cfg(unix)] + #[test] + fn saved_file_and_backups_are_private() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + fs::write(&path, "[general]\nport = 9090\n").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + + let mut config = AppConfig::load(&path).unwrap(); + config.general.port = 9191; + config.save(&path).unwrap(); + + let mode = |p: &Path| fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!(mode(&path), 0o600); + let backup = dir.path().join(BACKUP_DIR).join(&backups(dir.path())[0]); + assert_eq!(mode(&backup), 0o600); + } + + #[test] + fn crash_leftover_tmp_file_is_ignored() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let mut config = AppConfig::default(); + config.general.port = 7000; + config.save(&path).unwrap(); + + // A crash after a temp file was written but before the rename. + let stale = dir.path().join("config.toml.4242-0.tmp"); + fs::write(&stale, "[general]\nport = 1\nbroken = [").unwrap(); + + let loaded = AppConfig::load(&path).unwrap(); + assert_eq!(loaded.general.port, 7000); + + // The next save is unaffected and leaves no temp file of its own. + let mut next = loaded; + next.general.port = 7001; + next.save(&path).unwrap(); + assert_eq!(AppConfig::load(&path).unwrap().general.port, 7001); + let leftovers: Vec<_> = fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok()?.file_name().into_string().ok()) + .filter(|n| n.ends_with(".tmp") && n != "config.toml.4242-0.tmp") + .collect(); + assert!(leftovers.is_empty(), "{leftovers:?}"); + } + + #[test] + fn operator_config_path_is_resolved_and_confined() { + let dir = tempfile::tempdir().unwrap(); + let nested = dir.path().join("cfg"); + + let resolved = resolve_operator_config_path(&nested.join("config.toml")).unwrap(); + assert!(nested.is_dir(), "the config directory is created"); + assert_eq!(resolved, nested.canonicalize().unwrap().join("config.toml")); + + let escape = nested.join("..").join("config.toml"); + assert!(resolve_operator_config_path(&escape).is_err()); + assert!(resolve_operator_config_path(Path::new("../config.toml")).is_err()); + assert!(resolve_operator_config_path(Path::new("/")).is_err()); + } + + #[cfg(unix)] + #[test] + fn unchanged_save_does_not_replace_the_file() { + use std::os::unix::fs::MetadataExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let config = AppConfig::default(); + config.save(&path).unwrap(); + let before = fs::metadata(&path).unwrap().ino(); + + config.save(&path).unwrap(); + let after = fs::metadata(&path).unwrap().ino(); + assert_eq!(before, after, "an unchanged save must not rewrite the file"); + + let leftovers: Vec<_> = fs::read_dir(&path) + .map(|entries| { + entries + .filter_map(|e| e.ok()?.file_name().into_string().ok()) + .collect() + }) + .unwrap_or_default(); + assert!(leftovers.is_empty(), "temp file left behind: {leftovers:?}"); + } + + #[cfg(unix)] + #[test] + fn unchanged_save_still_tightens_permissions() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let config = AppConfig::default(); + config.save(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + + config.save(&path).unwrap(); + let mode = fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "a save must not leave the config readable"); + } + + #[test] + fn unchanged_save_writes_no_backup() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let config = AppConfig::default(); + config.save(&path).unwrap(); + config.save(&path).unwrap(); + assert!(backups(dir.path()).is_empty()); + } + + #[test] + fn every_change_keeps_the_previous_file_as_backup() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.toml"); + let mut config = AppConfig::default(); + config.save(&path).unwrap(); + config.general.port = 8001; + config.save(&path).unwrap(); + + let names = backups(dir.path()); + assert_eq!(names.len(), 1); + let kept = fs::read_to_string(dir.path().join(BACKUP_DIR).join(&names[0])).unwrap(); + assert!(kept.contains("port = 9090"), "{kept}"); + } + + #[test] + fn rotation_keeps_newest_and_one_per_recent_day() { + let dir = tempfile::tempdir().unwrap(); + let backup_dir = dir.path().join(BACKUP_DIR); + fs::create_dir_all(&backup_dir).unwrap(); + let today = NaiveDate::from_ymd_opt(2026, 10, 16).unwrap(); + + // 30 backups an hour apart across the last day, and 3 more on each of + // the 10 days before that. + let mut names = Vec::new(); + for h in 0..30u32 { + let t = format!("20261016T{:02}{:02}00000Z", h / 60 % 24, h % 60); + names.push(format!("{BACKUP_PREFIX}{t}.toml")); + } + for day in 1..=10u32 { + for k in 0..3u32 { + names.push(format!( + "{BACKUP_PREFIX}202610{:02}T10000{k}000Z.toml", + 16 - day + )); + } + } + for name in &names { + fs::write(backup_dir.join(name), "x").unwrap(); + } + + prune_backups(&backup_dir, today).unwrap(); + + let mut left: Vec = fs::read_dir(&backup_dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + + // The 20 newest are today's backups. The window holds Oct 10..=16, so + // the newest backup of each older day in it (Oct 10..=15) also stays. + let mut newest_first = names.clone(); + newest_first.sort_by(|a, b| b.cmp(a)); + for kept in &newest_first[..MAX_BACKUPS] { + assert!(left.contains(kept), "newest backup {kept} was pruned"); + } + assert_eq!(left.len(), MAX_BACKUPS + 6, "{left:?}"); + assert!(left.contains(&format!("{BACKUP_PREFIX}20261010T100002000Z.toml"))); + // Oct 9 and older fall outside the window and are gone. + assert!(!left.iter().any(|n| n.starts_with("config-20261009"))); + } + + #[test] + fn migrate_stamps_version_zero_files() { + let mut config = AppConfig { + config_version: 0, + ..AppConfig::default() + }; + migrate(&mut config); + assert_eq!(config.config_version, CONFIG_VERSION); + } +} diff --git a/src/damage_ledger.rs b/src/damage_ledger.rs index 01b0de2..11a0af5 100644 --- a/src/damage_ledger.rs +++ b/src/damage_ledger.rs @@ -319,6 +319,24 @@ impl Database { Ok(confirm) } + /// Record many fetch outcomes in one transaction, so a batch costs one + /// commit rather than one per article. Each note carries the time its + /// outcome was observed. Returns each outcome's [`ConfirmNeeded`], in + /// order. Outcomes for the same article apply in the order given. + pub fn ledger_note_batch( + &self, + notes: &[(ArticleKey, String, Outcome, DateTime)], + enabled_servers: &[String], + ) -> Result, NzbError> { + let tx = self.conn.unchecked_transaction()?; + let mut confirms = Vec::with_capacity(notes.len()); + for (key, message_id, outcome, at) in notes { + confirms.push(self.ledger_note(key, message_id, outcome, enabled_servers, *at)?); + } + tx.commit()?; + Ok(confirms) + } + /// Guidance before fetching an article: skip a live hole, re-probe an /// expired one, otherwise fetch normally. pub fn ledger_consult( @@ -395,6 +413,36 @@ impl Database { Ok(out) } + /// Drop ledger rows nothing can use any more, and return how many went. + /// + /// In this database a `scope_id` is a job id. A job in the queue keeps + /// every row. Otherwise rows go when the job is in neither the queue nor + /// history (removed, or aged out), and confirmed holes past `expires_at` + /// go even while the job is still in history, since a re-probe would + /// ignore them anyway. A retry runs under a new job id, so it never + /// depends on the rows of the history entry it came from. + pub fn ledger_prune(&self, now: DateTime) -> Result { + Ok(self.conn.execute( + "DELETE FROM damage_ledger + WHERE scope_id NOT IN (SELECT id FROM queue) + AND (scope_id NOT IN (SELECT id FROM history) + OR (state = 'confirmed_missing' + AND expires_at IS NOT NULL + AND julianday(expires_at) <= julianday(?1)))", + params![now.to_rfc3339()], + )?) + } + + /// Drop the rows of jobs that are in neither the queue nor history. + pub(crate) fn ledger_drop_orphans(&self) -> Result { + Ok(self.conn.execute( + "DELETE FROM damage_ledger + WHERE scope_id NOT IN (SELECT id FROM queue) + AND scope_id NOT IN (SELECT id FROM history)", + [], + )?) + } + fn ledger_get(&self, key: &ArticleKey) -> Result, NzbError> { let result = self.conn.query_row( "SELECT message_id, state, evidence, first_refused_at, @@ -724,4 +772,31 @@ mod tests { assert_eq!(db.ledger_file_gone_ratio("job-2", 0).unwrap(), 0.5); } + + #[test] + fn note_batch_applies_in_order_with_each_notes_time() { + let db = Database::open_memory().unwrap(); + let srv = servers(&["a", "b"]); + let key = ArticleKey::new("job-3", 0, 1); + let refused = |server: &str| Outcome::Refused { + server: server.into(), + }; + // Two fast refusals, then a confirming one past the guard: one batch. + let notes = vec![ + (key.clone(), "".to_string(), refused("a"), t(0)), + (key.clone(), "".to_string(), refused("b"), t(0)), + (key.clone(), "".to_string(), refused("a"), t(2)), + ]; + let confirms = db.ledger_note_batch(¬es, &srv).unwrap(); + assert_eq!( + confirms, + vec![ + ConfirmNeeded(true), + ConfirmNeeded(true), + ConfirmNeeded(false) + ] + ); + let records = db.ledger_load_file("job-3", 0).unwrap(); + assert_eq!(records[0].state, LedgerState::ConfirmedMissing); + } } diff --git a/src/db.rs b/src/db.rs index baa21f6..ae841e0 100644 --- a/src/db.rs +++ b/src/db.rs @@ -2,6 +2,7 @@ use std::path::Path; use chrono::Utc; use rusqlite::{Connection, OptionalExtension, params}; +use sha2::{Digest, Sha256}; use tracing::info; use crate::error::NzbError; @@ -10,6 +11,16 @@ use crate::models::*; #[allow(dead_code)] const SCHEMA_VERSION: u32 = 1; +/// The content digest that recognises a repeated NZB: `sha256:` followed by +/// the lowercase hex SHA-256 of the payload. +pub fn nzb_digest(payload: &[u8]) -> String { + let hex: String = Sha256::digest(payload) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + format!("sha256:{hex}") +} + /// Database handle for queue and history persistence. pub struct Database { pub(crate) conn: Connection, @@ -39,24 +50,45 @@ impl Database { } fn migrate(&self) -> Result<(), NzbError> { - self.conn.execute_batch( + // Apply every pending migration in one transaction. + // + // Each migration used to commit separately and stamp `schema_version` + // on its own. That cost one durable commit per migration — 31 fsyncs + // to create a fresh database, measured — and on a contended disk (the + // CI container shares a dind volume with concurrent builds) the serial + // commits dominated start-up: at 600 ms per fsync the backend took over + // 15 s to open its database, never reached the HTTP bind, and global + // setup timed out without a single log line (the migration notices are + // INFO, the e2e config logs at WARN). + // + // Batching leaves about ten fsyncs (SQLite's own commit and WAL + // bookkeeping), and the same 600 ms shim now reaches health in 4.3 s + // rather than failing, or 10.7 s at 1.5 s per fsync. + // + // Each migration still logs as it runs. The version row is written + // once, after the last one, and the batch rolls back together: an + // interrupted upgrade leaves the database at its previous version + // rather than half-migrated. + let tx = self.conn.unchecked_transaction()?; + // Creating the bookkeeping table is part of the same batch: as its own + // statement it would be a second durable commit before the first + // migration even ran. + tx.execute_batch( "CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );", )?; - - let version: u32 = self - .conn + let version: u32 = tx .query_row( "SELECT COALESCE(MAX(version), 0) FROM schema_version", [], |row| row.get(0), ) .unwrap_or(0); + let mut applied = version; if version < 1 { info!("Applying database migration v1"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " -- Active download queue @@ -111,12 +143,11 @@ impl Database { ); ", )?; - commit_migration(tx, 1)?; + applied = 1; } if version < 2 { info!("Applying database migration v2"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " -- Add NZB data storage and server stats to history @@ -130,24 +161,22 @@ impl Database { ALTER TABLE queue ADD COLUMN nzb_raw BLOB; ", )?; - commit_migration(tx, 2)?; + applied = 2; } if version < 3 { info!("Applying database migration v3"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " -- Per-job log storage for history ALTER TABLE history ADD COLUMN job_logs TEXT DEFAULT '[]'; ", )?; - commit_migration(tx, 3)?; + applied = 3; } if version < 4 { info!("Applying database migration v4"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " -- RSS feed items (persistent feed cache) @@ -179,12 +208,11 @@ impl Database { ); ", )?; - commit_migration(tx, 4)?; + applied = 4; } if version < 5 { info!("Applying database migration v5: settings table"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " CREATE TABLE IF NOT EXISTS settings ( @@ -193,12 +221,12 @@ impl Database { ); ", )?; - commit_migration(tx, 5)?; + applied = 5; } // Ensure settings table exists for databases that jumped to v5 // with groups tables but without settings (pre-extraction rustnzbd) - self.conn.execute_batch( + tx.execute_batch( "CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -208,7 +236,6 @@ impl Database { #[cfg(feature = "groups-db")] if version < 6 { info!("Applying database migration v6: newsgroup browsing"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " CREATE TABLE IF NOT EXISTS groups ( @@ -256,12 +283,11 @@ impl Database { END; ", )?; - commit_migration(tx, 6)?; + applied = 6; } if version < 7 { info!("Applying database migration v7: persistent download statistics"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " CREATE TABLE IF NOT EXISTS download_statistics ( @@ -294,29 +320,27 @@ impl Database { FROM history; ", )?; - commit_migration(tx, 7)?; + applied = 7; } if version < 8 { info!("Applying database migration v8: active download duration"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " ALTER TABLE history ADD COLUMN download_time_secs REAL; ", )?; - commit_migration(tx, 8)?; + applied = 8; } if version < 9 { info!("Applying database migration v9: per-article retry outcomes"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " ALTER TABLE history ADD COLUMN retry_data BLOB; ", )?; - commit_migration(tx, 9)?; + applied = 9; } if version < 10 { @@ -325,7 +349,6 @@ impl Database { // not damage evidence, so there is no backfill: it carries no // failure reason, per-server evidence, TTL, or server fingerprint // to reconstruct a confirmed-missing record from. See WI-143. - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " CREATE TABLE IF NOT EXISTS damage_ledger ( @@ -345,12 +368,11 @@ impl Database { ON damage_ledger (scope_id, file_index); ", )?; - commit_migration(tx, 10)?; + applied = 10; } if version < 11 { info!("Applying database migration v11: idempotent queue admissions"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " CREATE TABLE queue_admissions ( @@ -364,12 +386,11 @@ impl Database { ON queue_admissions(job_id); ", )?; - commit_migration(tx, 11)?; + applied = 11; } if version < 12 { info!("Applying database migration v12: typed terminal failure codes"); - let tx = self.conn.unchecked_transaction()?; tx.execute_batch( " ALTER TABLE history ADD COLUMN failure_code TEXT; @@ -378,7 +399,7 @@ impl Database { WHERE LOWER(status) = 'failed' AND failure_code IS NULL; ", )?; - commit_migration(tx, 12)?; + applied = 12; } if version < 13 { @@ -390,11 +411,10 @@ impl Database { [], |row| row.get(0), )?; - let tx = self.conn.unchecked_transaction()?; if has_queue > 0 { tx.execute_batch("ALTER TABLE queue ADD COLUMN pp_override INTEGER;")?; } - commit_migration(tx, 13)?; + applied = 13; } if version < 14 { @@ -405,11 +425,10 @@ impl Database { [], |row| row.get(0), )?; - let tx = self.conn.unchecked_transaction()?; if has_history > 0 { tx.execute_batch("ALTER TABLE history ADD COLUMN post_processing INTEGER;")?; } - commit_migration(tx, 14)?; + applied = 14; } if version < 15 { @@ -425,7 +444,6 @@ impl Database { )?; table_exists_map.insert(table, exists > 0); } - let tx = self.conn.unchecked_transaction()?; for table in ["queue", "history"] { if table_exists_map.get(&table).copied().unwrap_or(false) { tx.execute_batch(&format!( @@ -433,7 +451,7 @@ impl Database { ))?; } } - commit_migration(tx, 15)?; + applied = 15; } if version < 16 { @@ -461,7 +479,6 @@ impl Database { } else { 0 }; - let tx = self.conn.unchecked_transaction()?; if has_headers > 0 { tx.execute_batch( " @@ -486,7 +503,340 @@ impl Database { if has_groups > 0 && has_scan_server == 0 { tx.execute_batch("ALTER TABLE groups ADD COLUMN scan_server_id TEXT;")?; } - commit_migration(tx, 16)?; + applied = 16; + } + + if version < 17 { + info!("Applying database migration v17: NZB content digests for duplicate detection"); + // Partial schemas (as built by migration tests) may lack either + // table, so only the tables that exist gain the column. Existing + // rows are backfilled from their stored NZB so they can be matched. + for (table, blob_column) in [("queue", "nzb_raw"), ("history", "nzb_data")] { + let table_exists: i64 = tx.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + )?; + if table_exists == 0 { + continue; + } + tx.execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN nzb_digest TEXT; + CREATE INDEX IF NOT EXISTS idx_{table}_nzb_digest ON {table}(nzb_digest);" + ))?; + let has_blob: i64 = tx.query_row( + "SELECT COUNT(*) FROM pragma_table_info(?1) WHERE name = ?2", + params![table, blob_column], + |row| row.get(0), + )?; + if has_blob == 0 { + continue; + } + let rows: Vec<(String, Vec)> = { + let mut stmt = tx.prepare(&format!( + "SELECT id, {blob_column} FROM {table} WHERE length({blob_column}) > 0" + ))?; + stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>()? + }; + for (id, data) in rows { + tx.execute( + &format!("UPDATE {table} SET nzb_digest = ?2 WHERE id = ?1"), + params![id, nzb_digest(&data)], + )?; + } + } + applied = 17; + } + + if version < 18 { + info!("Applying database migration v18: history list columns ahead of the NZB blob"); + // Columns added to history by ALTER TABLE are stored after + // nzb_data, so reading them walks the blob's overflow chain (about + // 1.3 s for a 50-row page on a production database). List queries + // read them from this narrow copy instead; triggers keep it in step. + let has_history: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + )?; + // Partial schemas (as built by migration tests) may lack the + // source columns: they still get the table, so list queries work. + let source_columns: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM pragma_table_info('history') WHERE name IN + ('download_time_secs', 'server_stats', 'failure_code', + 'post_processing', 'delete_archives')", + [], + |row| row.get(0), + )?; + tx.execute_batch( + "CREATE TABLE IF NOT EXISTS history_list_meta ( + id TEXT PRIMARY KEY, + download_time_secs REAL, + server_stats TEXT, + failure_code TEXT, + post_processing INTEGER, + delete_archives INTEGER + );", + )?; + if has_history > 0 && source_columns == 5 { + tx.execute_batch( + "INSERT INTO history_list_meta + SELECT id, download_time_secs, server_stats, failure_code, + post_processing, delete_archives FROM history; + CREATE TRIGGER history_list_meta_insert AFTER INSERT ON history BEGIN + INSERT OR REPLACE INTO history_list_meta VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives); + END; + CREATE TRIGGER history_list_meta_update AFTER UPDATE OF + id, download_time_secs, server_stats, failure_code, + post_processing, delete_archives ON history BEGIN + DELETE FROM history_list_meta WHERE id = OLD.id; + INSERT OR REPLACE INTO history_list_meta VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives); + END; + CREATE TRIGGER history_list_meta_delete AFTER DELETE ON history BEGIN + DELETE FROM history_list_meta WHERE id = OLD.id; + END; + CREATE INDEX IF NOT EXISTS idx_history_category ON history(category);", + )?; + } + applied = 18; + } + + if version < 19 { + info!("Applying database migration v19: PAR2 volumes fetched and available"); + // Stored in history and copied to history_list_meta, so list + // queries read them without walking the NZB blob (v18). + let has_history: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + )?; + let has_triggers: i64 = self.conn.query_row( + "SELECT COUNT(*) FROM sqlite_master + WHERE type = 'trigger' AND name LIKE 'history_list_meta_%'", + [], + |row| row.get(0), + )?; + tx.execute_batch( + "ALTER TABLE history_list_meta ADD COLUMN par2_volumes_fetched INTEGER; + ALTER TABLE history_list_meta ADD COLUMN par2_volumes_available INTEGER;", + )?; + if has_history > 0 { + tx.execute_batch( + "ALTER TABLE history ADD COLUMN par2_volumes_fetched INTEGER; + ALTER TABLE history ADD COLUMN par2_volumes_available INTEGER;", + )?; + } + if has_triggers > 0 { + tx.execute_batch( + "DROP TRIGGER history_list_meta_insert; + DROP TRIGGER history_list_meta_update; + CREATE TRIGGER history_list_meta_insert AFTER INSERT ON history BEGIN + INSERT OR REPLACE INTO history_list_meta VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives, + NEW.par2_volumes_fetched, NEW.par2_volumes_available); + END; + CREATE TRIGGER history_list_meta_update AFTER UPDATE OF + id, download_time_secs, server_stats, failure_code, + post_processing, delete_archives, par2_volumes_fetched, + par2_volumes_available ON history BEGIN + DELETE FROM history_list_meta WHERE id = OLD.id; + INSERT OR REPLACE INTO history_list_meta VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives, + NEW.par2_volumes_fetched, NEW.par2_volumes_available); + END;", + )?; + } + applied = 19; + } + + if version < 20 { + info!("Applying database migration v20: restorable history deletions"); + // A history delete the UI offers to undo marks the row instead of + // dropping it: `trash = 1` hides it from every history read, and + // `trash_deadline` says when it stops being restorable. The row + // keeps its NZB and retry blobs, so an undo is lossless; a purge + // drops the row and its ledger entries exactly like a hard delete. + let has_history: bool = self + .conn + .query_row( + "SELECT COUNT(*) > 0 FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + ) + .unwrap_or(false); + if has_history { + add_column_if_missing(&tx, "history", "trash", "INTEGER NOT NULL DEFAULT 0")?; + add_column_if_missing(&tx, "history", "trash_deadline", "TEXT")?; + // `trash` is added after `nzb_data`, so reading it from + // `history` walks the blob's overflow chain (v18's reason for + // the narrow copy). List queries filter the copy instead. + let has_meta: bool = tx + .query_row( + "SELECT COUNT(*) > 0 FROM sqlite_master + WHERE type = 'table' AND name = 'history_list_meta'", + [], + |row| row.get(0), + ) + .unwrap_or(false); + if has_meta { + add_column_if_missing(&tx, "history_list_meta", "trash", "INTEGER")?; + tx.execute_batch( + "DROP TRIGGER IF EXISTS history_list_meta_insert; + DROP TRIGGER IF EXISTS history_list_meta_update; + DROP TRIGGER IF EXISTS history_list_meta_delete; + CREATE TRIGGER history_list_meta_insert AFTER INSERT ON history BEGIN + INSERT OR REPLACE INTO history_list_meta ( + id, download_time_secs, server_stats, failure_code, + post_processing, delete_archives, par2_volumes_fetched, + par2_volumes_available, trash) + VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives, + NEW.par2_volumes_fetched, NEW.par2_volumes_available, + NEW.trash); + END; + CREATE TRIGGER history_list_meta_update AFTER UPDATE ON history BEGIN + DELETE FROM history_list_meta WHERE id = OLD.id; + INSERT OR REPLACE INTO history_list_meta ( + id, download_time_secs, server_stats, failure_code, + post_processing, delete_archives, par2_volumes_fetched, + par2_volumes_available, trash) + VALUES ( + NEW.id, NEW.download_time_secs, NEW.server_stats, + NEW.failure_code, NEW.post_processing, NEW.delete_archives, + NEW.par2_volumes_fetched, NEW.par2_volumes_available, + NEW.trash); + END; + CREATE TRIGGER history_list_meta_delete AFTER DELETE ON history BEGIN + DELETE FROM history_list_meta WHERE id = OLD.id; + END; + UPDATE history_list_meta SET trash = COALESCE( + (SELECT trash FROM history h WHERE h.id = history_list_meta.id), 0);", + )?; + } + } + applied = 20; + } + + if version < 21 { + info!("Applying database migration v21: covering indexes for history aggregates"); + // The history panel's statistics (completed/failed counts, bytes, + // duration and failure reasons) used to load every summary row + // just to fold it in Rust, which meant a cold read of the whole + // table. `history_window_stats` and `history_failure_messages` + // aggregate in SQLite instead, and these indexes let them, the + // category list and the paged count run without opening the table + // or walking any row's stored NZB blob. + // + // `idx_h_stats_cover` also serves the page and its count; the + // partial `idx_h_fail_cover` carries only the failed rows' error + // text. Its predicate must spell `COLLATE NOCASE`, because the + // query compares status case-insensitively and SQLite only uses a + // partial index whose term matches the query's collation. + // + // `idx_meta_trash` matters most: the anti-join that hides trashed + // rows probes `history_list_meta(trash, id)` on every stats, + // category and count query, and without it each probe walks the + // meta table. + // + // Each index is created only when its table and columns exist: + // migration tests build partial schemas to exercise older steps, + // and those must still reach the newest schema version. + create_index_if_columns( + &tx, + "history", + &["completed_at", "status", "total_bytes", "added_at", "id"], + "CREATE INDEX IF NOT EXISTS idx_h_stats_cover + ON history(completed_at DESC, status, total_bytes, added_at, id);", + )?; + create_index_if_columns( + &tx, + "history", + &["category", "id"], + "CREATE INDEX IF NOT EXISTS idx_h_cat_cover ON history(category, id);", + )?; + create_index_if_columns( + &tx, + "history_list_meta", + &["trash", "id"], + "CREATE INDEX IF NOT EXISTS idx_meta_trash ON history_list_meta(trash, id);", + )?; + create_index_if_columns( + &tx, + "history", + &["completed_at", "error_message", "id", "status"], + "CREATE INDEX IF NOT EXISTS idx_h_fail_cover + ON history(completed_at DESC, error_message, id) + WHERE status = 'Failed' COLLATE NOCASE;", + )?; + applied = 21; + } + + if version < 22 { + info!("Applying database migration v22: indexed duplicate lookup by name"); + // The duplicate check on every add matches names ignoring case. + // Without an index on LOWER(name) it scanned the whole history + // table, whose rows carry their NZB blob inline: on a 2.3 GB + // database that took 18 s under the global DB lock and froze the + // GUI. These expression indexes (with the v17 digest indexes) let + // each branch of the lookup seek instead. + for table in ["queue", "history"] { + create_index_if_columns( + &tx, + table, + &["name"], + &format!( + "CREATE INDEX IF NOT EXISTS idx_{table}_name_lower ON {table}(LOWER(name));" + ), + )?; + } + applied = 22; + } + + if version < 23 { + info!("Applying database migration v23: history rows kept after an *arr delete"); + // An *arr "Remove Completed" deletes the job's files through the + // SABnzbd API but leaves rustnzb's own History row in place. The + // marker lives in its own narrow table, so list reads join it + // without touching columns stored after the NZB blob, and the + // SABnzbd history the *arrs poll can skip the marked rows. + tx.execute_batch( + "CREATE TABLE IF NOT EXISTS history_client_removals ( + id TEXT PRIMARY KEY, + removed_by TEXT NOT NULL, + removed_at TEXT NOT NULL, + files_deleted INTEGER NOT NULL DEFAULT 0 + );", + )?; + let has_history: bool = tx + .query_row( + "SELECT COUNT(*) > 0 FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + ) + .unwrap_or(false); + if has_history { + tx.execute_batch( + "CREATE TRIGGER IF NOT EXISTS history_client_removals_delete + AFTER DELETE ON history BEGIN + DELETE FROM history_client_removals WHERE id = OLD.id; + END;", + )?; + } + applied = 23; + } + + if applied > version { + stamp_schema_version(&tx, applied)?; + tx.commit()?; + info!(version = applied, "Database schema up to date"); } Ok(()) @@ -554,9 +904,9 @@ impl Database { "INSERT INTO queue (id, name, category, status, priority, total_bytes, downloaded_bytes, file_count, files_completed, article_count, articles_downloaded, articles_failed, added_at, work_dir, output_dir, password, - nzb_raw, pp_override, delete_archives) + nzb_raw, pp_override, delete_archives, nzb_digest) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, - ?16, ?17, ?18, ?19)", + ?16, ?17, ?18, ?19, ?20)", params![ job.id, job.name, @@ -577,6 +927,7 @@ impl Database { nzb_data, job.pp_override, job.delete_archives, + nzb_digest(nzb_data), ], )?; @@ -737,6 +1088,41 @@ impl Database { Ok(()) } + /// Update a job's progress counters without touching its status, for the + /// periodic progress save: status is written separately, from the job + /// list's current value, so a stale snapshot never overwrites a pause or + /// resume. + pub fn queue_update_counters( + &self, + id: &str, + downloaded_bytes: u64, + articles_downloaded: usize, + articles_failed: usize, + files_completed: usize, + ) -> Result<(), NzbError> { + self.conn.execute( + "UPDATE queue SET downloaded_bytes=?2, articles_downloaded=?3, + articles_failed=?4, files_completed=?5 WHERE id=?1", + params![ + id, + downloaded_bytes as i64, + articles_downloaded as i64, + articles_failed as i64, + files_completed as i64, + ], + )?; + Ok(()) + } + + /// Update only a queued job's status. + pub fn queue_update_status(&self, id: &str, status: JobStatus) -> Result<(), NzbError> { + self.conn.execute( + "UPDATE queue SET status=?2 WHERE id=?1", + params![id, status.to_string()], + )?; + Ok(()) + } + /// Update the non-terminal, user-visible queue message for a job. pub fn queue_update_error_message( &self, @@ -779,6 +1165,8 @@ impl Database { let jobs = stmt .query_map([], |row| { Ok(NzbJob { + slice_map: None, + held_par2_bytes: 0, id: row.get(0)?, name: row.get(1)?, category: row.get(2)?, @@ -815,6 +1203,37 @@ impl Database { // History operations // ----------------------------------------------------------------------- + /// Find queued and history jobs that an incoming NZB matches by name + /// (ignoring ASCII case) or, when `digest` is given, by content digest. + /// Only a history job that completed successfully matches; failed and + /// deleted (trashed) history jobs never do. An arr re-grabbing a release + /// whose earlier attempt failed or was removed is therefore admitted as a + /// new job, and the old Failed history row is kept next to it (it is not + /// replaced). Queued jobs always match. + pub fn find_duplicate_candidates( + &self, + name: &str, + digest: Option<&str>, + ) -> Result, NzbError> { + let mut stmt = self.conn.prepare(DUPLICATE_CANDIDATES_SQL)?; + let matches = stmt + .query_map(params![name, digest], |row| { + let location = match row.get::<_, String>(4)?.as_str() { + "queue" => DuplicateLocation::Queue, + _ => DuplicateLocation::History, + }; + Ok(DuplicateMatch { + id: row.get(0)?, + name: row.get(1)?, + by_name: row.get::<_, i64>(2)? != 0, + by_digest: row.get::<_, i64>(3)? != 0, + location, + }) + })? + .collect::>>()?; + Ok(matches) + } + /// Move a completed/failed job to history. pub fn history_insert(&self, entry: &HistoryEntry) -> Result<(), NzbError> { self.history_insert_with_carried_bytes(entry, 0) @@ -844,9 +1263,10 @@ impl Database { tx.execute( "INSERT INTO history (id, name, category, status, total_bytes, downloaded_bytes, added_at, completed_at, download_time_secs, output_dir, stages, error_message, - nzb_data, server_stats, retry_data, failure_code, post_processing, delete_archives) + nzb_data, server_stats, retry_data, failure_code, post_processing, delete_archives, + nzb_digest, par2_volumes_fetched, par2_volumes_available) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, - ?17, ?18)", + ?17, ?18, ?19, ?20, ?21)", params![ entry.id, entry.name, @@ -866,6 +1286,9 @@ impl Database { entry.failure_code.map(|code| code.to_string()), entry.post_processing, entry.delete_archives, + entry.nzb_data.as_deref().map(nzb_digest), + entry.par2_volumes_fetched, + entry.par2_volumes_available, ], )?; @@ -929,25 +1352,89 @@ impl Database { /// List history entries, most recent first. pub fn history_list(&self, limit: usize) -> Result, NzbError> { - let mut stmt = self.conn.prepare( - "SELECT id, name, category, status, total_bytes, downloaded_bytes, - added_at, completed_at, download_time_secs, output_dir, stages, error_message, server_stats, - CASE WHEN nzb_data IS NOT NULL THEN 1 ELSE 0 END as has_nzb, failure_code, - post_processing, delete_archives - FROM history ORDER BY completed_at DESC LIMIT ?1", - )?; + let mut stmt = self.conn.prepare(&format!( + "SELECT {HISTORY_LIST_COLUMNS} FROM {HISTORY_LIST_FROM} + WHERE COALESCE(m.trash, 0) = 0 ORDER BY h.completed_at DESC LIMIT ?1" + ))?; + let entries = stmt + .query_map(params![limit as i64], history_list_row)? + .collect::, _>>()?; + Ok(entries) + } + /// One page of history matching `filter`, most recent first, plus the + /// total number of matching rows. Filtering, ordering and paging all run + /// in SQL, and the stored NZB and retry blobs are never read. + pub fn history_page( + &self, + filter: &HistoryFilter, + offset: usize, + limit: usize, + ) -> Result<(Vec, usize), NzbError> { + let (where_sql, args) = filter.to_sql(); + // Count with an anti-join, not the page's `LEFT JOIN`: the page still + // needs the meta columns, but the count only needs to know that no + // trashed row matched, and the anti-join lets it run entirely on the + // covering indexes. + let total: i64 = self.conn.query_row( + &format!( + "SELECT COUNT(*) FROM history h{}", + with_condition( + &where_sql, + "h.id NOT IN (SELECT id FROM history_list_meta WHERE trash = 1)" + ) + ), + rusqlite::params_from_iter(args.iter()), + |row| row.get(0), + )?; + let mut stmt = self + .conn + .prepare(&history_page_sql(&where_sql, args.len()))?; + let mut page_args = args; + page_args.push(Box::new(limit.min(i64::MAX as usize) as i64)); + page_args.push(Box::new(offset.min(i64::MAX as usize) as i64)); let entries = stmt - .query_map(params![limit as i64], |row| { - let stages_json: String = row.get::<_, Option>(10)?.unwrap_or_default(); - let stages: Vec = - serde_json::from_str(&stages_json).unwrap_or_default(); - let stats_json: String = row.get::<_, Option>(12)?.unwrap_or_default(); - let server_stats: Vec = - serde_json::from_str(&stats_json).unwrap_or_default(); - let has_nzb: i64 = row.get(13)?; - - Ok(HistoryEntry { + .query_map( + rusqlite::params_from_iter(page_args.iter()), + history_list_row, + )? + .collect::, _>>()?; + Ok((entries, total as usize)) + } + + /// Distinct non-empty history categories, sorted. + /// + /// Trashed rows are excluded with an anti-join rather than the list + /// query's `LEFT JOIN` + `COALESCE(m.trash, 0) = 0`: the subquery only + /// ever yields rows whose `trash` is 1, so its ids are never NULL and + /// `NOT IN` and the `COALESCE` test select the same rows. Either way the + /// plan can read `idx_h_cat_cover` and probe `idx_meta_trash` without + /// opening the `history` table. + pub fn history_categories(&self) -> Result, NzbError> { + let mut stmt = self.conn.prepare(&history_categories_sql())?; + let categories = stmt + .query_map([], |row| row.get(0))? + .collect::, _>>()?; + Ok(categories) + } + + /// Lightweight view of every history row matching `filter`, most recent + /// first. Only the columns stored ahead of the NZB blob are read, so this + /// stays cheap however large the stored NZBs are. + pub fn history_summaries( + &self, + filter: &HistoryFilter, + ) -> Result, NzbError> { + let (where_sql, args) = filter.to_sql(); + let mut stmt = self.conn.prepare(&format!( + "SELECT h.id, h.name, h.category, h.status, h.total_bytes, h.downloaded_bytes, + h.added_at, h.completed_at, h.output_dir, h.error_message + FROM {HISTORY_LIST_FROM}{} ORDER BY h.completed_at DESC", + with_condition(&where_sql, "COALESCE(m.trash, 0) = 0") + ))?; + let rows = stmt + .query_map(rusqlite::params_from_iter(args.iter()), |row| { + Ok(HistorySummary { id: row.get(0)?, name: row.get(1)?, category: row.get(2)?, @@ -956,21 +1443,74 @@ impl Database { downloaded_bytes: row.get::<_, i64>(5)? as u64, added_at: parse_datetime(&row.get::<_, String>(6)?), completed_at: parse_datetime(&row.get::<_, String>(7)?), - download_time_secs: row.get(8)?, - output_dir: row.get::<_, String>(9)?.into(), - stages, - error_message: row.get(11)?, - failure_code: parse_failure_code(row.get(14)?)?, - post_processing: row.get(15)?, - delete_archives: row.get(16)?, - server_stats, - // Don't load actual blob in list - just note if it exists - nzb_data: if has_nzb != 0 { Some(Vec::new()) } else { None }, - retry_data: None, + output_dir: row.get::<_, String>(8)?.into(), + error_message: row.get(9)?, }) })? .collect::, _>>()?; + Ok(rows) + } + + /// Completed/failed counts, completed bytes and completed duration over + /// every history row matching `filter`. + /// + /// The history panel's statistics used to be folded in Rust from + /// [`Self::history_summaries`], which reads a summary row (including its + /// error text) for every entry in the window. The same fold runs here as + /// one aggregate over `idx_h_stats_cover`, so a cold panel load touches a + /// narrow index instead of the table. + pub fn history_window_stats( + &self, + filter: &HistoryFilter, + ) -> Result { + let (where_sql, args) = filter.to_sql(); + let sql = history_window_stats_sql(&where_sql); + let stats = self + .conn + .query_row(&sql, rusqlite::params_from_iter(args.iter()), |row| { + Ok(HistoryWindowStats { + completed: row.get::<_, i64>(0)? as u64, + completed_bytes: row.get::<_, i64>(1)? as u64, + completed_duration_ms: row.get(2)?, + failed: row.get::<_, i64>(3)? as u64, + }) + })?; + Ok(stats) + } + + /// Error text of every failed row matching `filter`, most recent first. + /// + /// Only the failure reasons need the text, so this reads the partial + /// `idx_h_fail_cover` index instead of the summary columns. The index is + /// named explicitly because the planner would otherwise prefer the + /// completed-at index when a days window narrows the scan, and then walk + /// the table for each row's error message. + pub fn history_failure_messages( + &self, + filter: &HistoryFilter, + ) -> Result>, NzbError> { + let (where_sql, args) = filter.to_sql(); + let sql = history_failure_messages_sql(&where_sql); + let mut stmt = self.conn.prepare(&sql)?; + let messages = stmt + .query_map(rusqlite::params_from_iter(args.iter()), |row| row.get(0))? + .collect::, _>>()?; + Ok(messages) + } + /// List-shaped history entries for `ids`, in the order given. Unknown ids + /// are skipped. Like [`Database::history_list`], blobs are not loaded. + pub fn history_get_listed(&self, ids: &[String]) -> Result, NzbError> { + let mut stmt = self.conn.prepare(&format!( + "SELECT {HISTORY_LIST_COLUMNS} FROM {HISTORY_LIST_FROM} + WHERE h.id = ?1 AND COALESCE(m.trash, 0) = 0" + ))?; + let mut entries = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(entry) = stmt.query_row(params![id], history_list_row).optional()? { + entries.push(entry); + } + } Ok(entries) } @@ -1002,32 +1542,183 @@ impl Database { } } - /// Enforce history retention limit by deleting oldest entries. + /// Enforce history retention limit by deleting oldest entries, and the + /// ledger rows of jobs now gone. /// /// A limit of `0` is treated as "keep all" and is a no-op: the /// `LIMIT 0` subquery would otherwise match nothing and delete every /// row (GH #136). Callers normalize zero away already; this is the last /// line of defence. + /// + /// Trashed entries are not part of the limit: they are already gone from + /// the user's view, and retention must not silently consume an undo + /// window that the purge deadline owns. pub fn history_enforce_retention(&self, max_entries: usize) -> Result<(), NzbError> { if max_entries == 0 { return Ok(()); } self.conn.execute( - "DELETE FROM history WHERE id NOT IN ( - SELECT id FROM history ORDER BY completed_at DESC LIMIT ?1 + "DELETE FROM history WHERE trash = 0 AND id NOT IN ( + SELECT id FROM history WHERE trash = 0 + ORDER BY completed_at DESC LIMIT ?1 )", params![max_entries as i64], )?; + self.ledger_drop_orphans()?; Ok(()) } + /// Move history rows into the undo window instead of deleting them. + /// + /// Each row keeps its NZB and retry blobs, so [`Self::history_restore`] is + /// lossless. Returns the summaries of the rows actually trashed, in the + /// order given; an id that is unknown or already trashed is skipped, since + /// the caller reports that as a per-id failure. + pub fn history_trash( + &self, + ids: &[String], + deadline: chrono::DateTime, + ) -> Result, NzbError> { + let mut trashed = Vec::with_capacity(ids.len()); + let tx = self.conn.unchecked_transaction()?; + for id in ids { + let changed = tx.execute( + "UPDATE history SET trash = 1, trash_deadline = ?2 + WHERE id = ?1 AND trash = 0", + params![id, deadline.to_rfc3339()], + )?; + if changed == 0 { + continue; + } + if let Some(summary) = history_summary_by_id(&tx, id)? { + trashed.push(summary); + } + } + tx.commit()?; + Ok(trashed) + } + + /// Bring trashed rows back before their deadline. Returns the ids that + /// were restored, in the order given; an unknown id, or one whose window + /// has closed, is skipped. + pub fn history_restore( + &self, + ids: &[String], + now: chrono::DateTime, + ) -> Result, NzbError> { + let mut restored = Vec::with_capacity(ids.len()); + let tx = self.conn.unchecked_transaction()?; + for id in ids { + let changed = tx.execute( + "UPDATE history SET trash = 0, trash_deadline = NULL + WHERE id = ?1 AND trash = 1 + AND (trash_deadline IS NULL OR trash_deadline >= ?2)", + params![id, now.to_rfc3339()], + )?; + if changed > 0 { + restored.push(id.clone()); + } + } + tx.commit()?; + Ok(restored) + } + + /// Delete every trashed row whose deadline has passed, and the ledger rows + /// of those jobs. Returns the summaries that were purged so the caller can + /// reclaim their retained work directories. + pub fn history_purge_expired( + &self, + now: chrono::DateTime, + ) -> Result, NzbError> { + let tx = self.conn.unchecked_transaction()?; + let mut purged = Vec::new(); + { + let mut stmt = tx.prepare( + "SELECT id, name, category, status, total_bytes, downloaded_bytes, added_at, + completed_at, output_dir, error_message + FROM history + WHERE trash = 1 AND (trash_deadline IS NULL OR trash_deadline < ?1)", + )?; + let rows = stmt.query_map(params![now.to_rfc3339()], |row| { + Ok(HistorySummary { + id: row.get(0)?, + name: row.get(1)?, + category: row.get(2)?, + status: parse_status(&row.get::<_, String>(3)?), + total_bytes: row.get::<_, i64>(4)? as u64, + downloaded_bytes: row.get::<_, i64>(5)? as u64, + added_at: parse_datetime(&row.get::<_, String>(6)?), + completed_at: parse_datetime(&row.get::<_, String>(7)?), + output_dir: row.get::<_, String>(8)?.into(), + error_message: row.get(9)?, + }) + })?; + for row in rows { + purged.push(row?); + } + } + tx.execute( + "DELETE FROM history + WHERE trash = 1 AND (trash_deadline IS NULL OR trash_deadline < ?1)", + params![now.to_rfc3339()], + )?; + tx.commit()?; + self.ledger_drop_orphans()?; + Ok(purged) + } + + /// Every trashed row, regardless of deadline. Used to keep a trashed + /// row's retained work directories out of the orphan sweep while its + /// undo window is open. + pub fn history_trashed_summaries(&self) -> Result, NzbError> { + let mut stmt = self.conn.prepare( + "SELECT id, name, category, status, total_bytes, downloaded_bytes, added_at, + completed_at, output_dir, error_message + FROM history WHERE trash = 1", + )?; + let rows = stmt + .query_map([], |row| { + Ok(HistorySummary { + id: row.get(0)?, + name: row.get(1)?, + category: row.get(2)?, + status: parse_status(&row.get::<_, String>(3)?), + total_bytes: row.get::<_, i64>(4)? as u64, + downloaded_bytes: row.get::<_, i64>(5)? as u64, + added_at: parse_datetime(&row.get::<_, String>(6)?), + completed_at: parse_datetime(&row.get::<_, String>(7)?), + output_dir: row.get::<_, String>(8)?.into(), + error_message: row.get(9)?, + }) + })? + .collect::, _>>()?; + Ok(rows) + } + + /// Ids of every trashed row, regardless of deadline. Used to keep their + /// retained work directories out of the orphan sweep while an undo is + /// still possible. + pub fn history_trashed_ids(&self) -> Result, NzbError> { + let mut stmt = self + .conn + .prepare("SELECT id FROM history WHERE trash = 1")?; + let ids = stmt + .query_map([], |row| row.get(0))? + .collect::, _>>()?; + Ok(ids) + } + /// Get a single history entry by ID. pub fn history_get(&self, id: &str) -> Result, NzbError> { let mut stmt = self.conn.prepare( - "SELECT id, name, category, status, total_bytes, downloaded_bytes, - added_at, completed_at, download_time_secs, output_dir, stages, error_message, server_stats, - failure_code, post_processing, delete_archives - FROM history WHERE id = ?1", + "SELECT h.id, h.name, h.category, h.status, h.total_bytes, h.downloaded_bytes, + h.added_at, h.completed_at, m.download_time_secs, h.output_dir, h.stages, + h.error_message, m.server_stats, m.failure_code, m.post_processing, m.delete_archives, + m.par2_volumes_fetched, m.par2_volumes_available, + r.removed_by, r.removed_at, r.files_deleted + FROM history h LEFT JOIN history_list_meta m ON m.id = h.id + LEFT JOIN history_client_removals r ON r.id = h.id + WHERE h.id = ?1 AND COALESCE(m.trash, 0) = 0", )?; let result = stmt.query_row(params![id], |row| { @@ -1056,6 +1747,9 @@ impl Database { server_stats, nzb_data: None, retry_data: None, + par2_volumes_fetched: row.get(16)?, + par2_volumes_available: row.get(17)?, + client_removal: client_removal_at(row, 18)?, }) }); @@ -1087,8 +1781,8 @@ impl Database { /// Store raw NZB data for a queue job. pub fn queue_store_nzb_data(&self, id: &str, nzb_data: &[u8]) -> Result<(), NzbError> { self.conn.execute( - "UPDATE queue SET nzb_raw = ?2 WHERE id = ?1", - params![id, nzb_data], + "UPDATE queue SET nzb_raw = ?2, nzb_digest = ?3 WHERE id = ?1", + params![id, nzb_data, nzb_digest(nzb_data)], )?; Ok(()) } @@ -1109,22 +1803,63 @@ impl Database { /// Count history entries. pub fn history_count(&self) -> Result { - let count: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM history", [], |row| row.get(0))?; + let count: i64 = self.conn.query_row( + &format!("SELECT COUNT(*) FROM {HISTORY_LIST_FROM} WHERE COALESCE(m.trash, 0) = 0"), + [], + |row| row.get(0), + )?; Ok(count as usize) } - /// Remove a history entry. + /// Mark history rows as removed by an external client (an *arr through + /// the SABnzbd API) instead of deleting them. Returns how many rows were + /// marked; an unknown id is skipped. + pub fn history_mark_client_removed( + &self, + ids: &[String], + removed_by: &str, + removed_at: chrono::DateTime, + files_deleted: bool, + ) -> Result { + let tx = self.conn.unchecked_transaction()?; + let mut marked = 0; + for id in ids { + marked += tx.execute( + "INSERT OR REPLACE INTO history_client_removals + (id, removed_by, removed_at, files_deleted) + SELECT id, ?2, ?3, ?4 FROM history WHERE id = ?1", + params![id, removed_by, removed_at.to_rfc3339(), files_deleted], + )?; + } + tx.commit()?; + Ok(marked) + } + + /// Ids of every history row an external client has removed. + pub fn history_client_removed_ids( + &self, + ) -> Result, NzbError> { + let mut stmt = self + .conn + .prepare("SELECT id FROM history_client_removals")?; + let ids = stmt + .query_map([], |row| row.get(0))? + .collect::>()?; + Ok(ids) + } + + /// Remove a history entry, and the ledger rows of a job now gone. pub fn history_remove(&self, id: &str) -> Result<(), NzbError> { self.conn .execute("DELETE FROM history WHERE id=?1", params![id])?; + self.ledger_drop_orphans()?; Ok(()) } - /// Clear all history. + /// Clear all history, and the ledger rows of jobs now gone. pub fn history_clear(&self) -> Result<(), NzbError> { self.conn.execute("DELETE FROM history", [])?; + self.ledger_drop_orphans()?; Ok(()) } @@ -1406,27 +2141,277 @@ impl Database { // Parse helpers // --------------------------------------------------------------------------- -fn parse_status(s: &str) -> JobStatus { - match s.to_lowercase().as_str() { - "queued" => JobStatus::Queued, - "downloading" => JobStatus::Downloading, - "paused" => JobStatus::Paused, - "verifying" => JobStatus::Verifying, - "repairing" => JobStatus::Repairing, - "extracting" => JobStatus::Extracting, - "postprocessing" => JobStatus::PostProcessing, - "completed" => JobStatus::Completed, - "failed" => JobStatus::Failed, - _ => JobStatus::Queued, +/// Columns of a history list row, selected `FROM` [`HISTORY_LIST_FROM`]. +/// `has_nzb` uses `typeof()`, which SQLite answers from the record header +/// without loading the blob itself. Columns stored after `nzb_data` in +/// `history` come from `history_list_meta`, so no blob overflow chain is +/// walked for a list row. +const HISTORY_LIST_COLUMNS: &str = "h.id, h.name, h.category, h.status, h.total_bytes, + h.downloaded_bytes, h.added_at, h.completed_at, m.download_time_secs, h.output_dir, h.stages, + h.error_message, m.server_stats, typeof(h.nzb_data) <> 'null' AS has_nzb, m.failure_code, + m.post_processing, m.delete_archives, m.par2_volumes_fetched, m.par2_volumes_available, + r.removed_by, r.removed_at, r.files_deleted"; + +/// Duplicate lookup for `find_duplicate_candidates`. Name and digest +/// matches are separate SELECTs joined by UNION (which also removes a row +/// matched both ways) so each one seeks its own index — +/// `idx_*_name_lower` or `idx_*_nzb_digest` — rather than SQLite falling +/// back to a full scan for the OR. Only matching rows are ever read. +const DUPLICATE_CANDIDATES_SQL: &str = " + SELECT id, name, LOWER(name) = LOWER(?1), COALESCE(nzb_digest = ?2, 0), 'queue' + FROM queue WHERE LOWER(name) = LOWER(?1) + UNION + SELECT id, name, LOWER(name) = LOWER(?1), COALESCE(nzb_digest = ?2, 0), 'queue' + FROM queue WHERE nzb_digest = ?2 + UNION + SELECT id, name, LOWER(name) = LOWER(?1), COALESCE(nzb_digest = ?2, 0), 'history' + FROM history WHERE LOWER(name) = LOWER(?1) AND trash = 0 + AND status = 'Completed' COLLATE NOCASE + UNION + SELECT id, name, LOWER(name) = LOWER(?1), COALESCE(nzb_digest = ?2, 0), 'history' + FROM history WHERE nzb_digest = ?2 AND trash = 0 + AND status = 'Completed' COLLATE NOCASE"; + +/// Source of [`HISTORY_LIST_COLUMNS`]. +const HISTORY_LIST_FROM: &str = "history h LEFT JOIN history_list_meta m ON m.id = h.id + LEFT JOIN history_client_removals r ON r.id = h.id"; + +/// Append `condition` to a filter's ` WHERE ...` clause (or start one). +fn with_condition(where_sql: &str, condition: &str) -> String { + match where_sql.strip_prefix(" WHERE ") { + Some(rest) => format!(" WHERE {condition} AND {rest}"), + None => format!(" WHERE {condition}"), } } -fn parse_failure_code(value: Option) -> rusqlite::Result> { - value - .map(|value| { - value.parse().map_err(|_| { - rusqlite::Error::FromSqlConversionFailure( - 0, +fn history_page_sql(where_sql: &str, arg_count: usize) -> String { + // `trash = 0` is part of the page, not the caller's filter, so a paged + // request can never see a row that is inside its undo window. + format!( + "SELECT {HISTORY_LIST_COLUMNS} FROM {HISTORY_LIST_FROM}{} + ORDER BY h.completed_at DESC LIMIT ?{} OFFSET ?{}", + with_condition(where_sql, "COALESCE(m.trash, 0) = 0"), + arg_count + 1, + arg_count + 2 + ) +} + +/// One [`HistorySummary`] by id, read inside an open transaction. +fn history_summary_by_id(conn: &Connection, id: &str) -> Result, NzbError> { + let mut stmt = conn.prepare( + "SELECT id, name, category, status, total_bytes, downloaded_bytes, added_at, + completed_at, output_dir, error_message + FROM history WHERE id = ?1", + )?; + Ok(stmt + .query_row(params![id], |row| { + Ok(HistorySummary { + id: row.get(0)?, + name: row.get(1)?, + category: row.get(2)?, + status: parse_status(&row.get::<_, String>(3)?), + total_bytes: row.get::<_, i64>(4)? as u64, + downloaded_bytes: row.get::<_, i64>(5)? as u64, + added_at: parse_datetime(&row.get::<_, String>(6)?), + completed_at: parse_datetime(&row.get::<_, String>(7)?), + output_dir: row.get::<_, String>(8)?.into(), + error_message: row.get(9)?, + }) + }) + .optional()?) +} + +fn history_list_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let stages_json: String = row.get::<_, Option>(10)?.unwrap_or_default(); + let stages: Vec = serde_json::from_str(&stages_json).unwrap_or_default(); + let stats_json: String = row.get::<_, Option>(12)?.unwrap_or_default(); + let server_stats: Vec = + serde_json::from_str(&stats_json).unwrap_or_default(); + let has_nzb: bool = row.get(13)?; + + Ok(HistoryEntry { + id: row.get(0)?, + name: row.get(1)?, + category: row.get(2)?, + status: parse_status(&row.get::<_, String>(3)?), + total_bytes: row.get::<_, i64>(4)? as u64, + downloaded_bytes: row.get::<_, i64>(5)? as u64, + added_at: parse_datetime(&row.get::<_, String>(6)?), + completed_at: parse_datetime(&row.get::<_, String>(7)?), + download_time_secs: row.get(8)?, + output_dir: row.get::<_, String>(9)?.into(), + stages, + error_message: row.get(11)?, + failure_code: parse_failure_code(row.get(14)?)?, + post_processing: row.get(15)?, + delete_archives: row.get(16)?, + server_stats, + // Don't load actual blob in list - just note if it exists + nzb_data: if has_nzb { Some(Vec::new()) } else { None }, + retry_data: None, + par2_volumes_fetched: row.get(17)?, + par2_volumes_available: row.get(18)?, + client_removal: client_removal_at(row, 19)?, + }) +} + +/// The [`HistoryClientRemoval`] read from `removed_by`, `removed_at` and +/// `files_deleted`, starting at column `first`. +fn client_removal_at( + row: &rusqlite::Row<'_>, + first: usize, +) -> rusqlite::Result> { + let Some(removed_by) = row.get::<_, Option>(first)? else { + return Ok(None); + }; + Ok(Some(HistoryClientRemoval { + removed_by, + removed_at: parse_datetime(&row.get::<_, String>(first + 1)?), + files_deleted: row.get(first + 2)?, + })) +} + +/// SQL-side history filter. Every set field narrows the match. +#[derive(Debug, Clone, Default)] +pub struct HistoryFilter { + /// Only rows completed at or after this instant. + pub completed_since: Option>, + /// Exact status, compared case-insensitively. + pub status: Option, + /// Exact category. + pub category: Option, + /// Case-insensitive substring of the name. + pub search: Option, +} + +impl HistoryFilter { + fn to_sql(&self) -> (String, Vec>) { + let mut clauses: Vec = Vec::new(); + let mut args: Vec> = Vec::new(); + if let Some(since) = self.completed_since { + args.push(Box::new(since.to_rfc3339())); + clauses.push(format!("completed_at >= ?{}", args.len())); + } + if let Some(status) = self.status { + args.push(Box::new(status.to_string())); + clauses.push(format!("status = ?{} COLLATE NOCASE", args.len())); + } + if let Some(category) = &self.category { + args.push(Box::new(category.clone())); + clauses.push(format!("category = ?{}", args.len())); + } + if let Some(search) = &self.search { + args.push(Box::new(search.to_lowercase())); + clauses.push(format!("instr(lower(name), ?{}) > 0", args.len())); + } + if clauses.is_empty() { + (String::new(), args) + } else { + (format!(" WHERE {}", clauses.join(" AND ")), args) + } + } +} + +/// The history columns stored ahead of the NZB blob: enough to filter, +/// count, total and delete without touching the blob's overflow pages. +#[derive(Debug, Clone)] +pub struct HistorySummary { + pub id: String, + pub name: String, + pub category: String, + pub status: JobStatus, + pub total_bytes: u64, + pub downloaded_bytes: u64, + pub added_at: chrono::DateTime, + pub completed_at: chrono::DateTime, + pub output_dir: std::path::PathBuf, + pub error_message: Option, +} + +/// The aggregate half of the history panel's statistics: completed count, +/// completed bytes, completed duration and failed count. +/// +/// Kept beside [`HISTORY_LIST_FROM`]'s query helpers so the query-plan test +/// asserts on the SQL the endpoint actually runs. +fn history_window_stats_sql(where_sql: &str) -> String { + format!( + "SELECT + COALESCE(SUM(CASE WHEN h.status = 'Completed' COLLATE NOCASE + THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN h.status = 'Completed' COLLATE NOCASE + THEN h.total_bytes ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN h.status = 'Completed' COLLATE NOCASE + THEN CAST(ROUND((julianday(h.completed_at) + - julianday(h.added_at)) * 86400000.0) + AS INTEGER) + ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN h.status = 'Failed' COLLATE NOCASE + THEN 1 ELSE 0 END), 0) + FROM history h{}", + with_condition( + where_sql, + "h.id NOT IN (SELECT id FROM history_list_meta WHERE trash = 1)" + ) + ) +} + +/// Error text of the failed rows matching the same filter, newest first. +fn history_failure_messages_sql(where_sql: &str) -> String { + format!( + "SELECT h.error_message FROM history h INDEXED BY idx_h_fail_cover{} + ORDER BY h.completed_at DESC", + with_condition( + &with_condition(where_sql, "h.status = 'Failed' COLLATE NOCASE"), + "h.id NOT IN (SELECT id FROM history_list_meta WHERE trash = 1)" + ) + ) +} + +/// The distinct non-empty categories of the untrashed rows, in name order. +fn history_categories_sql() -> String { + "SELECT DISTINCT h.category FROM history h + WHERE h.id NOT IN (SELECT id FROM history_list_meta WHERE trash = 1) + AND h.category <> '' + ORDER BY h.category" + .to_string() +} + +/// The history panel's aggregates over a time window, computed in SQL. +/// +/// `completed_duration_ms` is the sum of each completed row's +/// `(completed_at - added_at)` in whole milliseconds. SQLite derives it from +/// `julianday`, whose double precision is finer than a millisecond, and rounds +/// to the nearest one, so the totals match the Rust fold's `num_milliseconds()` +/// except where a row's duration has a sub-millisecond remainder. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct HistoryWindowStats { + pub completed: u64, + pub completed_bytes: u64, + pub completed_duration_ms: i64, + pub failed: u64, +} + +fn parse_status(s: &str) -> JobStatus { + match s.to_lowercase().as_str() { + "queued" => JobStatus::Queued, + "downloading" => JobStatus::Downloading, + "paused" => JobStatus::Paused, + "verifying" => JobStatus::Verifying, + "repairing" => JobStatus::Repairing, + "extracting" => JobStatus::Extracting, + "postprocessing" => JobStatus::PostProcessing, + "completed" => JobStatus::Completed, + "failed" => JobStatus::Failed, + _ => JobStatus::Queued, + } +} + +fn parse_failure_code(value: Option) -> rusqlite::Result> { + value + .map(|value| { + value.parse().map_err(|_| { + rusqlite::Error::FromSqlConversionFailure( + 0, rusqlite::types::Type::Text, std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -1455,11 +2440,12 @@ fn parse_datetime(s: &str) -> chrono::DateTime { .unwrap_or_else(|_| Utc::now()) } -fn commit_migration(tx: rusqlite::Transaction<'_>, version: u32) -> Result<(), NzbError> { +/// Record the schema version a batch of migrations reached. The caller owns +/// the transaction and commits it once, after the last migration. +fn stamp_schema_version(tx: &rusqlite::Transaction<'_>, version: u32) -> Result<(), NzbError> { tx.execute_batch(&format!( "DELETE FROM schema_version; INSERT INTO schema_version (version) VALUES ({version});" ))?; - tx.commit()?; Ok(()) } @@ -1487,12 +2473,43 @@ fn add_column_if_missing( Ok(()) } +/// Run `definition` to create an index only when `table` exists and carries +/// every column it indexes. +/// +/// Migration tests build partial schemas to reach an older version, so a +/// later step must not assume the full table shape. An index over a column a +/// partial schema lacks is skipped, exactly as `add_column_if_missing` would +/// leave the column absent. +fn create_index_if_columns( + tx: &rusqlite::Transaction<'_>, + table: &str, + columns: &[&str], + definition: &str, +) -> Result<(), NzbError> { + for column in columns { + let exists: bool = tx + .query_row( + "SELECT COUNT(*) > 0 FROM pragma_table_info(?) WHERE name = ?", + [table, column], + |row| row.get(0), + ) + .unwrap_or(false); + if !exists { + return Ok(()); + } + } + tx.execute_batch(definition)?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; fn make_job(id: &str, name: &str) -> NzbJob { NzbJob { + slice_map: None, + held_par2_bytes: 0, id: id.into(), name: name.into(), category: "Default".into(), @@ -1568,6 +2585,7 @@ mod tests { fn make_history(id: &str, name: &str) -> HistoryEntry { HistoryEntry { + client_removal: None, id: id.into(), name: name.into(), category: "movies".into(), @@ -1591,9 +2609,125 @@ mod tests { server_stats: Vec::new(), nzb_data: None, retry_data: None, + par2_volumes_fetched: None, + par2_volumes_available: None, } } + #[test] + fn duplicate_candidates_match_queue_and_history_by_name_or_digest() { + let db = Database::open_memory().unwrap(); + let payload = b"queued"; + let digest = nzb_digest(payload); + + let queued = make_job("queued-job", "Show.S01E01"); + db.queue_insert(&queued).unwrap(); + db.queue_store_nzb_data(&queued.id, payload).unwrap(); + + let mut finished = make_history("done-job", "Other.Release"); + finished.nzb_data = Some(b"finished".to_vec()); + db.history_insert(&finished).unwrap(); + + // Name match is case-insensitive and reports the queue row. + let by_name = db + .find_duplicate_candidates("show.s01e01", Some(&nzb_digest(b"unrelated"))) + .unwrap(); + assert_eq!(by_name.len(), 1); + assert_eq!(by_name[0].id, "queued-job"); + assert_eq!(by_name[0].location, DuplicateLocation::Queue); + assert!(by_name[0].by_name && !by_name[0].by_digest); + + // Digest match finds a history row under a different name. + let by_digest = db + .find_duplicate_candidates("Renamed.Upload", Some(&nzb_digest(b"finished"))) + .unwrap(); + assert_eq!(by_digest.len(), 1); + assert_eq!(by_digest[0].id, "done-job"); + assert_eq!(by_digest[0].location, DuplicateLocation::History); + assert!(!by_digest[0].by_name && by_digest[0].by_digest); + + // The queued payload's digest also matches the queue row by content. + let both = db + .find_duplicate_candidates("Show.S01E01", Some(&digest)) + .unwrap(); + assert_eq!(both.len(), 1); + assert!(both[0].by_name && both[0].by_digest); + + assert!( + db.find_duplicate_candidates("Nothing.Here", Some(&nzb_digest(b"new"))) + .unwrap() + .is_empty() + ); + } + + #[test] + fn duplicate_candidates_seek_indexes_not_scan() { + let db = Database::open_memory().unwrap(); + let mut stmt = db + .conn + .prepare(&format!("EXPLAIN QUERY PLAN {DUPLICATE_CANDIDATES_SQL}")) + .unwrap(); + let plan: Vec = stmt + .query_map(params!["x", "sha256:y"], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + for index in [ + "idx_queue_name_lower", + "idx_queue_nzb_digest", + "idx_history_name_lower", + "idx_history_nzb_digest", + ] { + assert!( + plan.iter().any(|step| step.contains(index)), + "{index} unused: {plan:?}" + ); + } + assert!( + !plan + .iter() + .any(|step| step.starts_with("SCAN queue") || step.starts_with("SCAN history")), + "duplicate lookup scans a table: {plan:?}" + ); + } + + #[test] + fn duplicate_lookup_stays_fast_with_large_history() { + // A file database, like production: 2,000 history rows each carrying + // a 256 KiB NZB blob inline (~500 MB). The lookup must not read them. + let dir = tempfile::tempdir().unwrap(); + let db = Database::open(&dir.path().join("dup.db")).unwrap(); + let blob = vec![b'x'; 256 * 1024]; + db.conn.execute_batch("PRAGMA synchronous = OFF;").unwrap(); + for i in 0..2000 { + let mut entry = make_history(&format!("h{i}"), &format!("Release.{i}")); + let mut data = blob.clone(); + data.extend_from_slice(i.to_string().as_bytes()); + entry.nzb_data = Some(data); + db.history_insert(&entry).unwrap(); + } + + let start = std::time::Instant::now(); + let found = db + .find_duplicate_candidates("release.1999", Some(&nzb_digest(b"new"))) + .unwrap(); + let elapsed = start.elapsed(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].id, "h1999"); + assert!( + elapsed < std::time::Duration::from_millis(200), + "duplicate lookup took {elapsed:?}" + ); + } + + #[test] + fn nzb_digest_is_sha256_hex_with_prefix() { + assert_eq!( + nzb_digest(b""), + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } + fn make_rss_item(id: &str, feed: &str, title: &str) -> RssItem { RssItem { id: id.into(), @@ -1736,6 +2870,34 @@ mod tests { assert_eq!(level.post_processing, Some(1)); } + #[test] + fn history_round_trips_par2_volume_counts() { + let db = Database::open_memory().unwrap(); + let mut stored = make_history("par2-vols", "Vols"); + stored.par2_volumes_fetched = Some(2); + stored.par2_volumes_available = Some(5); + db.history_insert(&stored).unwrap(); + db.history_insert(&make_history("par2-legacy", "Legacy")) + .unwrap(); + + let got = db.history_get("par2-vols").unwrap().unwrap(); + assert_eq!( + (got.par2_volumes_fetched, got.par2_volumes_available), + (Some(2), Some(5)) + ); + let listed = db.history_list(10).unwrap(); + let vols = listed.iter().find(|entry| entry.id == "par2-vols").unwrap(); + assert_eq!( + (vols.par2_volumes_fetched, vols.par2_volumes_available), + (Some(2), Some(5)) + ); + let legacy = listed + .iter() + .find(|entry| entry.id == "par2-legacy") + .unwrap(); + assert_eq!(legacy.par2_volumes_available, None); + } + #[test] fn typed_failure_migration_backfills_only_failed_history() { let directory = tempfile::tempdir().unwrap(); @@ -2132,6 +3294,207 @@ mod tests { ); } + /// A trashed row disappears from every history read, keeps its blobs for + /// the undo, and comes back unchanged when it is restored. + #[test] + fn history_trash_hides_the_row_and_restore_brings_it_back() { + let db = Database::open_memory().unwrap(); + let mut entry = make_history("undo-1", "Undo.Me"); + entry.nzb_data = Some(b"".to_vec()); + entry.retry_data = Some(b"retry".to_vec()); + db.history_insert(&entry).unwrap(); + db.history_insert(&make_history("other", "Other")).unwrap(); + + let now = Utc::now(); + let trashed = db + .history_trash( + &["undo-1".into(), "missing".into()], + now + chrono::Duration::seconds(10), + ) + .unwrap(); + assert_eq!(trashed.len(), 1, "only the existing row is reported"); + assert_eq!(trashed[0].id, "undo-1"); + assert_eq!(db.history_count().unwrap(), 1); + + assert!(db.history_get("undo-1").unwrap().is_none()); + assert!( + db.history_get_listed(&["undo-1".into()]) + .unwrap() + .is_empty() + ); + assert!( + db.history_list(10) + .unwrap() + .iter() + .all(|e| e.id != "undo-1") + ); + let (page, total) = db.history_page(&HistoryFilter::default(), 0, 50).unwrap(); + assert_eq!(total, 1); + assert!(page.iter().all(|e| e.id != "undo-1")); + assert!( + !db.history_categories().unwrap().is_empty(), + "another row still supplies the category" + ); + assert!( + db.history_summaries(&HistoryFilter::default()) + .unwrap() + .iter() + .all(|e| e.id != "undo-1") + ); + assert!( + db.find_duplicate_candidates("Undo.Me", None) + .unwrap() + .is_empty(), + "a trashed row must not block a re-add" + ); + + // The blobs survive the trash, which is what makes undo lossless. + assert_eq!( + db.history_get_nzb_data("undo-1").unwrap().as_deref(), + Some(&b""[..]) + ); + assert_eq!( + db.history_get_retry_data("undo-1").unwrap().as_deref(), + Some(&b"retry"[..]) + ); + assert_eq!(db.history_trashed_ids().unwrap(), vec!["undo-1"]); + + let restored = db + .history_restore(&["undo-1".into(), "missing".into()], now) + .unwrap(); + assert_eq!(restored, vec!["undo-1"]); + assert_eq!(db.history_count().unwrap(), 2); + let got = db.history_get("undo-1").unwrap().unwrap(); + assert_eq!(got.name, "Undo.Me"); + assert_eq!( + db.find_duplicate_candidates("Undo.Me", None).unwrap().len(), + 1 + ); + assert!(db.history_trashed_ids().unwrap().is_empty()); + } + + /// The deadline is the whole contract: a restore is accepted up to it and + /// refused after it, and only a purge removes the expired row. + #[test] + fn history_restore_is_refused_after_the_deadline_until_a_purge() { + let db = Database::open_memory().unwrap(); + db.history_insert(&make_history("expiring", "Expiring")) + .unwrap(); + let trashed_at = Utc::now(); + let deadline = trashed_at + chrono::Duration::seconds(10); + db.history_trash(&["expiring".into()], deadline).unwrap(); + + let after = deadline + chrono::Duration::seconds(1); + assert!( + db.history_restore(&["expiring".into()], after) + .unwrap() + .is_empty() + ); + assert_eq!(db.history_count().unwrap(), 0, "still hidden from reads"); + + // Not yet expired, and not purged by a sweep that runs before the + // deadline, so the row is still physically there. + assert!(db.history_purge_expired(trashed_at).unwrap().is_empty()); + assert_eq!(db.history_trashed_ids().unwrap(), vec!["expiring"]); + + let purged = db.history_purge_expired(after).unwrap(); + assert_eq!(purged.len(), 1); + assert_eq!(purged[0].id, "expiring"); + assert!(db.history_get_nzb_data("expiring").unwrap().is_none()); + assert!(db.history_trashed_ids().unwrap().is_empty()); + } + + /// Expired rows are purged even when the deadline is already in the past + /// when the sweep runs, and the job's ledger rows go with them. + #[test] + fn history_purge_removes_the_row_and_its_ledger_entries() { + let db = Database::open_memory().unwrap(); + db.history_insert(&make_history("trash-ledger", "Ledger")) + .unwrap(); + note_present(&db, "trash-ledger", 0); + db.history_trash( + &["trash-ledger".into()], + Utc::now() - chrono::Duration::seconds(1), + ) + .unwrap(); + + assert_eq!(db.history_purge_expired(Utc::now()).unwrap().len(), 1); + assert_eq!(ledger_rows(&db, "trash-ledger"), 0); + } + + /// Retention counts only visible rows: a trashed row must not consume a + /// slot, and must not be deleted by the retention pass either. + #[test] + fn history_enforce_retention_ignores_trashed_rows() { + let db = Database::open_memory().unwrap(); + for i in 0..3 { + db.history_insert(&make_history(&format!("keep-{i}"), &format!("Keep {i}"))) + .unwrap(); + } + db.history_insert(&make_history("trashed", "Trashed")) + .unwrap(); + let future = Utc::now() + chrono::Duration::seconds(10); + db.history_trash(&["trashed".into()], future).unwrap(); + + db.history_enforce_retention(3).unwrap(); + assert_eq!(db.history_count().unwrap(), 3); + assert_eq!( + db.history_trashed_ids().unwrap(), + vec!["trashed"], + "retention must leave the undo window to the purge deadline" + ); + + // Restoring after a retention pass still works while the window is open. + db.history_restore(&["trashed".into()], Utc::now()).unwrap(); + assert_eq!(db.history_count().unwrap(), 4); + } + + /// Applying the schema is one transaction, not one per migration. + /// + /// Each migration used to commit separately, and every commit is an fsync. + /// On a contended volume those serialise: 19 commits were enough to keep + /// the backend from ever binding its port inside the e2e health timeout. + /// The version row must therefore appear only once the whole batch has + /// succeeded, and a re-open of an up-to-date database must write nothing. + #[test] + fn fresh_database_migrates_in_one_transaction() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("queue.db"); + let db = Database::open(&path).unwrap(); + + let version: u32 = db + .conn + .query_row("SELECT MAX(version) FROM schema_version", [], |row| { + row.get(0) + }) + .unwrap(); + assert!(version >= 21, "schema should reach v21, got {version}"); + + // Every migration's objects are present, which is only true if the + // single batch ran to the end before stamping the version. + for table in ["queue", "history", "settings", "history_list_meta"] { + let exists: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "{table} should exist after migrating"); + } + + // Re-opening an up-to-date database has no migration to apply, so it + // must not touch schema_version at all. + drop(db); + let reopened = Database::open(&path).unwrap(); + let versions: i64 = reopened + .conn + .query_row("SELECT COUNT(*) FROM schema_version", [], |row| row.get(0)) + .unwrap(); + assert_eq!(versions, 1, "the version row is written exactly once"); + } + /// A lock held briefly by another connection (backup tool, second /// process) must be waited on, not turned into an immediate SQLITE_BUSY. #[test] @@ -2370,4 +3733,635 @@ mod tests { let db = Database::open_memory().unwrap(); assert_eq!(db.get_setting("nonexistent"), None); } + + fn seeded_history_db() -> (Database, Vec) { + let db = Database::open_memory().unwrap(); + let base = Utc::now(); + for i in 0..40u32 { + let mut entry = make_history(&format!("h{i:02}"), &format!("Show.S01E{i:02}")); + if i % 3 == 0 { + entry.name = format!("Movie.Ünïcode.{i}"); + } + entry.category = ["movies", "tv", ""][i as usize % 3].into(); + if i % 4 == 0 { + entry.status = JobStatus::Failed; + entry.failure_code = Some(JobFailureCode::ArchivePasswordRequired); + entry.error_message = Some("Password required: archive".into()); + } + entry.completed_at = base - chrono::Duration::hours(i64::from(i) * 7); + entry.added_at = entry.completed_at - chrono::Duration::minutes(5); + if i % 5 == 0 { + entry.nzb_data = Some(vec![7u8; 64 * 1024]); + entry.retry_data = Some(vec![9u8; 64 * 1024]); + } + db.history_insert(&entry).unwrap(); + } + let all = db.history_list(usize::MAX).unwrap(); + (db, all) + } + + #[test] + fn history_page_matches_the_in_memory_filter() { + let (db, all) = seeded_history_db(); + let now = Utc::now(); + let filters = [ + HistoryFilter::default(), + HistoryFilter { + completed_since: Some(now - chrono::Duration::days(3)), + ..Default::default() + }, + HistoryFilter { + status: Some(JobStatus::Failed), + ..Default::default() + }, + HistoryFilter { + category: Some("tv".into()), + search: Some("s01e1".into()), + ..Default::default() + }, + HistoryFilter { + completed_since: Some(now - chrono::Duration::days(7)), + status: Some(JobStatus::Completed), + category: Some("movies".into()), + search: Some("MOVIE".into()), + }, + ]; + for filter in &filters { + let expected: Vec<&HistoryEntry> = all + .iter() + .filter(|e| filter.completed_since.is_none_or(|c| e.completed_at >= c)) + .filter(|e| filter.status.is_none_or(|s| e.status == s)) + .filter(|e| filter.category.as_ref().is_none_or(|c| &e.category == c)) + .filter(|e| { + filter + .search + .as_ref() + .is_none_or(|s| e.name.to_lowercase().contains(&s.to_lowercase())) + }) + .collect(); + for (offset, limit) in [(0, 50), (0, 3), (2, 5), (100, 10)] { + let (page, total) = db.history_page(filter, offset, limit).unwrap(); + assert_eq!(total, expected.len(), "{filter:?}"); + let want: Vec<&str> = expected + .iter() + .skip(offset) + .take(limit) + .map(|e| e.id.as_str()) + .collect(); + let got: Vec<&str> = page.iter().map(|e| e.id.as_str()).collect(); + assert_eq!(got, want, "{filter:?} offset {offset} limit {limit}"); + } + let summaries: Vec = db + .history_summaries(filter) + .unwrap() + .into_iter() + .map(|s| s.id) + .collect(); + let want: Vec = expected.iter().map(|e| e.id.clone()).collect(); + assert_eq!(summaries, want, "{filter:?}"); + } + assert_eq!(db.history_categories().unwrap(), vec!["movies", "tv"]); + } + + /// The panel's statistics now come from SQL aggregates plus the failed + /// rows' error text. They must equal the fold the endpoint used to run + /// over whole summaries, for every filter the panel sends: the unfiltered + /// window, a days window, a status filter and a category+search filter. + #[test] + fn history_window_stats_match_the_fold_over_summaries() { + let (db, all) = seeded_history_db(); + let now = Utc::now(); + let filters = [ + HistoryFilter::default(), + HistoryFilter { + completed_since: Some(now - chrono::Duration::days(3)), + ..Default::default() + }, + HistoryFilter { + status: Some(JobStatus::Failed), + ..Default::default() + }, + HistoryFilter { + completed_since: Some(now - chrono::Duration::days(7)), + status: Some(JobStatus::Completed), + category: Some("movies".into()), + search: Some("MOVIE".into()), + }, + ]; + for filter in &filters { + let in_window: Vec<&HistoryEntry> = all + .iter() + .filter(|e| filter.completed_since.is_none_or(|c| e.completed_at >= c)) + .filter(|e| filter.status.is_none_or(|s| e.status == s)) + .filter(|e| filter.category.as_ref().is_none_or(|c| &e.category == c)) + .filter(|e| { + filter + .search + .as_ref() + .is_none_or(|s| e.name.to_lowercase().contains(&s.to_lowercase())) + }) + .collect(); + + let completed: Vec<&&HistoryEntry> = in_window + .iter() + .filter(|e| e.status == JobStatus::Completed) + .collect(); + let expected = HistoryWindowStats { + completed: completed.len() as u64, + completed_bytes: completed.iter().map(|e| e.total_bytes).sum(), + completed_duration_ms: completed + .iter() + .map(|e| (e.completed_at - e.added_at).num_milliseconds()) + .sum(), + failed: in_window + .iter() + .filter(|e| e.status == JobStatus::Failed) + .count() as u64, + }; + assert_eq!( + db.history_window_stats(filter).unwrap(), + expected, + "{filter:?}" + ); + + // Failure reasons are grouped from the messages, newest first. + let want_messages: Vec> = { + let mut failed: Vec<&&HistoryEntry> = in_window + .iter() + .filter(|e| e.status == JobStatus::Failed) + .collect(); + failed.sort_by_key(|e| std::cmp::Reverse(e.completed_at)); + failed.iter().map(|e| e.error_message.clone()).collect() + }; + assert_eq!( + db.history_failure_messages(filter).unwrap(), + want_messages, + "{filter:?}" + ); + } + } + + /// Reason text that differs only after the first clause must group the way + /// the old Rust fold grouped it: split on the first `.` or `:`, keep 32 + /// chars, and treat a missing message as `unknown`. + #[test] + fn history_window_stats_aggregate_edge_case_statuses_and_messages() { + let db = Database::open_memory().unwrap(); + let now = Utc::now(); + let cases: [(&str, JobStatus, Option<&str>); 5] = [ + ("a", JobStatus::Completed, None), + ("b", JobStatus::Completed, None), + ("c", JobStatus::Failed, Some("CRC mismatch: volume 2")), + ("d", JobStatus::Failed, Some("CRC mismatch. volume 3")), + ("e", JobStatus::Failed, None), + ]; + for (i, (id, status, message)) in cases.iter().enumerate() { + let mut entry = make_history(id, id); + entry.status = *status; + if *status == JobStatus::Failed { + entry.failure_code = Some(JobFailureCode::ArchiveInvalid); + } + entry.total_bytes = 1_000; + entry.completed_at = now - chrono::Duration::minutes(i as i64); + entry.added_at = entry.completed_at - chrono::Duration::seconds(30); + entry.error_message = message.map(str::to_owned); + db.history_insert(&entry).unwrap(); + } + + let stats = db.history_window_stats(&HistoryFilter::default()).unwrap(); + assert_eq!( + stats, + HistoryWindowStats { + completed: 2, + completed_bytes: 2_000, + completed_duration_ms: 60_000, + failed: 3, + } + ); + // Newest first, and a NULL message survives as NULL so the caller can + // render its `unknown` fallback. + assert_eq!( + db.history_failure_messages(&HistoryFilter::default()) + .unwrap(), + vec![ + Some("CRC mismatch: volume 2".into()), + Some("CRC mismatch. volume 3".into()), + None, + ] + ); + + // A trashed row is in neither aggregate nor the message list. + db.history_trash(&["c".to_string()], now + chrono::Duration::seconds(5)) + .unwrap(); + let stats = db.history_window_stats(&HistoryFilter::default()).unwrap(); + assert_eq!((stats.completed, stats.failed), (2, 2)); + assert_eq!( + db.history_failure_messages(&HistoryFilter::default()) + .unwrap(), + vec![Some("CRC mismatch. volume 3".into()), None] + ); + } + + /// The history panel's aggregates, category list and page count must all + /// run off an index and never `SCAN history`, so a cold load neither + /// walks the wide table nor reaches its overflow pages, where the stored + /// NZB blobs live. The panel sends two windows: the whole history (the + /// aggregates and page) and a days window. + /// + /// The category list is deliberately unfiltered — the panel offers every + /// category in the history — and a days filter on it would have no index + /// to scan in date order, so it is asserted once, outside the window loop. + #[test] + fn history_panel_aggregates_use_covering_indexes() { + let (db, _) = seeded_history_db(); + let filters = [ + HistoryFilter::default(), + HistoryFilter { + completed_since: Some(Utc::now() - chrono::Duration::days(3)), + ..Default::default() + }, + ]; + for filter in &filters { + let (where_sql, args) = filter.to_sql(); + // Without a date bound the count is answered by the implicit + // unique index on `history.id`; a days window narrows it through + // the statistics' covering index instead. + let count_index = if filter.completed_since.is_some() { + "idx_h_stats_cover" + } else { + "sqlite_autoindex_history_1" + }; + let statements = [ + ( + "window stats", + history_window_stats_sql(&where_sql), + "idx_h_stats_cover", + ), + ( + "failure messages", + history_failure_messages_sql(&where_sql), + "idx_h_fail_cover", + ), + ( + "page count", + format!( + "SELECT COUNT(*) FROM history h{}", + with_condition( + &where_sql, + "h.id NOT IN (SELECT id FROM history_list_meta WHERE trash = 1)" + ) + ), + count_index, + ), + ]; + for (label, sql, index) in &statements { + let mut stmt = db + .conn + .prepare(&format!("EXPLAIN QUERY PLAN {sql}")) + .unwrap(); + let bound: Vec<&dyn rusqlite::ToSql> = + args.iter().map(|a| a as &dyn rusqlite::ToSql).collect(); + let plan: Vec = stmt + .query_map(rusqlite::params_from_iter(bound), |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + let plan = plan.join("\n"); + assert!( + !plan.contains("SCAN history"), + "{label} scans the history table: {sql}\n{plan}" + ); + assert!( + plan.contains(&format!("USING INDEX {index}")) + || plan.contains(&format!("USING COVERING INDEX {index}")), + "{label} does not use {index}: {sql}\n{plan}" + ); + } + } + + let mut stmt = db + .conn + .prepare(&format!("EXPLAIN QUERY PLAN {}", history_categories_sql())) + .unwrap(); + let plan: Vec = stmt + .query_map([], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + let plan = plan.join("\n"); + assert!( + plan.contains("USING COVERING INDEX idx_h_cat_cover"), + "categories do not use idx_h_cat_cover\n{plan}" + ); + } + + #[test] + fn history_list_rows_flag_stored_nzbs_and_keep_full_fields() { + let (db, all) = seeded_history_db(); + let (page, _) = db.history_page(&HistoryFilter::default(), 0, 100).unwrap(); + assert_eq!(page.len(), all.len()); + for entry in &page { + let has_nzb = entry.id[1..].parse::().unwrap() % 5 == 0; + assert_eq!(entry.nzb_data.is_some(), has_nzb, "{}", entry.id); + assert!(entry.nzb_data.as_ref().is_none_or(Vec::is_empty)); + assert!(entry.retry_data.is_none()); + assert_eq!(entry.stages.len(), 1); + } + let ids = vec!["h05".to_string(), "missing".to_string(), "h01".to_string()]; + let listed = db.history_get_listed(&ids).unwrap(); + let got: Vec<&str> = listed.iter().map(|e| e.id.as_str()).collect(); + assert_eq!(got, ["h05", "h01"]); + assert!(listed[0].nzb_data.is_some()); + } + + /// A list page must never load `nzb_data` or `retry_data` content: the + /// only access to `nzb_data` is the header-only `typeof()` probe + /// (OP_Column with OPFLAG_TYPEOFARG), and `retry_data` is not touched. + #[test] + fn history_page_query_does_not_read_blob_columns() { + const OPFLAG_TYPEOFARG: i64 = 0x80; + let (db, _) = seeded_history_db(); + let column_index = |name: &str| -> i64 { + db.conn + .query_row( + "SELECT cid FROM pragma_table_info('history') WHERE name = ?1", + params![name], + |row| row.get(0), + ) + .unwrap() + }; + let nzb_col = column_index("nzb_data"); + let retry_col = column_index("retry_data"); + let root: i64 = db + .conn + .query_row( + "SELECT rootpage FROM sqlite_master WHERE type = 'table' AND name = 'history'", + [], + |row| row.get(0), + ) + .unwrap(); + + let filters = [ + HistoryFilter::default(), + HistoryFilter { + completed_since: Some(Utc::now()), + status: Some(JobStatus::Failed), + category: Some("tv".into()), + search: Some("x".into()), + }, + ]; + for filter in &filters { + let (where_sql, args) = filter.to_sql(); + let sql = format!("EXPLAIN {}", history_page_sql(&where_sql, args.len())); + let mut stmt = db.conn.prepare(&sql).unwrap(); + let ops: Vec<(String, i64, i64, i64)> = stmt + .query_map( + rusqlite::params_from_iter( + args.iter() + .map(|a| a.as_ref()) + .chain([&1i64 as &dyn rusqlite::ToSql, &0i64]), + ), + |row| Ok((row.get(1)?, row.get(2)?, row.get(3)?, row.get(6)?)), + ) + .unwrap() + .collect::>() + .unwrap(); + let table_cursors: Vec = ops + .iter() + .filter(|(op, _, p2, _)| op == "OpenRead" && *p2 == root) + .map(|(_, p1, _, _)| *p1) + .collect(); + assert!(!table_cursors.is_empty()); + let mut probed_nzb = false; + for (op, p1, p2, p5) in &ops { + if op != "Column" || !table_cursors.contains(p1) { + continue; + } + assert_ne!(*p2, retry_col, "list page reads retry_data"); + assert!( + *p2 <= nzb_col, + "list page reads history column {p2}, stored after nzb_data" + ); + if *p2 == nzb_col { + assert_ne!(p5 & OPFLAG_TYPEOFARG, 0, "list page loads nzb_data"); + probed_nzb = true; + } + } + assert!(probed_nzb, "has_nzb probe missing from the plan"); + } + } + + #[test] + fn history_client_removal_marks_the_row_until_it_is_deleted() { + let db = Database::open_memory().unwrap(); + db.history_insert(&make_history("kept", "Kept")).unwrap(); + db.history_insert(&make_history("plain", "Plain")).unwrap(); + let marked = db + .history_mark_client_removed( + &["kept".to_string(), "missing".to_string()], + "Sonarr", + Utc::now(), + true, + ) + .unwrap(); + assert_eq!(marked, 1, "an unknown id is skipped"); + + let removal = db.history_get("kept").unwrap().unwrap().client_removal; + assert_eq!( + removal.as_ref().map(|r| r.removed_by.as_str()), + Some("Sonarr") + ); + assert!(removal.unwrap().files_deleted); + let listed = db.history_list(10).unwrap(); + assert_eq!(listed.len(), 2); + assert!( + listed + .iter() + .any(|e| e.id == "kept" && e.client_removal.is_some()) + ); + assert!( + listed + .iter() + .any(|e| e.id == "plain" && e.client_removal.is_none()) + ); + assert_eq!( + db.history_client_removed_ids().unwrap(), + std::collections::HashSet::from(["kept".to_string()]) + ); + + // A UI delete still goes through the undo window, then drops the + // marker with the row. + let trashed = db + .history_trash( + &["kept".to_string()], + Utc::now() + chrono::Duration::hours(1), + ) + .unwrap(); + assert_eq!(trashed.len(), 1); + assert!(db.history_get("kept").unwrap().is_none()); + db.history_remove("kept").unwrap(); + assert!(db.history_client_removed_ids().unwrap().is_empty()); + } + + #[test] + fn history_list_meta_follows_inserts_deletes_and_retention() { + let (db, _) = seeded_history_db(); + let meta_count = |db: &Database| -> i64 { + db.conn + .query_row("SELECT COUNT(*) FROM history_list_meta", [], |row| { + row.get(0) + }) + .unwrap() + }; + let history_count: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM history", [], |row| row.get(0)) + .unwrap(); + assert_eq!(meta_count(&db), history_count); + + let listed = db.history_list(1).unwrap(); + let first = listed[0].clone(); + let full = db.history_get(&first.id).unwrap().unwrap(); + assert_eq!(full.failure_code, first.failure_code); + assert_eq!(full.download_time_secs, first.download_time_secs); + + db.history_remove(&first.id).unwrap(); + assert_eq!(meta_count(&db), history_count - 1); + db.history_enforce_retention(2).unwrap(); + assert_eq!(meta_count(&db), 2); + db.history_clear().unwrap(); + assert_eq!(meta_count(&db), 0); + } + + // --- Damage-ledger pruning (WI-1868) --- + + use crate::damage_ledger::{ArticleKey, Outcome}; + + fn ledger_t(secs: i64) -> chrono::DateTime { + chrono::DateTime::from_timestamp(1_700_000_000 + secs, 0).unwrap() + } + + /// Record a confirmed hole for `scope`, confirmed at `ledger_t(2)`, so it + /// expires at `ledger_t(2) + HOLE_TTL_SECS`. + fn note_hole(db: &Database, scope: &str, segment: u32) { + let key = ArticleKey::new(scope, 0, segment); + let srv = vec!["a".to_string()]; + let refused = Outcome::Refused { server: "a".into() }; + db.ledger_note(&key, "", &refused, &srv, ledger_t(0)) + .unwrap(); + db.ledger_note(&key, "", &refused, &srv, ledger_t(2)) + .unwrap(); + } + + fn note_present(db: &Database, scope: &str, segment: u32) { + let key = ArticleKey::new(scope, 0, segment); + db.ledger_note( + &key, + "", + &Outcome::Present, + &["a".to_string()], + ledger_t(0), + ) + .unwrap(); + } + + fn ledger_rows(db: &Database, scope: &str) -> usize { + db.ledger_load_file(scope, 0).unwrap().len() + } + + #[test] + fn history_remove_drops_the_jobs_ledger_rows() { + let db = Database::open_memory().unwrap(); + db.history_insert(&make_history("done-1", "A")).unwrap(); + db.history_insert(&make_history("done-2", "B")).unwrap(); + note_present(&db, "done-1", 0); + note_hole(&db, "done-1", 1); + note_present(&db, "done-2", 0); + + db.history_remove("done-1").unwrap(); + + assert_eq!(ledger_rows(&db, "done-1"), 0); + assert_eq!(ledger_rows(&db, "done-2"), 1, "other jobs keep their rows"); + } + + #[test] + fn rows_of_a_retried_job_survive_deleting_its_history_entry() { + let db = Database::open_memory().unwrap(); + // The failed original sits in history; its retry runs under a new id. + db.history_insert(&make_history("orig", "Release")).unwrap(); + db.queue_insert(&make_job("retry", "Release")).unwrap(); + note_hole(&db, "orig", 0); + note_hole(&db, "retry", 0); + note_present(&db, "retry", 1); + + db.history_remove("orig").unwrap(); + + assert_eq!(ledger_rows(&db, "orig"), 0); + assert_eq!(ledger_rows(&db, "retry"), 2); + } + + #[test] + fn a_job_still_queued_keeps_its_rows_when_its_history_row_goes() { + // A failed job can be in history and the queue at once until the + // queue view is removed. + let db = Database::open_memory().unwrap(); + db.queue_insert(&make_job("both", "R")).unwrap(); + db.history_insert(&make_history("both", "R")).unwrap(); + note_hole(&db, "both", 0); + + db.history_remove("both").unwrap(); + assert_eq!(ledger_rows(&db, "both"), 1); + + db.queue_remove("both").unwrap(); + assert_eq!(db.ledger_prune(ledger_t(3)).unwrap(), 1); + assert_eq!(ledger_rows(&db, "both"), 0); + } + + #[test] + fn history_clear_and_retention_drop_ledger_rows() { + let db = Database::open_memory().unwrap(); + let mut old = make_history("old", "Old"); + old.completed_at = Utc::now() - chrono::Duration::days(2); + db.history_insert(&old).unwrap(); + db.history_insert(&make_history("new", "New")).unwrap(); + db.queue_insert(&make_job("live", "Live")).unwrap(); + for scope in ["old", "new", "live"] { + note_present(&db, scope, 0); + } + + db.history_enforce_retention(1).unwrap(); + assert_eq!(ledger_rows(&db, "old"), 0, "aged out with its history row"); + assert_eq!(ledger_rows(&db, "new"), 1); + + db.history_clear().unwrap(); + assert_eq!(ledger_rows(&db, "new"), 0); + assert_eq!(ledger_rows(&db, "live"), 1, "queued jobs are untouched"); + } + + #[test] + fn prune_drops_expired_holes_only_for_jobs_no_longer_queued() { + let db = Database::open_memory().unwrap(); + db.history_insert(&make_history("hist", "H")).unwrap(); + db.queue_insert(&make_job("live", "L")).unwrap(); + note_hole(&db, "hist", 0); + note_present(&db, "hist", 1); + note_hole(&db, "live", 0); + // A late ledger write for a job already removed everywhere. + note_present(&db, "gone", 0); + + // Before the TTL: only the orphan goes. + let before_expiry = ledger_t(2 + crate::damage_ledger::HOLE_TTL_SECS - 1); + assert_eq!(db.ledger_prune(before_expiry).unwrap(), 1); + assert_eq!(ledger_rows(&db, "gone"), 0); + assert_eq!(ledger_rows(&db, "hist"), 2); + + // At the TTL: the history job's expired hole goes; its present row and + // the queued job's hole stay. + let at_expiry = ledger_t(2 + crate::damage_ledger::HOLE_TTL_SECS); + assert_eq!(db.ledger_prune(at_expiry).unwrap(), 1); + let left = db.ledger_load_file("hist", 0).unwrap(); + assert_eq!(left.len(), 1); + assert_eq!(left[0].key.segment_number, 1); + assert_eq!(ledger_rows(&db, "live"), 1); + } } diff --git a/src/error.rs b/src/error.rs index e95b92c..7fbda62 100644 --- a/src/error.rs +++ b/src/error.rs @@ -14,6 +14,9 @@ pub enum NzbError { #[error("Idempotency key is already bound to another payload")] AdmissionConflict, + #[error("{0}")] + DuplicateNzb(String), + #[error("Server not found: {0}")] ServerNotFound(String), diff --git a/src/lib.rs b/src/lib.rs index 754b7ca..f7242ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,7 @@ pub mod config; +pub mod config_backup; +pub mod config_registry; +pub mod config_store; pub mod damage_ledger; pub mod db; pub mod error; @@ -6,14 +9,19 @@ pub mod error; pub mod groups_db; pub mod models; pub mod nzb_parser; +pub mod overrides; +pub mod par2_match; pub mod path; pub mod sabnzbd_import; +pub mod schedule; pub use config::AppConfig; pub use damage_ledger::{ ArticleKey, ConfirmNeeded, Consult, LedgerRecord, LedgerState, Outcome, server_fingerprint, }; -pub use db::Database; +pub use db::{Database, HistoryFilter, HistorySummary, HistoryWindowStats}; pub use error::{NzbError, Result}; pub use models::*; pub use nzb_nntp; +pub use overrides::{OverrideSet, OverrideSource, OverrideValue}; +pub use par2_match::{CandidateFile, PAR2_HASH_16K_LEN, Par2FileRef, match_files_by_hash16k}; diff --git a/src/models.rs b/src/models.rs index 3938d12..914bdf5 100644 --- a/src/models.rs +++ b/src/models.rs @@ -2,12 +2,13 @@ use std::path::PathBuf; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; // --------------------------------------------------------------------------- // Job status lifecycle // --------------------------------------------------------------------------- -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "snake_case")] pub enum JobStatus { Queued, @@ -41,7 +42,7 @@ impl std::fmt::Display for JobStatus { /// /// These carry stable snake_case wire values so consumers key off a typed code /// instead of parsing human-readable diagnostic prose. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "snake_case")] pub enum JobFailureCode { ArticlesUnavailable, @@ -85,7 +86,7 @@ impl std::str::FromStr for JobFailureCode { // Priority // --------------------------------------------------------------------------- -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, ToSchema)] pub enum Priority { Low = 0, #[default] @@ -131,7 +132,7 @@ impl<'de> Deserialize<'de> for Priority { // --------------------------------------------------------------------------- /// Per-server article download statistics for a job. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] pub struct ServerArticleStats { pub server_id: String, pub server_name: String, @@ -140,8 +141,35 @@ pub struct ServerArticleStats { pub bytes_downloaded: u64, } +/// The verdict for one PAR2 slice, as the download engine left it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SliceVerdict { + /// Not fully written, not checked, or its only gap is an article whose + /// absence was not confirmed by every server. + Unknown, + /// Fully written and its MD5 and CRC32 match the PAR2 checksums. + Verified, + /// A checksum mismatched, or an article overlapping it is confirmed + /// missing on every server. + Damaged, +} + +/// The per-job slice map the download engine emits with `JobFinished` +/// (WI-1228), read by the hopeless check (WI-145) and the verify skip +/// (WI-147). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +pub struct SliceMapSummary { + /// One verdict per global slice, in PAR2 Main-packet file order. + pub states: Vec, + /// Recovery slices carried by the PAR2 file the map was built from. + pub recovery_block_count: u32, + pub damaged: usize, + pub unknown: usize, +} + /// A complete download job (parsed from one NZB file). -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct NzbJob { /// Unique job identifier pub id: String, @@ -172,8 +200,10 @@ pub struct NzbJob { /// When the job completed (if applicable) pub completed_at: Option>, /// Working directory for this job (incomplete) + #[schema(value_type = String)] pub work_dir: PathBuf, /// Final output directory + #[schema(value_type = String)] pub output_dir: PathBuf, /// Optional password for extraction pub password: Option, @@ -195,13 +225,23 @@ pub struct NzbJob { /// "+Delete"); `None` or `Some(true)` deletes them, the default. #[serde(default)] pub delete_archives: Option, + /// In-flight PAR2 slice verdicts from the download phase (WI-144). Set + /// when the download ends, if the job's PAR2 index arrived during it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub slice_map: Option, + /// Bytes of PAR2 recovery volumes held back by `par2.defer_volumes` and + /// not yet released. Queue sizes and ETAs leave them out, as SABnzbd + /// does; a release grows the job's size by the volumes it lets through. + /// Filled in by the queue snapshot, never persisted. + #[serde(default)] + pub held_par2_bytes: u64, /// Files in this job #[serde(skip)] pub files: Vec, } /// A single file within an NZB job (collection of NNTP articles). -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct NzbFile { /// Unique file identifier pub id: String, @@ -236,7 +276,7 @@ pub use nzb_nntp::Article; // --------------------------------------------------------------------------- /// A history entry for a completed or failed job. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct HistoryEntry { pub id: String, pub name: String, @@ -249,6 +289,7 @@ pub struct HistoryEntry { /// Active download duration, excluding queue wait and pauses. #[serde(default)] pub download_time_secs: Option, + #[schema(value_type = String)] pub output_dir: PathBuf, /// Post-processing stages with results pub stages: Vec, @@ -267,6 +308,19 @@ pub struct HistoryEntry { /// Per-server download statistics #[serde(default)] pub server_stats: Vec, + /// PAR2 recovery volumes downloaded, out of + /// [`par2_volumes_available`](Self::par2_volumes_available) in the NZB. + /// Fewer than available when `par2.defer_volumes` left some unfetched. + /// `None` for rows recorded before it was stored. + #[serde(default)] + pub par2_volumes_fetched: Option, + /// PAR2 recovery volumes the NZB carries. + #[serde(default)] + pub par2_volumes_available: Option, + /// Set when an *arr removed the job through the SABnzbd API and the row + /// was kept (`general.keep_history_on_arr_delete`). + #[serde(default)] + pub client_removal: Option, /// Raw NZB XML data (for retry) #[serde(skip_serializing)] pub nzb_data: Option>, @@ -275,10 +329,21 @@ pub struct HistoryEntry { pub retry_data: Option>, } +/// How an external client (an *arr, through the SABnzbd API) removed a +/// history entry that rustnzb kept. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +pub struct HistoryClientRemoval { + /// Client name, from its `User-Agent` (e.g. `Sonarr`). + pub removed_by: String, + pub removed_at: DateTime, + /// Whether the client asked for the files to be deleted (`del_files=1`). + pub files_deleted: bool, +} + /// Immutable statistics ledger row recorded when a job leaves the queue. /// Unlike history, these compact rows are retained when users clear or prune /// individual history entries so lifetime counters remain meaningful. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct DownloadStatistic { pub job_id: String, pub completed_at: DateTime, @@ -294,7 +359,7 @@ pub struct DownloadStatistic { /// /// This row intentionally outlives queue and history retention so a caller can /// resolve an ambiguous admission response without creating another job. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct QueueAdmission { pub idempotency_key: String, pub payload_digest: String, @@ -302,13 +367,14 @@ pub struct QueueAdmission { pub accepted_at: DateTime, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(tag = "location", rename_all = "snake_case")] pub enum QueueAdmissionState { Queue { status: JobStatus, total_bytes: u64, downloaded_bytes: u64, + #[schema(value_type = String)] output_dir: PathBuf, error_message: Option, }, @@ -317,6 +383,7 @@ pub enum QueueAdmissionState { total_bytes: u64, downloaded_bytes: u64, completed_at: DateTime, + #[schema(value_type = String)] output_dir: PathBuf, error_message: Option, failure_code: Option, @@ -325,7 +392,7 @@ pub enum QueueAdmissionState { } /// Exact observation of a durable admission and its current engine location. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct QueueAdmissionObservation { pub admission: QueueAdmission, pub state: QueueAdmissionState, @@ -337,7 +404,7 @@ pub enum QueueAdmissionOutcome { Existing(QueueAdmission), } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct StageResult { pub name: String, pub status: StageStatus, @@ -345,7 +412,7 @@ pub struct StageResult { pub duration_secs: f64, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "snake_case")] pub enum StageStatus { Success, @@ -358,7 +425,7 @@ pub enum StageStatus { // --------------------------------------------------------------------------- /// A discovered item from an RSS feed, persisted in the database. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct RssItem { /// Feed entry ID (from the RSS feed) pub id: String, @@ -383,7 +450,7 @@ pub struct RssItem { } /// A download rule that automatically enqueues matching RSS feed items. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct RssRule { /// Unique rule identifier pub id: String, @@ -406,7 +473,7 @@ pub struct RssRule { // --------------------------------------------------------------------------- #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct GroupRow { pub id: i64, pub name: String, @@ -423,7 +490,7 @@ pub struct GroupRow { } #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct HeaderRow { pub id: i64, pub group_id: i64, @@ -440,7 +507,7 @@ pub struct HeaderRow { } #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ThreadSummary { pub root_message_id: String, pub subject: String, @@ -452,7 +519,7 @@ pub struct ThreadSummary { } #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ThreadArticle { #[serde(flatten)] pub header: HeaderRow, @@ -460,13 +527,13 @@ pub struct ThreadArticle { } #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, ToSchema)] pub struct MarkReadInput { pub header_ids: Vec, } #[cfg(feature = "groups-db")] -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, ToSchema)] pub struct DownloadSelectedInput { pub message_ids: Vec, pub name: Option, @@ -638,3 +705,25 @@ mod tests { assert_eq!(restored.bytes_downloaded, 75_000_000); } } + +// --------------------------------------------------------------------------- +// Duplicate detection +// --------------------------------------------------------------------------- + +/// Where a duplicate candidate was found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DuplicateLocation { + Queue, + History, +} + +/// A queued or history job that an incoming NZB matches by name (ignoring +/// ASCII case) or by content digest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DuplicateMatch { + pub id: String, + pub name: String, + pub location: DuplicateLocation, + pub by_name: bool, + pub by_digest: bool, +} diff --git a/src/nzb_parser.rs b/src/nzb_parser.rs index 867a2bd..407ed80 100644 --- a/src/nzb_parser.rs +++ b/src/nzb_parser.rs @@ -264,6 +264,8 @@ pub fn parse_nzb(name: &str, data: &[u8]) -> Result { let file_count = files.len(); Ok(NzbJob { + slice_map: None, + held_par2_bytes: 0, id: uuid::Uuid::new_v4().to_string(), name: crate::path::sanitize_job_name(name), category: "Default".into(), diff --git a/src/overrides.rs b/src/overrides.rs new file mode 100644 index 0000000..90d16f9 --- /dev/null +++ b/src/overrides.rs @@ -0,0 +1,230 @@ +//! Values that the command line, the environment or a desktop shell set for +//! the life of one process. +//! +//! An override is applied on top of the file layer to produce the effective +//! configuration. It is never written to `config.toml`, so the file keeps the +//! value the user chose, and the override disappears when its variable does. + +use std::path::PathBuf; + +use crate::config::AppConfig; + +/// Where an override came from, shown to the user as `cli`, `env:NAME` or +/// `desktop`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OverrideSource { + Cli, + Env(&'static str), + Desktop, +} + +impl std::fmt::Display for OverrideSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Cli => f.write_str("cli"), + Self::Env(name) => write!(f, "env:{name}"), + Self::Desktop => f.write_str("desktop"), + } + } +} + +/// One overridden setting and its value. +#[derive(Debug, Clone, PartialEq)] +pub enum OverrideValue { + ListenAddr(String), + Port(u16), + DataDir(PathBuf), + OtelEnabled(bool), + OtelEndpoint(String), + OtelLogsEnabled(bool), + OtelLogsEndpoint(String), + OtelMetricsEnabled(bool), + OtelMetricsEndpoint(String), + OtelServiceName(String), + DavEnabled(bool), +} + +impl OverrideValue { + /// The dotted config key this value overrides, as shown in the API. + pub fn key(&self) -> &'static str { + match self { + Self::ListenAddr(_) => "general.listen_addr", + Self::Port(_) => "general.port", + Self::DataDir(_) => "general.data_dir", + Self::OtelEnabled(_) => "otel.enabled", + Self::OtelEndpoint(_) => "otel.endpoint", + Self::OtelLogsEnabled(_) => "otel.logs_enabled", + Self::OtelLogsEndpoint(_) => "otel.logs_endpoint", + Self::OtelMetricsEnabled(_) => "otel.metrics_enabled", + Self::OtelMetricsEndpoint(_) => "otel.metrics_endpoint", + Self::OtelServiceName(_) => "otel.service_name", + Self::DavEnabled(_) => "dav.enabled", + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Override { + pub value: OverrideValue, + pub source: OverrideSource, +} + +/// The overrides in force for one process. Built once at startup and never +/// changed, because the environment cannot change under a running process. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct OverrideSet { + entries: Vec, +} + +impl OverrideSet { + /// Add an override, replacing any earlier one for the same key. + pub fn set(&mut self, value: OverrideValue, source: OverrideSource) { + let key = value.key(); + self.entries.retain(|o| o.value.key() != key); + self.entries.push(Override { value, source }); + } + + /// The override for a dotted config key, if one is in force. + pub fn get(&self, key: &str) -> Option<&Override> { + self.entries.iter().find(|o| o.value.key() == key) + } + + pub fn iter(&self) -> impl Iterator { + self.entries.iter() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Write every override into `config`. + pub fn apply(&self, config: &mut AppConfig) { + for entry in &self.entries { + match &entry.value { + OverrideValue::ListenAddr(v) => config.general.listen_addr = v.clone(), + OverrideValue::Port(v) => config.general.port = *v, + OverrideValue::DataDir(v) => config.general.data_dir = v.clone(), + OverrideValue::OtelEnabled(v) => config.otel.enabled = *v, + OverrideValue::OtelEndpoint(v) => config.otel.endpoint = v.clone(), + OverrideValue::OtelLogsEnabled(v) => config.otel.logs_enabled = Some(*v), + OverrideValue::OtelLogsEndpoint(v) => config.otel.logs_endpoint = Some(v.clone()), + OverrideValue::OtelMetricsEnabled(v) => config.otel.metrics_enabled = Some(*v), + OverrideValue::OtelMetricsEndpoint(v) => { + config.otel.metrics_endpoint = Some(v.clone()); + } + OverrideValue::OtelServiceName(v) => config.otel.service_name = v.clone(), + OverrideValue::DavEnabled(v) => config.dav.enabled = *v, + } + } + } + + /// Copy `file`'s value into `next` for every overridden key, so a save + /// never writes an override value. A GUI round-trip sends the effective + /// value back, and this drops it again. + pub fn keep_file_values(&self, next: &mut AppConfig, file: &AppConfig) { + for entry in &self.entries { + match &entry.value { + OverrideValue::ListenAddr(_) => { + next.general.listen_addr = file.general.listen_addr.clone(); + } + OverrideValue::Port(_) => next.general.port = file.general.port, + OverrideValue::DataDir(_) => { + next.general.data_dir = file.general.data_dir.clone(); + } + OverrideValue::OtelEnabled(_) => next.otel.enabled = file.otel.enabled, + OverrideValue::OtelEndpoint(_) => { + next.otel.endpoint = file.otel.endpoint.clone(); + } + OverrideValue::OtelLogsEnabled(_) => { + next.otel.logs_enabled = file.otel.logs_enabled; + } + OverrideValue::OtelLogsEndpoint(_) => { + next.otel.logs_endpoint = file.otel.logs_endpoint.clone(); + } + OverrideValue::OtelMetricsEnabled(_) => { + next.otel.metrics_enabled = file.otel.metrics_enabled; + } + OverrideValue::OtelMetricsEndpoint(_) => { + next.otel.metrics_endpoint = file.otel.metrics_endpoint.clone(); + } + OverrideValue::OtelServiceName(_) => { + next.otel.service_name = file.otel.service_name.clone(); + } + OverrideValue::DavEnabled(_) => next.dav.enabled = file.dav.enabled, + } + } + } + + /// The effective configuration: `file` with every override applied. + pub fn overlay(&self, file: &AppConfig) -> AppConfig { + let mut effective = file.clone(); + self.apply(&mut effective); + effective + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn later_override_for_same_key_replaces_earlier() { + let mut set = OverrideSet::default(); + set.set(OverrideValue::Port(8000), OverrideSource::Cli); + set.set( + OverrideValue::Port(8001), + OverrideSource::Env("RUSTNZB_PORT"), + ); + assert_eq!(set.iter().count(), 1); + let entry = set.get("general.port").unwrap(); + assert_eq!(entry.value, OverrideValue::Port(8001)); + assert_eq!(entry.source.to_string(), "env:RUSTNZB_PORT"); + } + + #[test] + fn overlay_leaves_the_file_layer_untouched() { + let mut file = AppConfig::default(); + file.general.port = 9090; + let mut set = OverrideSet::default(); + set.set(OverrideValue::Port(7777), OverrideSource::Cli); + set.set(OverrideValue::DavEnabled(true), OverrideSource::Desktop); + + let effective = set.overlay(&file); + assert_eq!(effective.general.port, 7777); + assert!(effective.dav.enabled); + assert_eq!(file.general.port, 9090); + assert!(!file.dav.enabled); + } + + #[test] + fn keep_file_values_drops_override_values_from_a_save() { + let mut file = AppConfig::default(); + file.general.data_dir = "/srv/file".into(); + file.dav.enabled = false; + let mut set = OverrideSet::default(); + set.set(OverrideValue::DataDir("/data".into()), OverrideSource::Cli); + set.set( + OverrideValue::DavEnabled(true), + OverrideSource::Env("RUSTNZB_DAV_ENABLED"), + ); + + // A GUI save carries the effective values back, plus a real edit. + let mut next = set.overlay(&file); + next.general.speed_limit_bps = 42; + set.keep_file_values(&mut next, &file); + + assert_eq!(next.general.data_dir, PathBuf::from("/srv/file")); + assert!(!next.dav.enabled); + assert_eq!(next.general.speed_limit_bps, 42); + } + + #[test] + fn source_labels_match_the_api_vocabulary() { + assert_eq!(OverrideSource::Cli.to_string(), "cli"); + assert_eq!(OverrideSource::Desktop.to_string(), "desktop"); + assert_eq!( + OverrideSource::Env("OTEL_ENABLED").to_string(), + "env:OTEL_ENABLED" + ); + } +} diff --git a/src/par2_match.rs b/src/par2_match.rs new file mode 100644 index 0000000..51101d0 --- /dev/null +++ b/src/par2_match.rs @@ -0,0 +1,145 @@ +//! Map NZB files to PAR2 files (WI-144). +//! +//! Subject names in an NZB are often obfuscated, so a downloaded file's name +//! cannot be trusted to match the name in the PAR2 recovery set. PAR2 records +//! the MD5 of each file's first 16 KiB (`hash_16k`), which identifies a file +//! from its first article alone. This is the shared home of that matcher; the +//! download engine and the streamer both use it. Inputs are plain data so this +//! crate needs no PAR2 parser. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +/// Bytes covered by a PAR2 `hash_16k`. +pub const PAR2_HASH_16K_LEN: usize = 16 * 1024; + +/// One file as described by a PAR2 FileDesc packet. +#[derive(Debug, Clone)] +pub struct Par2FileRef<'a, P> { + pub id: P, + pub name: &'a str, + pub hash_16k: [u8; 16], + pub size: u64, +} + +/// One NZB file to match. `first_16k_md5` is the MD5 of its first +/// [`PAR2_HASH_16K_LEN`] bytes (or of the whole file if shorter), `None` when +/// those bytes have not arrived yet; `size` is `None` when not yet known. +#[derive(Debug, Clone)] +pub struct CandidateFile<'a, N> { + pub id: N, + pub name: &'a str, + pub first_16k_md5: Option<[u8; 16]>, + pub size: Option, +} + +/// Match NZB files to PAR2 files: exact name first, then `hash_16k` (and size +/// when both sides know it). Each PAR2 file is claimed at most once; files that +/// match nothing are absent from the result. +pub fn match_files_by_hash16k( + par2_files: &[Par2FileRef<'_, P>], + candidates: &[CandidateFile<'_, N>], +) -> HashMap +where + P: Clone + Eq + Hash, + N: Clone + Eq + Hash, +{ + let mut out = HashMap::new(); + let mut claimed: HashSet = HashSet::new(); + + for cand in candidates { + if let Some(pi) = par2_files + .iter() + .position(|p| p.name == cand.name) + .filter(|pi| !claimed.contains(pi)) + { + claimed.insert(pi); + out.insert(cand.id.clone(), par2_files[pi].id.clone()); + } + } + + for cand in candidates { + if out.contains_key(&cand.id) { + continue; + } + let Some(md5) = cand.first_16k_md5 else { + continue; + }; + let found = par2_files.iter().enumerate().position(|(pi, p)| { + !claimed.contains(&pi) && p.hash_16k == md5 && cand.size.is_none_or(|s| s == p.size) + }); + if let Some(pi) = found { + claimed.insert(pi); + out.insert(cand.id.clone(), par2_files[pi].id.clone()); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(id: u8, name: &str, h: u8, size: u64) -> Par2FileRef<'_, u8> { + Par2FileRef { + id, + name, + hash_16k: [h; 16], + size, + } + } + fn c(id: u32, name: &str, h: Option, size: Option) -> CandidateFile<'_, u32> { + CandidateFile { + id, + name, + first_16k_md5: h.map(|h| [h; 16]), + size, + } + } + + #[test] + fn name_match_wins() { + let par2 = [p(1, "a.mkv", 1, 10), p(2, "b.nfo", 2, 5)]; + let cands = [c(10, "b.nfo", None, None), c(11, "a.mkv", Some(9), None)]; + let m = match_files_by_hash16k(&par2, &cands); + assert_eq!(m[&10], 2); + assert_eq!(m[&11], 1); + } + + #[test] + fn obfuscated_names_match_by_hash() { + let par2 = [p(1, "a.mkv", 1, 10), p(2, "b.nfo", 2, 5)]; + let cands = [ + c(10, "x7f3", Some(2), Some(5)), + c(11, "q9z1", Some(1), None), + ]; + let m = match_files_by_hash16k(&par2, &cands); + assert_eq!(m[&10], 2); + assert_eq!(m[&11], 1); + } + + #[test] + fn size_disambiguates_and_each_par2_file_claimed_once() { + // Two PAR2 files share a 16 KiB prefix; size decides. + let par2 = [p(1, "a", 7, 100), p(2, "b", 7, 200)]; + let cands = [ + c(10, "x", Some(7), Some(200)), + c(11, "y", Some(7), Some(100)), + c(12, "z", Some(7), Some(100)), + ]; + let m = match_files_by_hash16k(&par2, &cands); + assert_eq!(m[&10], 2); + assert_eq!(m[&11], 1); + assert!(!m.contains_key(&12)); + } + + #[test] + fn unknown_hash_is_unmatched() { + let par2 = [p(1, "a", 1, 10)]; + let m = match_files_by_hash16k( + &par2, + &[c(10, "x", None, Some(10)), c(11, "y", Some(3), None)], + ); + assert!(m.is_empty()); + } +} diff --git a/src/sabnzbd_import.rs b/src/sabnzbd_import.rs index 0f43088..2f7a213 100644 --- a/src/sabnzbd_import.rs +++ b/src/sabnzbd_import.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use crate::config::{CategoryConfig, RssFeedConfig, ServerConfig}; @@ -14,7 +15,7 @@ use crate::config::{CategoryConfig, RssFeedConfig, ServerConfig}; // --------------------------------------------------------------------------- /// Preview returned by both INI and API import — same shape for both. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct SabnzbdImportPreview { pub servers: Vec, pub categories: Vec, @@ -27,7 +28,7 @@ pub struct SabnzbdImportPreview { } /// A server imported from SABnzbd. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ImportedServer { pub name: String, pub host: String, @@ -67,7 +68,7 @@ impl ImportedServer { } /// General settings imported from SABnzbd. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ImportedGeneral { pub api_key: Option, pub complete_dir: Option, diff --git a/src/schedule.rs b/src/schedule.rs new file mode 100644 index 0000000..d2803d4 --- /dev/null +++ b/src/schedule.rs @@ -0,0 +1,537 @@ +//! Time-of-day download schedule. +//! +//! A rule names a set of weekdays and a window, and what should happen inside +//! it: pause downloads, cap the speed, or take news servers out of rotation. +//! Windows may cross midnight; such a window belongs to the day it starts on. +//! +//! Evaluation is a pure function of the rules and a wall-clock time, so the +//! behaviour around midnight and at window edges is testable without a clock, +//! a timer or a running queue. + +use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike, Weekday}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// What an active rule does to the queue. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum ScheduleAction { + /// Pause all downloads. + #[default] + Pause, + /// Apply a speed limit for the window. + SpeedLimit, + /// Take the listed servers out of rotation. + DisableServers, +} + +/// The whole `[schedule]` section. +#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] +pub struct ScheduleConfig { + /// Rules are only evaluated when this is on. Turning it off restores the + /// configured speed limit, resumes a pause the schedule applied, and + /// returns every server to rotation. + pub enabled: bool, + pub rules: Vec, +} + +/// One time-of-day window. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(default, deny_unknown_fields)] +pub struct ScheduleRule { + /// Stable identifier, assigned by the API when a rule is created. + pub id: String, + /// Label shown in the UI. + pub name: String, + /// Weekdays the window starts on: `mon`..`sun` (case-insensitive, full + /// names are accepted too). + pub days: Vec, + /// Window start as `HH:MM`, 24-hour clock. + pub start: String, + /// Window end as `HH:MM`. Earlier than `start` means the window runs past + /// midnight. `24:00` is accepted as the end of the day. + pub end: String, + pub action: ScheduleAction, + /// Speed cap in bytes/sec for `speed_limit`; `0` means unlimited, which + /// explicitly overrides a lower cap from another active rule. + pub speed_limit_bps: u64, + /// Server ids taken out of rotation for `disable_servers`. + pub servers: Vec, +} + +impl Default for ScheduleRule { + fn default() -> Self { + Self { + id: String::new(), + name: String::new(), + days: Vec::new(), + start: "00:00".into(), + end: "24:00".into(), + action: ScheduleAction::Pause, + speed_limit_bps: 0, + servers: Vec::new(), + } + } +} + +/// What the schedule wants right now. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ScheduleDecision { + /// Rules active at the evaluated time, in configuration order. + pub active_rule_ids: Vec, + /// Whether downloads should be paused. Any active pause rule pauses. + pub pause: bool, + /// Effective scheduled cap. `None` means no scheduled cap, so the + /// configured `general.speed_limit_bps` applies. `Some(0)` means an + /// active rule asked for unlimited speed, which overrides both the + /// configured limit and any lower cap from another rule. + pub speed_limit_bps: Option, + /// Server ids to take out of rotation, in configuration order and without + /// duplicates. + pub disabled_servers: Vec, + /// Every open pause window *occurrence*, as `rule id@the date the window + /// started on`, sorted and without duplicates. + /// + /// The occurrences, not a single joined key, are what the queue compares: + /// with two windows overlapping, one closing must not look like a new + /// window opening. Two evaluations a tick apart inside one window share + /// their occurrence, while the next day's window (or a window the user + /// reopened after a manual resume) has a different one. + pub pause_windows: Vec, +} + +impl ScheduleDecision { + /// Nothing to do: no pause, no cap and every server enabled. + pub fn is_idle(&self) -> bool { + !self.pause + && self.speed_limit_bps.is_none() + && self.disabled_servers.is_empty() + && self.pause_windows.is_empty() + } +} + +/// Parse a weekday name. +pub fn parse_weekday(name: &str) -> Option { + let name = name.trim().to_ascii_lowercase(); + Some(match name.as_str() { + "mon" | "monday" => Weekday::Mon, + "tue" | "tues" | "tuesday" => Weekday::Tue, + "wed" | "weds" | "wednesday" => Weekday::Wed, + "thu" | "thur" | "thurs" | "thursday" => Weekday::Thu, + "fri" | "friday" => Weekday::Fri, + "sat" | "saturday" => Weekday::Sat, + "sun" | "sunday" => Weekday::Sun, + _ => return None, + }) +} + +/// Parse `HH:MM` into minutes since midnight. `24:00` is accepted as 1440. +pub fn parse_minutes(value: &str) -> Option { + let (hour, minute) = value.trim().split_once(':')?; + if hour.len() != 2 || minute.len() != 2 { + return None; + } + let hour: u32 = hour.parse().ok()?; + let minute: u32 = minute.parse().ok()?; + if minute > 59 { + return None; + } + match (hour, minute) { + (24, 0) => Some(24 * 60), + (0..=23, _) => Some(hour * 60 + minute), + _ => None, + } +} + +/// Why a rule cannot be evaluated, phrased for the settings page. +pub fn rule_problem(rule: &ScheduleRule) -> Option { + if rule.days.is_empty() { + return Some("pick at least one day".into()); + } + if let Some(day) = rule.days.iter().find(|day| parse_weekday(day).is_none()) { + return Some(format!("`{day}` is not a weekday")); + } + let Some(start) = parse_minutes(&rule.start) else { + return Some(format!("start `{}` is not HH:MM", rule.start)); + }; + let Some(end) = parse_minutes(&rule.end) else { + return Some(format!("end `{}` is not HH:MM", rule.end)); + }; + if start >= 24 * 60 { + return Some("start must be earlier than 24:00".into()); + } + if start == end { + return Some("start and end must differ".into()); + } + if rule.action == ScheduleAction::DisableServers && rule.servers.is_empty() { + return Some("pick at least one server".into()); + } + None +} + +/// The day a rule's window started on, when the rule is active at `now`. +/// +/// `None` means the rule is not active. For a window that crossed midnight +/// the tail belongs to the previous day, so this is the date the window +/// started, not the date being evaluated. +fn active_window_start(rule: &ScheduleRule, now: NaiveDateTime) -> Option { + let (start, end) = (parse_minutes(&rule.start)?, parse_minutes(&rule.end)?); + let days: Vec = rule + .days + .iter() + .filter_map(|day| parse_weekday(day)) + .collect(); + let minute = now.hour() * 60 + now.minute(); + + if start < end { + return (days.contains(&now.weekday()) && minute >= start && minute < end) + .then_some(now.date()); + } + + // The window runs past midnight. The day it belongs to is the start day, + // so the tail belongs to the previous day's entry. + if days.contains(&now.weekday()) && minute >= start { + return Some(now.date()); + } + now.date() + .pred_opt() + .filter(|yesterday| days.contains(&yesterday.weekday()) && minute < end) +} + +/// Evaluate every rule against `now` (local wall-clock time). +/// +/// Invalid rules are skipped: they cannot be saved through the API, and a +/// hand-edited file with one must not take the queue down with it. +pub fn evaluate(rules: &[ScheduleRule], now: NaiveDateTime) -> ScheduleDecision { + let mut decision = ScheduleDecision::default(); + let mut saw_unlimited = false; + let mut pause_windows: Vec = Vec::new(); + + for rule in rules { + if rule_problem(rule).is_some() { + continue; + } + let Some(window_start) = active_window_start(rule, now) else { + continue; + }; + decision.active_rule_ids.push(rule.id.clone()); + match rule.action { + ScheduleAction::Pause => { + decision.pause = true; + pause_windows.push(format!("{}@{}", rule.id, window_start)); + } + ScheduleAction::SpeedLimit => { + if rule.speed_limit_bps == 0 { + saw_unlimited = true; + } else { + decision.speed_limit_bps = Some( + decision + .speed_limit_bps + .map_or(rule.speed_limit_bps, |bps| bps.min(rule.speed_limit_bps)), + ); + } + } + ScheduleAction::DisableServers => { + for server in &rule.servers { + if !decision.disabled_servers.contains(server) { + decision.disabled_servers.push(server.clone()); + } + } + } + } + } + + // An active rule that says "unlimited" wins over a cap: it is an explicit + // statement about this window, not the absence of one. It has to be + // `Some(0)`, not `None` -- `None` means "no scheduled cap", which would + // let the configured `general.speed_limit_bps` through and run the window + // at the user's limit instead of unlimited. + if saw_unlimited { + decision.speed_limit_bps = Some(0); + } + // Sorted so the occurrences do not depend on config order, which the + // settings page may reorder without changing what is open. + pause_windows.sort(); + pause_windows.dedup(); + decision.pause_windows = pause_windows; + decision +} + +/// Evaluate a whole config: a disabled schedule is always idle. +pub fn evaluate_config(config: &ScheduleConfig, now: NaiveDateTime) -> ScheduleDecision { + if !config.enabled { + return ScheduleDecision::default(); + } + evaluate(&config.rules, now) +} + +/// The next minute boundary after `now`, used to align the monitor's wakeups. +pub fn next_minute(now: NaiveTime) -> NaiveTime { + let minute = now.hour() * 60 + now.minute(); + let next = (minute + 1) % (24 * 60); + NaiveTime::from_hms_opt(next / 60, next % 60, 0).expect("valid minute of day") +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::NaiveDate; + + fn at(day: u32, hour: u32, minute: u32) -> NaiveDateTime { + NaiveDate::from_ymd_opt(2026, 10, day) + .unwrap() + .and_hms_opt(hour, minute, 0) + .unwrap() + } + + fn rule(days: &[&str], start: &str, end: &str, action: ScheduleAction) -> ScheduleRule { + ScheduleRule { + id: format!("{start}-{end}"), + name: String::new(), + days: days.iter().map(|d| (*d).to_string()).collect(), + start: start.into(), + end: end.into(), + action, + ..ScheduleRule::default() + } + } + + #[test] + fn parses_clock_times_and_rejects_bad_ones() { + assert_eq!(parse_minutes("00:00"), Some(0)); + assert_eq!(parse_minutes("09:05"), Some(545)); + assert_eq!(parse_minutes("23:59"), Some(1439)); + assert_eq!(parse_minutes("24:00"), Some(1440)); + assert_eq!(parse_minutes("24:01"), None); + assert_eq!(parse_minutes("7:00"), None); + assert_eq!(parse_minutes("07:0"), None); + assert_eq!(parse_minutes("07:60"), None); + assert_eq!(parse_minutes("noon"), None); + } + + #[test] + fn parses_weekday_names_case_insensitively() { + assert_eq!(parse_weekday("Mon"), Some(Weekday::Mon)); + assert_eq!(parse_weekday("thursday"), Some(Weekday::Thu)); + assert_eq!(parse_weekday("sunday "), Some(Weekday::Sun)); + assert_eq!(parse_weekday("funday"), None); + } + + #[test] + fn a_daytime_window_covers_its_start_and_excludes_its_end() { + let rules = [rule(&["tue"], "09:00", "17:00", ScheduleAction::Pause)]; + // 2026-10-20 is a Tuesday. + assert!(!evaluate(&rules, at(20, 8, 59)).pause); + assert!(evaluate(&rules, at(20, 9, 0)).pause); + assert!(evaluate(&rules, at(20, 16, 59)).pause); + assert!(!evaluate(&rules, at(20, 17, 0)).pause); + assert!(!evaluate(&rules, at(21, 12, 0)).pause, "wrong weekday"); + } + + #[test] + fn an_overnight_window_belongs_to_the_day_it_starts_on() { + let rules = [rule(&["mon"], "22:00", "06:00", ScheduleAction::Pause)]; + // Monday 2026-10-19 at 23:00 is inside; Tuesday 05:00 is the tail. + assert!(evaluate(&rules, at(19, 23, 0)).pause); + assert!(evaluate(&rules, at(20, 5, 59)).pause); + assert!(!evaluate(&rules, at(20, 6, 0)).pause, "tail ends at 06:00"); + assert!( + !evaluate(&rules, at(20, 22, 30)).pause, + "Tuesday starts nothing" + ); + assert!(!evaluate(&rules, at(19, 21, 59)).pause); + } + + #[test] + fn a_full_day_window_is_active_all_day() { + let rules = [rule(&["sat"], "00:00", "24:00", ScheduleAction::Pause)]; + assert!(evaluate(&rules, at(24, 0, 0)).pause); + assert!(evaluate(&rules, at(24, 23, 59)).pause); + assert!(!evaluate(&rules, at(25, 0, 0)).pause); + } + + #[test] + fn the_lowest_applied_cap_wins_and_unlimited_overrides_one() { + let mut off_peak = rule(&["tue"], "00:00", "24:00", ScheduleAction::SpeedLimit); + off_peak.id = "off-peak".into(); + off_peak.speed_limit_bps = 1_000_000; + let mut strict = rule(&["tue"], "00:00", "24:00", ScheduleAction::SpeedLimit); + strict.id = "strict".into(); + strict.speed_limit_bps = 500_000; + + let decision = evaluate(&[off_peak.clone(), strict.clone()], at(20, 12, 0)); + assert_eq!(decision.speed_limit_bps, Some(500_000)); + assert_eq!(decision.active_rule_ids, vec!["off-peak", "strict"]); + + let mut unlimited = rule(&["tue"], "00:00", "24:00", ScheduleAction::SpeedLimit); + unlimited.id = "unlimited".into(); + let decision = evaluate(&[off_peak, strict, unlimited], at(20, 12, 0)); + assert_eq!( + decision.speed_limit_bps, + Some(0), + "unlimited is an explicit 0 cap, not the absence of one" + ); + } + + #[test] + fn an_unlimited_rule_is_distinct_from_having_no_cap_rule() { + let mut unlimited = rule(&["tue"], "00:00", "24:00", ScheduleAction::SpeedLimit); + unlimited.id = "unlimited".into(); + assert_eq!( + evaluate(&[unlimited], at(20, 12, 0)).speed_limit_bps, + Some(0) + ); + // With no cap rule at all the configured limit must still apply, so + // this is the case that has to stay `None`. + assert_eq!(evaluate(&[], at(20, 12, 0)).speed_limit_bps, None); + } + + #[test] + fn a_pause_window_carries_an_occurrence_that_changes_with_the_window() { + let rules = [ + rule(&["mon"], "22:00", "06:00", ScheduleAction::Pause), + rule(&["tue"], "22:00", "06:00", ScheduleAction::Pause), + ]; + // Inside Monday's window, before and after midnight: the same + // occurrence, because it belongs to the day it started on. + let monday_late = evaluate(&rules[..1], at(19, 23, 0)).pause_windows; + let tuesday_early = evaluate(&rules[..1], at(20, 5, 0)).pause_windows; + assert_eq!(monday_late, tuesday_early); + assert_eq!(monday_late.len(), 1); + assert!(monday_late[0].contains("2026-10-19"), "{monday_late:?}"); + + // Tuesday's own window is a different occurrence, so the queue sees + // it as a new window and pauses again. + let tuesday_late = evaluate(&rules[1..], at(20, 23, 0)).pause_windows; + assert_ne!(monday_late, tuesday_late); + assert!(tuesday_late[0].contains("2026-10-20"), "{tuesday_late:?}"); + + // Outside every window there is no occurrence at all. + assert!(evaluate(&rules, at(20, 12, 0)).pause_windows.is_empty()); + // A cap rule opens no pause window. + let mut cap = rule(&["tue"], "00:00", "24:00", ScheduleAction::SpeedLimit); + cap.speed_limit_bps = 1_000_000; + assert!(evaluate(&[cap], at(20, 12, 0)).pause_windows.is_empty()); + } + + #[test] + fn overlapping_windows_keep_their_own_occurrences() { + // A 20:00-24:00 and B 22:00-02:00: at 23:00 both are open and each + // contributes its own occurrence. At 01:00 only B is left, so B must + // still be in the new set for the queue to recognise it as the same + // window rather than a new one. + let mut a = rule(&["mon"], "20:00", "24:00", ScheduleAction::Pause); + a.id = "a".into(); + let mut b = rule(&["mon"], "22:00", "02:00", ScheduleAction::Pause); + b.id = "b".into(); + let rules = [a, b]; + + let both = evaluate(&rules, at(19, 23, 0)).pause_windows; + assert_eq!(both, vec!["a@2026-10-19", "b@2026-10-19"]); + let after_a_closes = evaluate(&rules, at(20, 1, 0)).pause_windows; + assert_eq!(after_a_closes, vec!["b@2026-10-19"]); + // The occurrence that is still open is unchanged: a set difference + // says nothing new opened, so the queue keeps its pause rather than + // re-pausing a manual resume. + assert!( + after_a_closes.iter().all(|window| both.contains(window)), + "{after_a_closes:?} vs {both:?}" + ); + } + + #[test] + fn the_window_occurrence_does_not_depend_on_rule_order() { + let mut first = rule(&["tue"], "00:00", "24:00", ScheduleAction::Pause); + first.id = "a".into(); + let mut second = rule(&["tue"], "00:00", "24:00", ScheduleAction::Pause); + second.id = "b".into(); + // Reordering the rules in settings must not look like a new window. + assert_eq!( + evaluate(&[first.clone(), second.clone()], at(20, 12, 0)).pause_windows, + evaluate(&[second, first], at(20, 12, 0)).pause_windows + ); + } + + #[test] + fn a_cap_outside_its_window_does_not_apply() { + let mut cap = rule(&["tue"], "01:00", "05:00", ScheduleAction::SpeedLimit); + cap.speed_limit_bps = 1_000_000; + assert_eq!(evaluate(&[cap], at(20, 12, 0)).speed_limit_bps, None); + } + + #[test] + fn disabled_servers_are_collected_once_in_rule_order() { + let mut first = rule(&["tue"], "00:00", "24:00", ScheduleAction::DisableServers); + first.id = "a".into(); + first.servers = vec!["backup".into(), "primary".into()]; + let mut second = rule(&["tue"], "00:00", "24:00", ScheduleAction::DisableServers); + second.id = "b".into(); + second.servers = vec!["backup".into()]; + + let decision = evaluate(&[first, second], at(20, 12, 0)); + assert_eq!(decision.disabled_servers, vec!["backup", "primary"]); + } + + #[test] + fn a_disabled_schedule_is_idle_whatever_the_clock_says() { + let config = ScheduleConfig { + enabled: false, + rules: vec![rule(&["tue"], "00:00", "24:00", ScheduleAction::Pause)], + }; + assert!(evaluate_config(&config, at(20, 12, 0)).is_idle()); + let config = ScheduleConfig { + enabled: true, + ..config + }; + assert!(!evaluate_config(&config, at(20, 12, 0)).is_idle()); + } + + #[test] + fn invalid_rules_are_skipped_rather_than_applied() { + let broken = ScheduleRule { + id: "broken".into(), + days: vec!["mon".into()], + start: "nope".into(), + end: "06:00".into(), + ..ScheduleRule::default() + }; + assert!(evaluate(&[broken], at(19, 23, 0)).is_idle()); + } + + #[test] + fn validation_names_the_field_at_fault() { + let mut bad_day = rule(&["funday"], "09:00", "17:00", ScheduleAction::Pause); + assert!(rule_problem(&bad_day).unwrap().contains("funday")); + bad_day.days = vec!["mon".into()]; + assert!(rule_problem(&bad_day).is_none()); + + let mut bad_time = bad_day.clone(); + bad_time.start = "9:00".into(); + assert!(rule_problem(&bad_time).unwrap().contains("HH:MM")); + + let mut empty = bad_day.clone(); + empty.days.clear(); + assert!(rule_problem(&empty).unwrap().contains("at least one day")); + + let mut same = bad_day.clone(); + same.end = "09:00".into(); + assert!(rule_problem(&same).unwrap().contains("must differ")); + + let mut no_servers = bad_day.clone(); + no_servers.action = ScheduleAction::DisableServers; + assert!(rule_problem(&no_servers).unwrap().contains("server")); + } + + #[test] + fn next_minute_wraps_at_midnight() { + assert_eq!( + next_minute(NaiveTime::from_hms_opt(10, 15, 30).unwrap()), + NaiveTime::from_hms_opt(10, 16, 0).unwrap() + ); + assert_eq!( + next_minute(NaiveTime::from_hms_opt(23, 59, 10).unwrap()), + NaiveTime::from_hms_opt(0, 0, 0).unwrap() + ); + } +} diff --git a/tests/groups_db.rs b/tests/groups_db.rs new file mode 100644 index 0000000..afa8572 --- /dev/null +++ b/tests/groups_db.rs @@ -0,0 +1,268 @@ +//! Public-surface tests for the newsgroup browsing database (`groups_db`). +//! +//! The module only compiles with the `groups-db` feature, so this whole file +//! is gated on it. Run with `cargo test -p nzb-core --features groups-db`. +#![cfg(feature = "groups-db")] + +use nzb_core::db::Database; +use nzb_nntp::XoverEntry; + +fn entry(article_num: u64, message_id: &str, references: &str, subject: &str) -> XoverEntry { + XoverEntry { + article_num, + subject: subject.into(), + from: "poster".into(), + date: format!("2026-01-01 {article_num:08}"), + message_id: message_id.into(), + references: references.into(), + bytes: 1, + lines: 1, + } +} + +fn group_id(db: &Database, name: &str) -> i64 { + db.group_list(false, Some(name), 10, 0) + .unwrap() + .into_iter() + .find(|g| g.name == name) + .unwrap_or_else(|| panic!("group {name} missing")) + .id +} + +fn seed(db: &Database, names: &[&str]) { + let groups: Vec<(String, u64, u64)> = names + .iter() + .map(|name| ((*name).to_string(), 30, 10)) + .collect(); + db.group_upsert_batch(&groups).unwrap(); +} + +#[test] +fn upsert_refreshes_counts_of_an_existing_group_without_duplicating_it() { + let db = Database::open_memory().unwrap(); + assert_eq!( + db.group_upsert_batch(&[("alt.binaries.test".into(), 30, 10)]) + .unwrap(), + 1 + ); + let first = db.group_list(false, None, 10, 0).unwrap().pop().unwrap(); + assert_eq!(first.article_count, 20); + assert_eq!(first.first_article, 10); + assert_eq!(first.last_article, 30); + + db.group_upsert_batch(&[("alt.binaries.test".into(), 50, 40)]) + .unwrap(); + + let groups = db.group_list(false, None, 10, 0).unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].id, first.id, "upsert must keep the row id"); + assert_eq!(groups[0].article_count, 10); + assert_eq!(groups[0].first_article, 40); + assert_eq!(groups[0].last_article, 50); +} + +#[test] +fn group_list_filters_by_subscription_and_case_insensitive_name() { + let db = Database::open_memory().unwrap(); + seed( + &db, + &["alt.binaries.linux", "alt.binaries.music", "comp.lang.rust"], + ); + db.group_set_subscribed(group_id(&db, "alt.binaries.linux"), true) + .unwrap(); + + let subscribed: Vec = db + .group_list(true, None, 10, 0) + .unwrap() + .into_iter() + .map(|g| g.name) + .collect(); + assert_eq!(subscribed, ["alt.binaries.linux"]); + assert_eq!(db.group_count(true, None).unwrap(), 1); + + let binaries = db.group_list(false, Some("BINARIES"), 10, 0).unwrap(); + assert_eq!(binaries.len(), 2); + assert_eq!(db.group_count(false, Some("BINARIES")).unwrap(), 2); + + let none = db.group_list(false, Some("no.such.group"), 10, 0).unwrap(); + assert!(none.is_empty()); + assert_eq!(db.group_count(false, Some("no.such.group")).unwrap(), 0); +} + +#[test] +fn group_list_pages_in_name_order() { + let db = Database::open_memory().unwrap(); + seed(&db, &["e.g", "a.b", "d.f", "b.c", "c.d"]); + + assert_eq!(db.group_count(false, None).unwrap(), 5); + let page: Vec = db + .group_list(false, None, 2, 2) + .unwrap() + .into_iter() + .map(|g| g.name) + .collect(); + assert_eq!(page, ["c.d", "d.f"]); + + let tail = db.group_list(false, None, 10, 4).unwrap(); + assert_eq!(tail.len(), 1); + assert_eq!(tail[0].name, "e.g"); +} + +#[test] +fn group_subscription_toggles_and_missing_group_reads_as_none() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + + db.group_set_subscribed(id, true).unwrap(); + assert!(db.group_get(id).unwrap().unwrap().subscribed); + db.group_set_subscribed(id, false).unwrap(); + assert!(!db.group_get(id).unwrap().unwrap().subscribed); + + assert!(db.group_get(id + 1_000).unwrap().is_none()); +} + +#[test] +fn scan_watermark_and_its_server_round_trip_and_clear() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + assert_eq!(db.group_scan_server(id).unwrap(), None); + + db.group_update_watermark(id, 42, Some("primary")).unwrap(); + assert_eq!(db.group_get(id).unwrap().unwrap().last_scanned, 42); + assert_eq!( + db.group_scan_server(id).unwrap().as_deref(), + Some("primary") + ); + + db.group_update_watermark(id, 57, None).unwrap(); + assert_eq!(db.group_get(id).unwrap().unwrap().last_scanned, 57); + assert_eq!(db.group_scan_server(id).unwrap(), None); + + assert_eq!(db.group_scan_server(id + 1_000).unwrap(), None); +} + +#[test] +fn mark_all_read_only_touches_the_named_group() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.one", "alt.two"]); + let one = group_id(&db, "alt.one"); + let two = group_id(&db, "alt.two"); + db.header_insert_batch( + one, + &[entry(10, "a@test", "", "A"), entry(11, "b@test", "", "B")], + ) + .unwrap(); + db.header_insert_batch(two, &[entry(10, "c@test", "", "C")]) + .unwrap(); + + assert_eq!(db.header_mark_all_read(one).unwrap(), 2); + assert_eq!(db.header_unread_count(one).unwrap(), 0); + assert_eq!(db.header_unread_count(two).unwrap(), 1); + assert_eq!(db.header_mark_all_read(one).unwrap(), 0); + assert_eq!(db.header_mark_all_read(one + 1_000).unwrap(), 0); +} + +#[test] +fn header_lookup_by_message_id_finds_the_stored_article() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + db.header_insert_batch(id, &[entry(12, "wanted@test", "", "Wanted")]) + .unwrap(); + + let found = db + .header_get_by_message_id("wanted@test") + .unwrap() + .expect("stored header"); + assert_eq!(found.group_id, id); + assert_eq!(found.article_num, 12); + assert_eq!(found.subject, "Wanted"); + assert!(!found.read); + + assert!( + db.header_get_by_message_id("absent@test") + .unwrap() + .is_none() + ); +} + +#[test] +fn header_list_pages_newest_first() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + let entries: Vec = (10..15) + .map(|n| entry(n, &format!("m{n}@test"), "", &format!("S{n}"))) + .collect(); + db.header_insert_batch(id, &entries).unwrap(); + + assert_eq!(db.header_count(id, None).unwrap(), 5); + let page: Vec = db + .header_list(id, None, 2, 1) + .unwrap() + .into_iter() + .map(|h| h.article_num) + .collect(); + assert_eq!(page, [13, 12]); +} + +#[test] +fn thread_list_pages_by_most_recent_reply_with_exact_totals() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + db.header_insert_batch( + id, + &[ + entry(10, "t1@test", "", "Thread one"), + entry(11, "t2@test", "", "Thread two"), + entry(12, "t3@test", "", "Thread three"), + ], + ) + .unwrap(); + + let (first, total) = db.header_list_threads(id, 2, 0).unwrap(); + assert_eq!(total, 3); + assert_eq!(first.len(), 2); + assert_eq!(first[0].root_message_id, "t3@test"); + assert_eq!(first[1].root_message_id, "t2@test"); + + let (last, total) = db.header_list_threads(id, 2, 2).unwrap(); + assert_eq!(total, 3); + assert_eq!(last.len(), 1); + assert_eq!(last[0].root_message_id, "t1@test"); +} + +#[test] +fn thread_article_depth_counts_references() { + let db = Database::open_memory().unwrap(); + seed(&db, &["alt.test"]); + let id = group_id(&db, "alt.test"); + db.header_insert_batch( + id, + &[ + entry(10, "root@test", "", "Root"), + entry(11, "reply@test", "root@test", "Re: Root"), + entry(12, "deep@test", "root@test reply@test", "Re: Re: Root"), + ], + ) + .unwrap(); + + let thread = db.header_get_thread(id, "root@test").unwrap(); + let depths: Vec<(String, i32)> = thread + .iter() + .map(|a| (a.header.message_id.clone(), a.depth)) + .collect(); + assert_eq!( + depths, + [ + ("deep@test".to_string(), 2), + ("reply@test".to_string(), 1), + ("root@test".to_string(), 0), + ] + ); + + assert!(db.header_get_thread(id, "unknown@test").unwrap().is_empty()); +}