From 5d87fb2d7d8118279f0f653392353dda43d044fe Mon Sep 17 00:00:00 2001 From: Charles Gauthereau Date: Sun, 4 Oct 2026 10:09:56 +0200 Subject: [PATCH 01/11] feat(files): files sources with Archive, Snapshots and Sync methods Back up and restore plain directories, alongside the database sources. Files source and Archive (P0) - `files` type: absolute path (not /), options.exclude and options.one_file_system; walk + tar.gz with symlinks kept as links, special files skipped, unreadable entries fail the run. - Mirror restore: archive validated first, then everything not excluded is wiped (mount points and special files kept) and the archive is extracted, retrying without metadata on filesystems refusing chown/chmod. - Windows-safe platform helpers; helm extraVolumes/extraVolumeMounts. Universal rclone mapper (P1) - rclone_target(storage) for s3, blob, gcs, drive, rclone and sftp; sftp goes through it; rclone obscure reads the password from stdin. Snapshots, restic (P2) - restic in the image; repository per source and channel, password derived from the master key (HKDF); stable host, bs: tags, translated excludes; exit 3 fails and forgets the snapshot. - Mirror restore with `restic restore --delete`; restore payload carries the encrypted channel; snapshot ids must be full ids. - Engine in the ping and in scheduled task metadata. One locked method per source (spec A) - databases.json declares options.method (archive, snapshot, sync), reported in the ping for local sources only. - Sync: rclone sync per channel onto /sync//current, refuses an empty source, reports transferred/deleted counters. Local channel (spec B) - Snapshots and Sync reach the dashboard's local storage through its append-only restic REST server and its WebDAV server. Security - Strict rclone config validation: one section, no duplicate keys, no command, credential or host-file keys, no quoted values, valid backend type, blocked backends. - Scheduled task metadata (decrypted storages) is no longer logged. --- Cargo.lock | 26 + Cargo.toml | 7 +- docker/Dockerfile | 23 + helm/README.md | 20 + helm/templates/deployment.yaml | 6 + helm/values.yaml | 11 + src/core/agent.rs | 6 +- src/domain/factory.rs | 3 + src/domain/files/backup.rs | 133 ++++ src/domain/files/database.rs | 45 ++ src/domain/files/matcher.rs | 73 +++ src/domain/files/mod.rs | 6 + src/domain/files/ping.rs | 16 + src/domain/files/platform.rs | 26 + src/domain/files/restore.rs | 247 +++++++ src/domain/mod.rs | 1 + .../api/endpoints/agent/backup/upload/init.rs | 5 + .../endpoints/agent/backup/upload/status.rs | 42 +- src/services/api/endpoints/agent/status.rs | 4 +- src/services/api/models/agent/status.rs | 17 +- src/services/backup/dispatcher.rs | 4 +- src/services/backup/executor.rs | 31 +- src/services/backup/result.rs | 8 +- src/services/backup/uploader.rs | 1 + src/services/config.rs | 41 +- src/services/cron.rs | 3 +- src/services/dashboard_config.rs | 10 + src/services/mod.rs | 2 + src/services/restic/backup.rs | 201 ++++++ src/services/restic/command.rs | 254 ++++++++ src/services/restic/excludes.rs | 79 +++ src/services/restic/json.rs | 52 ++ src/services/restic/mod.rs | 7 + src/services/restic/password.rs | 42 ++ src/services/restic/restore.rs | 124 ++++ src/services/restore/archive.rs | 5 + src/services/restore/dispatcher.rs | 13 + src/services/restore/executor.rs | 39 ++ src/services/status.rs | 36 +- .../providers/google_cloud_storage/helpers.rs | 21 +- .../providers/google_cloud_storage/mod.rs | 2 +- .../storage/providers/google_drive/mod.rs | 2 +- .../storage/providers/rclone/helpers.rs | 150 +++-- src/services/storage/providers/rclone/mod.rs | 1 + .../storage/providers/rclone/target.rs | 137 ++++ src/services/storage/providers/s3/mod.rs | 2 +- src/services/storage/providers/sftp/mod.rs | 29 +- src/services/sync/backup.rs | 147 +++++ src/services/sync/excludes.rs | 65 ++ src/services/sync/mod.rs | 6 + src/services/sync/rclone.rs | 86 +++ src/services/sync/stats.rs | 56 ++ src/tests/domain/files.rs | 616 ++++++++++++++++++ src/tests/domain/mod.rs | 1 + src/tests/services/api_models_tests.rs | 102 +++ src/tests/services/backup_uploader_tests.rs | 2 +- src/tests/services/config_tests.rs | 109 +++- src/tests/services/dashboard_config_tests.rs | 35 + src/tests/services/local_storage_tests.rs | 190 ++++++ src/tests/services/mod.rs | 5 +- src/tests/services/restic_tests.rs | 348 ++++++++++ src/tests/services/sync_tests.rs | 250 +++++++ .../rclone-target/blob-account-key.ini | 6 + .../rclone-target/blob-connection-string.ini | 6 + .../rclone-target/blob-endpoint-url.ini | 6 + .../google-cloud-storage-emulator.ini | 7 + .../rclone-target/google-cloud-storage.ini | 6 + .../fixtures/rclone-target/google-drive.ini | 8 + .../fixtures/rclone-target/inputs.json | 52 ++ .../storage/fixtures/rclone-target/rclone.ini | 6 + .../fixtures/rclone-target/s3-port-no-ssl.ini | 9 + .../storage/fixtures/rclone-target/s3.ini | 9 + .../fixtures/rclone-target/sftp-key.ini | 6 + .../fixtures/rclone-validation/cases.json | 345 ++++++++++ src/tests/storage/google_cloud_storage.rs | 2 +- src/tests/storage/mod.rs | 1 + src/tests/storage/rclone.rs | 60 +- src/tests/storage/rclone_target.rs | 301 +++++++++ src/utils/task_manager/scheduler.rs | 24 +- 79 files changed, 4760 insertions(+), 125 deletions(-) create mode 100644 src/domain/files/backup.rs create mode 100644 src/domain/files/database.rs create mode 100644 src/domain/files/matcher.rs create mode 100644 src/domain/files/mod.rs create mode 100644 src/domain/files/ping.rs create mode 100644 src/domain/files/platform.rs create mode 100644 src/domain/files/restore.rs create mode 100644 src/services/restic/backup.rs create mode 100644 src/services/restic/command.rs create mode 100644 src/services/restic/excludes.rs create mode 100644 src/services/restic/json.rs create mode 100644 src/services/restic/mod.rs create mode 100644 src/services/restic/password.rs create mode 100644 src/services/restic/restore.rs create mode 100644 src/services/storage/providers/rclone/target.rs create mode 100644 src/services/sync/backup.rs create mode 100644 src/services/sync/excludes.rs create mode 100644 src/services/sync/mod.rs create mode 100644 src/services/sync/rclone.rs create mode 100644 src/services/sync/stats.rs create mode 100644 src/tests/domain/files.rs create mode 100644 src/tests/services/local_storage_tests.rs create mode 100644 src/tests/services/restic_tests.rs create mode 100644 src/tests/services/sync_tests.rs create mode 100644 src/tests/storage/fixtures/rclone-target/blob-account-key.ini create mode 100644 src/tests/storage/fixtures/rclone-target/blob-connection-string.ini create mode 100644 src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini create mode 100644 src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini create mode 100644 src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini create mode 100644 src/tests/storage/fixtures/rclone-target/google-drive.ini create mode 100644 src/tests/storage/fixtures/rclone-target/inputs.json create mode 100644 src/tests/storage/fixtures/rclone-target/rclone.ini create mode 100644 src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini create mode 100644 src/tests/storage/fixtures/rclone-target/s3.ini create mode 100644 src/tests/storage/fixtures/rclone-target/sftp-key.ini create mode 100644 src/tests/storage/fixtures/rclone-validation/cases.json create mode 100644 src/tests/storage/rclone_target.rs diff --git a/Cargo.lock b/Cargo.lock index 521105f..6a4b2dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,6 +994,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -1956,6 +1966,19 @@ dependencies = [ "polyval", ] +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "google-cloud-auth" version = "1.16.0" @@ -3664,11 +3687,13 @@ dependencies = [ "futures", "futures-util", "generic-array", + "globset", "google-cloud-auth", "google-cloud-storage", "hex", "hmac 0.12.1", "hyper 1.11.1", + "libc", "log", "mockall", "mongodb", @@ -3702,6 +3727,7 @@ dependencies = [ "typenum", "url", "uuid", + "walkdir", "wiremock", ] diff --git a/Cargo.toml b/Cargo.toml index 1b5bda2..ab3bded 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.29" toml = "0.9.10" reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] } anyhow = "1.0.100" -tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] } +tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util", "time"] } async-trait = "0.1.89" tempfile = "3.24.0" hmac = "0.12" @@ -62,6 +62,11 @@ url = "2.5.8" percent-encoding = "2.3.2" bollard = "0.20.0" rustls = "0.23" +globset = "0.4" +walkdir = "2.5" + +[target.'cfg(unix)'.dependencies] +libc = "0.2" [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/docker/Dockerfile b/docker/Dockerfile index 7634388..86c5bed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,6 +58,17 @@ RUN ARCH=$(dpkg --print-architecture) \ && dpkg -i /tmp/rclone.deb \ && rm /tmp/rclone.deb +# ========================= +# restic (Snapshots mode) +# ========================= +ARG RESTIC_VERSION=0.19.1 +RUN ARCH=$(dpkg --print-architecture) \ + && curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \ + && bzip2 -d /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm /tmp/restic \ + && restic version + ARG TARGETARCH # ========================= @@ -146,6 +157,7 @@ RUN apt-get update && apt-get install -y \ libncurses6 \ zlib1g \ curl \ + bzip2 \ mariadb-client \ sqlite3 \ redis-tools \ @@ -159,6 +171,17 @@ RUN ARCH=$(dpkg --print-architecture) \ && dpkg -i /tmp/rclone.deb \ && rm /tmp/rclone.deb +# ========================= +# restic (Snapshots mode) +# ========================= +ARG RESTIC_VERSION=0.19.1 +RUN ARCH=$(dpkg --print-architecture) \ + && curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \ + && bzip2 -d /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm /tmp/restic \ + && restic version + ENV DOTNET_ROOT=/usr/local/dotnet RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \ && chmod +x /tmp/dotnet-install.sh \ diff --git a/helm/README.md b/helm/README.md index c41f381..b69522a 100644 --- a/helm/README.md +++ b/helm/README.md @@ -57,3 +57,23 @@ kubectl logs portabase-agent-6f7d4f5c6b-abc12 ``` bash helm uninstall portabase-agent ``` + +## Mount a directory for a `files` source + +A `files` source backs up a directory the agent can see. Mount it into the pod +with `extraVolumes` and `extraVolumeMounts`: + +```yaml +extraVolumes: + - name: shared-files + hostPath: + path: /srv/files + type: Directory + +extraVolumeMounts: + - name: shared-files + mountPath: /data/files +``` + +Mount it read-write: restores write into it. The `mountPath` (here `/data/files`) +is the `path` to enter for the source in Portabase. diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 97b1c41..d38a231 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -44,6 +44,9 @@ spec: # uses ConfigMap content {{- end }} {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} volumes: {{- if .Values.volume.configFile.enabled }} {{- if .Values.volume.configFile.hostPath }} @@ -59,4 +62,7 @@ spec: - key: config.json path: config.json {{- end }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} {{- end }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index dad96c4..46b3130 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -42,6 +42,17 @@ volume: ] } +# Extra volumes for the agent pod, e.g. a host directory used by a `files` +# source. Mount it read-write: restores write into it. +extraVolumes: [] +# - name: shared-files +# hostPath: +# path: /srv/files +# type: Directory + +extraVolumeMounts: [] +# - name: shared-files +# mountPath: /data/files # use this path as the source `path` network: hostAliases: diff --git a/src/core/agent.rs b/src/core/agent.rs index 9c78d4e..aaf12a4 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -4,7 +4,7 @@ use crate::core::context::Context; use crate::services::backup::BackupService; use crate::services::config::{ConfigService, DatabaseConfig}; use crate::services::cron::CronService; -use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache}; +use crate::services::dashboard_config::{collect_configs, load_cache, local_only_ids, merge, persist_cache}; use crate::services::restore::RestoreService; use crate::services::status::StatusService; use crate::settings::CONFIG; @@ -50,9 +50,10 @@ impl Agent { pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box> { let local = self.config_service.load_optional(None); + let local_ids = local_only_ids(&local.databases, &self.dashboard_cache); let merged_in = merge(&local.databases, &self.dashboard_cache); - let ping_result = self.status_service.ping(&merged_in.databases).await?; + let ping_result = self.status_service.ping(&merged_in.databases, &local_ids).await?; self.dashboard_cache = collect_configs(&ping_result); if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) { @@ -85,6 +86,7 @@ impl Agent { method.clone(), &db.storages, db.encrypt, + db.data.backup.engine.as_deref().unwrap_or("archive"), ) .await; } else if db.data.restore.action { diff --git a/src/domain/factory.rs b/src/domain/factory.rs index d4d1975..c755522 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -1,4 +1,5 @@ use crate::domain::docker_volume::database::DockerVolumeDatabase; +use crate::domain::files::database::FilesDatabase; use crate::domain::mongodb::database::MongoDatabase; use crate::domain::mysql::database::MySQLDatabase; use crate::domain::postgres::cluster::database::PostgresClusterDatabase; @@ -43,6 +44,7 @@ impl DatabaseFactory { DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)), DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)), DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)), + DbType::Files => Arc::new(FilesDatabase::new(cfg)), } } @@ -62,6 +64,7 @@ impl DatabaseFactory { DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)), DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)), DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)), + DbType::Files => Arc::new(FilesDatabase::new(cfg)), } } } diff --git a/src/domain/files/backup.rs b/src/domain/files/backup.rs new file mode 100644 index 0000000..4169eac --- /dev/null +++ b/src/domain/files/backup.rs @@ -0,0 +1,133 @@ +use super::matcher::ExcludeMatcher; +use super::platform::dev; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use flate2::Compression; +use flate2::write::GzEncoder; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use walkdir::WalkDir; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + logger: Arc, +) -> Result { + tokio::task::spawn_blocking(move || archive(&cfg, &backup_dir, &logger)).await? +} + +fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Result { + // Canonical paths so the output location can be recognised while walking. + let root = std::fs::canonicalize(&cfg.path) + .with_context(|| format!("cannot read source directory {}", cfg.path))?; + if !root.is_dir() { + bail!("source path {} is not a directory", root.display()); + } + let backup_real = std::fs::canonicalize(backup_dir) + .with_context(|| format!("cannot read backup directory {}", backup_dir.display()))?; + let matcher = ExcludeMatcher::from_config(cfg)?; + let one_file_system = cfg + .options + .get("one_file_system") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let root_dev = dev(&std::fs::metadata(&root) + .with_context(|| format!("cannot read source directory {}", root.display()))?); + + let file_name = format!("{}.tar.gz", cfg.generated_id); + let out = backup_dir.join(&file_name); + let out_real = backup_real.join(&file_name); + let gz = GzEncoder::new(File::create(&out)?, Compression::default()); + let mut tar = tar::Builder::new(gz); + tar.follow_symlinks(false); + + logger.log("info", format!("Archiving {}", root.display())); + let mut files = 0u64; + // Entries that could not be read: a mirror restore of this archive would delete them. + let mut unreadable = 0usize; + let mut walker = WalkDir::new(&root) + .follow_links(false) + .min_depth(1) + .into_iter(); + + while let Some(entry) = walker.next() { + let entry = match entry { + Ok(e) => e, + Err(e) => { + logger.log("warn", format!("Skipping unreadable entry: {e}")); + unreadable += 1; + continue; + } + }; + let rel = entry.path().strip_prefix(&root)?.to_path_buf(); + let ft = entry.file_type(); + + // Never archive our own output (backup dir under the source, or the archive itself). + let mut skip = + matcher.is_excluded(&rel) || entry.path() == backup_real || entry.path() == out_real; + if !skip && one_file_system && ft.is_dir() { + match entry.metadata() { + Ok(m) => skip = dev(&m) != root_dev, + Err(e) => { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + skip = true; + } + } + } + if skip { + if ft.is_dir() { + walker.skip_current_dir(); + } + continue; + } + + if ft.is_file() { + // Only a failure to open is skippable: it happens before anything is + // written. Every later error leaves the tar stream misaligned and must abort. + let mut f = match File::open(entry.path()) { + Ok(f) => f, + Err(e) => { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + continue; + } + }; + let meta = f.metadata()?; + let size = meta.len(); + let mut header = tar::Header::new_gnu(); + header.set_metadata(&meta); + // The header size is fixed up front: cap the copy if the file grew and + // zero-pad if it shrank, so the entry always matches its header. + let reader = (&mut f).take(size).chain(std::io::repeat(0)).take(size); + tar.append_data(&mut header, &rel, reader) + .with_context(|| format!("failed to archive {}", rel.display()))?; + files += 1; + } else if ft.is_dir() || ft.is_symlink() { + // These fail on lstat/readlink before the header is written. + if let Err(e) = tar.append_path_with_name(entry.path(), &rel) { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + } + } else { + // Sockets, fifos and devices hold no data to restore; restore keeps them too. + logger.log("warn", format!("Skipping special file {}", rel.display())); + } + } + + if unreadable > 0 { + let noun = if unreadable == 1 { "entry" } else { "entries" }; + bail!( + "{unreadable} {noun} could not be read; backup aborted so a mirror restore cannot delete unarchived data (fix permissions or add an exclude)" + ); + } + tar.into_inner()?.finish()?; + logger.log( + "info", + format!("Archived {files} file(s) into {}", out.display()), + ); + Ok(out) +} diff --git a/src/domain/files/database.rs b/src/domain/files/database.rs new file mode 100644 index 0000000..a205886 --- /dev/null +++ b/src/domain/files/database.rs @@ -0,0 +1,45 @@ +use anyhow::Result; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use super::{backup, ping, restore}; +use crate::domain::factory::Database; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct FilesDatabase { + cfg: DatabaseConfig, +} + +impl FilesDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for FilesDatabase { + fn file_extension(&self) -> &'static str { + ".tar.gz" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path, logger: Arc) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } + + async fn restore(&self, file: &Path, logger: Arc) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } +} diff --git a/src/domain/files/matcher.rs b/src/domain/files/matcher.rs new file mode 100644 index 0000000..3c3f5c0 --- /dev/null +++ b/src/domain/files/matcher.rs @@ -0,0 +1,73 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use std::path::{Path, PathBuf}; + +/// Exclude patterns, matched against paths relative to the source root. +/// No leading `/` matches at any depth; a leading `/` anchors at the root. +pub struct ExcludeMatcher { + set: GlobSet, + literal: Vec, +} + +impl ExcludeMatcher { + pub fn new(patterns: &[String]) -> Result { + let mut builder = GlobSetBuilder::new(); + for raw in patterns { + let pattern = raw.trim(); + if pattern.is_empty() { + continue; + } + let glob = match pattern.strip_prefix('/') { + Some(anchored) => anchored.trim_end_matches('/').to_string(), + None => format!("**/{}", pattern.trim_end_matches('/')), + }; + builder.add( + GlobBuilder::new(&glob) + .literal_separator(true) + .build() + .with_context(|| format!("invalid exclude pattern '{pattern}'"))?, + ); + } + Ok(Self { + set: builder.build()?, + literal: Vec::new(), + }) + } + + pub fn from_config(cfg: &DatabaseConfig) -> Result { + Self::new(&exclude_patterns(cfg)) + } + + /// Also exclude exactly this root-relative path (and its subtree), compared + /// literally so glob characters in the name mean nothing. + pub fn exclude_path(&mut self, relative: PathBuf) { + self.literal.push(relative); + } + + pub fn is_excluded(&self, relative: &Path) -> bool { + self.set.is_match(relative) || self.literal.iter().any(|p| p == relative) + } +} + +/// The `options.exclude` patterns of a files source, as configured. +pub fn exclude_patterns(cfg: &DatabaseConfig) -> Vec { + cfg.options + .get("exclude") + .and_then(|v| v.as_array()) + .map(|items| { + items + .iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +/// `options.one_file_system` of a files source. +pub fn one_file_system(cfg: &DatabaseConfig) -> bool { + cfg.options + .get("one_file_system") + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} diff --git a/src/domain/files/mod.rs b/src/domain/files/mod.rs new file mode 100644 index 0000000..d112947 --- /dev/null +++ b/src/domain/files/mod.rs @@ -0,0 +1,6 @@ +pub mod backup; +pub mod database; +pub mod matcher; +pub mod ping; +pub mod platform; +pub mod restore; diff --git a/src/domain/files/ping.rs b/src/domain/files/ping.rs new file mode 100644 index 0000000..ee61cd3 --- /dev/null +++ b/src/domain/files/ping.rs @@ -0,0 +1,16 @@ +use crate::services::config::DatabaseConfig; +use anyhow::Result; +use std::path::Path; +use tokio::time::{Duration, timeout}; + +pub async fn run(cfg: DatabaseConfig) -> Result { + // A hung mount (NFS) blocks is_dir/read_dir: keep it off the runtime and bounded. + let check = tokio::task::spawn_blocking(move || { + let root = Path::new(&cfg.path); + root.is_dir() && std::fs::read_dir(root).is_ok() + }); + Ok(matches!( + timeout(Duration::from_secs(10), check).await, + Ok(Ok(true)) + )) +} diff --git a/src/domain/files/platform.rs b/src/domain/files/platform.rs new file mode 100644 index 0000000..eddb031 --- /dev/null +++ b/src/domain/files/platform.rs @@ -0,0 +1,26 @@ +//! Unix-only metadata behind `cfg`, with inert fallbacks so the Windows release +//! build compiles. + +/// Device id, used to spot mount points. Off unix every entry reports 0, so no +/// directory is ever treated as a mount point. +#[cfg(unix)] +pub fn dev(meta: &std::fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + meta.dev() +} + +#[cfg(not(unix))] +pub fn dev(_meta: &std::fs::Metadata) -> u64 { + 0 +} + +/// Whether the agent can restore file ownership. +#[cfg(unix)] +pub fn is_root() -> bool { + unsafe { libc::geteuid() == 0 } +} + +#[cfg(not(unix))] +pub fn is_root() -> bool { + false +} diff --git a/src/domain/files/restore.rs b/src/domain/files/restore.rs new file mode 100644 index 0000000..fb07abc --- /dev/null +++ b/src/domain/files/restore.rs @@ -0,0 +1,247 @@ +use super::matcher::ExcludeMatcher; +use super::platform::{dev, is_root}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use flate2::read::GzDecoder; +use std::fs::{self, File}; +use std::io::{self, Read, Seek}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc) -> Result<()> { + tokio::task::spawn_blocking(move || restore(&cfg, &archive, &logger)).await? +} + +fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<()> { + // Everything below works on the canonical path, like backup.rs. + let root = ensure_restorable(Path::new(&cfg.path))?; + let mut matcher = ExcludeMatcher::from_config(cfg)?; + let one_file_system = cfg + .options + .get("one_file_system") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + // Opened once: the wipe may delete the archive's path, never an open handle. + let file = File::open(archive) + .with_context(|| format!("cannot open archive {}", archive.display()))?; + + // The wipe is irreversible: prove the whole archive reads back before touching data. + logger.log("info", format!("Validating archive {}", archive.display())); + validate_archive(&file).context("archive is corrupt or truncated, nothing was changed")?; + + // Keep the archive's own directory (or the archive itself when it sits directly + // in the target), like backup.rs prunes its output. + let archive_real = fs::canonicalize(archive)?; + let keep = match archive_real.parent() { + Some(dir) if dir != root => dir, + _ => archive_real.as_path(), + }; + if let Ok(rel) = keep.strip_prefix(&root) { + matcher.exclude_path(rel.to_path_buf()); + } + + logger.log( + "info", + format!("Removing non-excluded content of {}", root.display()), + ); + // Best effort: whatever could not be removed is kept, and extraction always runs. + let failures = wipe_except_excluded(&root, &matcher, one_file_system, logger)?; + + logger.log( + "info", + format!("Extracting archive into {}", root.display()), + ); + unpack(&file, &root, is_root(), logger)?; + + if failures > 0 { + let noun = if failures == 1 { "entry" } else { "entries" }; + bail!( + "restore extracted the archive but {failures} existing {noun} could not be removed; see warnings" + ); + } + logger.log("info", "Files restore finished".to_string()); + Ok(()) +} + +/// Extract the whole archive into `root`. Filesystems that refuse chown/chmod/utime +/// (NFS root_squash, CIFS, NFSv4 idmap mismatch) fail the first pass, after the wipe, +/// so retry once without metadata: the content matters more than owner and mode. +pub fn unpack( + file: &File, + root: &Path, + preserve_ownerships: bool, + logger: &JobLogger, +) -> Result<()> { + if let Err(e) = unpack_once(file, root, true, preserve_ownerships) { + logger.log( + "warn", + format!("extraction with ownership/permissions failed: {e:#}; retrying without metadata preservation"), + ); + // ponytail: tar-rs still chmods to `mode & 0o777` with preservation off, so a + // filesystem that rejects chmod on files the agent just created fails this pass + // too; per-entry extraction with deferred directories if that shows up. + unpack_once(file, root, false, false)?; + } + Ok(()) +} + +fn unpack_once( + mut file: &File, + root: &Path, + preserve: bool, + preserve_ownerships: bool, +) -> Result<()> { + file.rewind()?; + let mut ar = tar::Archive::new(GzDecoder::new(file)); + ar.set_preserve_permissions(preserve); + ar.set_preserve_mtime(preserve); + ar.set_preserve_ownerships(preserve_ownerships); + ar.set_overwrite(true); + ar.unpack(root)?; + Ok(()) +} + +/// Read every entry to the end, then drain the gzip stream so its CRC is checked. +fn validate_archive(file: &File) -> Result<()> { + let mut gz = GzDecoder::new(file); + let mut ar = tar::Archive::new(&mut gz); + for entry in ar.entries()? { + io::copy(&mut entry?, &mut io::sink())?; + } + drop(ar); + // tar stops at its end marker, before the gzip trailer. + gz.read_to_end(&mut Vec::new())?; + Ok(()) +} + +/// Refuse targets a mirror restore must never touch, and return the canonical path to +/// restore into. The canonical path is checked too, because `/data/..` style escapes +/// resolve to `/`. +pub fn ensure_restorable(root: &Path) -> Result { + if root.as_os_str().is_empty() || !root.is_absolute() || root == Path::new("/") { + bail!( + "refusing to restore into {:?}: path must be absolute and not /", + root + ); + } + if !root.is_dir() { + bail!( + "restore target {} is not an existing directory", + root.display() + ); + } + let canonical = fs::canonicalize(root) + .with_context(|| format!("cannot resolve restore target {}", root.display()))?; + if canonical == Path::new("/") { + bail!("refusing to restore into {:?}: it resolves to /", root); + } + Ok(canonical) +} + +/// Delete everything under `root` that does not match an exclude pattern. +/// Excluded entries (and their subtrees) are kept; `root` itself is kept. +/// +/// Mount points (a directory on a different device than its parent): with +/// `one_file_system` they are kept untouched like an excluded path, matching what +/// backup never archived. Without it their contents are wiped but the mount point +/// itself is never removed (`remove_dir` on it fails with EBUSY) and counts as kept. +/// Special files (sockets, fifos, devices) are kept: backup never archives them. +/// +/// Best effort: an entry that cannot be read or removed is logged, kept (so its +/// parents are kept too) and counted; the returned count is how many failed. +/// Only an unreadable `root` is an error, and then nothing was deleted. +pub fn wipe_except_excluded( + root: &Path, + matcher: &ExcludeMatcher, + one_file_system: bool, + logger: &JobLogger, +) -> Result { + let meta = fs::symlink_metadata(root).with_context(|| format!("reading {}", root.display()))?; + let mut wipe = Wipe { + matcher, + one_file_system, + logger, + failures: 0, + }; + wipe.dir(root, Path::new(""), dev(&meta)); + Ok(wipe.failures) +} + +struct Wipe<'a> { + matcher: &'a ExcludeMatcher, + one_file_system: bool, + logger: &'a JobLogger, + failures: usize, +} + +impl Wipe<'_> { + fn fail(&mut self, what: &str, path: &Path, e: io::Error) { + self.logger + .log("warn", format!("Could not {what} {}: {e}", path.display())); + self.failures += 1; + } + + /// Returns true when something was kept inside `dir`. `rel` is `dir` relative to + /// the root and `dir_dev` its device. + fn dir(&mut self, dir: &Path, rel: &Path, dir_dev: u64) -> bool { + let entries = match fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) => { + self.fail("read", dir, e); + return true; + } + }; + let mut kept = false; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + self.fail("read", dir, e); + kept = true; + continue; + } + }; + let path = entry.path(); + let rel = rel.join(entry.file_name()); + if self.matcher.is_excluded(&rel) { + kept = true; + continue; + } + // symlink_metadata: a symlinked directory is removed as a link, never followed. + let meta = match fs::symlink_metadata(&path) { + Ok(meta) => meta, + Err(e) => { + self.fail("read", &path, e); + kept = true; + continue; + } + }; + let ft = meta.file_type(); + let removed = if ft.is_dir() { + let mount_point = dev(&meta) != dir_dev; + if mount_point && self.one_file_system { + kept = true; + continue; + } + if self.dir(&path, &rel, dev(&meta)) || mount_point { + kept = true; + continue; + } + fs::remove_dir(&path) + } else if ft.is_file() || ft.is_symlink() { + fs::remove_file(&path) + } else { + // Socket, fifo or device: backup never archives them, so keep them. + kept = true; + continue; + }; + if let Err(e) = removed { + self.fail("remove", &path, e); + kept = true; + } + } + kept + } +} diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 095cbe1..16fcba1 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -1,4 +1,5 @@ pub mod docker_volume; +pub mod files; pub mod factory; pub mod mongodb; pub mod mysql; diff --git a/src/services/api/endpoints/agent/backup/upload/init.rs b/src/services/api/endpoints/agent/backup/upload/init.rs index cd39651..c3840c3 100644 --- a/src/services/api/endpoints/agent/backup/upload/init.rs +++ b/src/services/api/endpoints/agent/backup/upload/init.rs @@ -12,6 +12,9 @@ pub struct InitUploadRequest { pub storage_channel_id: String, #[serde(rename = "backupId")] pub backup_id: String, + /// "restic" for snapshots; omitted for archives (older dashboards ignore it). + #[serde(skip_serializing_if = "Option::is_none")] + pub engine: Option, } impl ApiClient { @@ -21,11 +24,13 @@ impl ApiClient { generated_id: impl Into, storage_channel_id: impl Into, backup_id: impl Into, + engine: Option<&str>, ) -> Result, ApiError> { let body = InitUploadRequest { generated_id: generated_id.into(), storage_channel_id: storage_channel_id.into(), backup_id: backup_id.into(), + engine: engine.map(str::to_string), }; let agent_id = agent_id.into(); diff --git a/src/services/api/endpoints/agent/backup/upload/status.rs b/src/services/api/endpoints/agent/backup/upload/status.rs index 41b1acc..9ce4e00 100644 --- a/src/services/api/endpoints/agent/backup/upload/status.rs +++ b/src/services/api/endpoints/agent/backup/upload/status.rs @@ -1,5 +1,6 @@ use crate::services::api::models::agent::backup::BackupUploadResponse; use crate::services::api::{ApiClient, ApiError}; +use crate::services::sync::stats::SyncStats; use anyhow::Result; use reqwest::Method; use serde::Serialize; @@ -15,6 +16,10 @@ pub struct StatusUploadRequest { pub size: u64, #[serde(rename = "backupId")] pub backup_id: String, + #[serde(rename = "filesTransferred", skip_serializing_if = "Option::is_none")] + pub files_transferred: Option, + #[serde(rename = "filesDeleted", skip_serializing_if = "Option::is_none")] + pub files_deleted: Option, } impl ApiClient { @@ -35,12 +40,41 @@ impl ApiClient { path: remote_path.into(), size: total_size.into(), backup_id: backup_id.into(), + files_transferred: None, + files_deleted: None, }; + self.send_upload_status(agent_id.into(), &body).await + } - let agent_id = agent_id.into(); - let path = format!("/agent/{}/backup/upload/status", agent_id); + /// Success status of one sync replica, with rclone's counters (`size` = bytes transferred). + pub async fn backup_upload_sync_status( + &self, + agent_id: impl Into, + generated_id: impl Into, + backup_storage_id: impl Into, + path: String, + stats: &SyncStats, + backup_id: impl Into, + ) -> Result, ApiError> { + let body = StatusUploadRequest { + generated_id: generated_id.into(), + backup_storage_id: backup_storage_id.into(), + status: "success".into(), + path, + size: stats.bytes, + backup_id: backup_id.into(), + files_transferred: Some(stats.transfers), + files_deleted: Some(stats.deletes), + }; + self.send_upload_status(agent_id.into(), &body).await + } - self.request_with_body(Method::PATCH, path.as_str(), &body) - .await + async fn send_upload_status( + &self, + agent_id: String, + body: &StatusUploadRequest, + ) -> Result, ApiError> { + let path = format!("/agent/{}/backup/upload/status", agent_id); + self.request_with_body(Method::PATCH, path.as_str(), body).await } } diff --git a/src/services/api/endpoints/agent/status.rs b/src/services/api/endpoints/agent/status.rs index 8c7a5ce..73acba0 100644 --- a/src/services/api/endpoints/agent/status.rs +++ b/src/services/api/endpoints/agent/status.rs @@ -12,7 +12,9 @@ pub struct DatabasePayload<'a> { #[serde(rename = "generatedId")] pub generated_id: &'a str, #[serde(rename = "pingStatus")] - pub ping_status: bool + pub ping_status: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub method: Option<&'a str>, } #[derive(Serialize)] diff --git a/src/services/api/models/agent/status.rs b/src/services/api/models/agent/status.rs index 1a5aeab..3a75d7e 100644 --- a/src/services/api/models/agent/status.rs +++ b/src/services/api/models/agent/status.rs @@ -61,14 +61,29 @@ pub struct DatabaseData { pub struct BackupInfo { pub action: bool, pub cron: Option, + /// "archive" | "restic"; absent from dashboards older than P2 (→ archive). + #[serde(default)] + pub engine: Option, } #[derive(Debug, Deserialize)] pub struct RestoreInfo { pub action: bool, + #[serde(default)] pub file: Option, - #[serde(rename = "metaFile")] + #[serde(default, rename = "metaFile")] pub meta_file: Option, #[serde(default, deserialize_with = "string_or_number_to_string")] pub size: Option, + /// "restic" for a snapshot restore; absent for archives. + #[serde(default)] + pub engine: Option, + #[serde(default, rename = "snapshotId")] + pub snapshot_id: Option, + /// AES-GCM JSON array holding the snapshot's storage channel. + #[serde(default, rename = "storageCiphertext")] + pub storage_ciphertext: Option, + /// Filled in memory from `storage_ciphertext`; never on the wire. + #[serde(skip)] + pub storage: Option, } diff --git a/src/services/backup/dispatcher.rs b/src/services/backup/dispatcher.rs index bf9145a..469cc0b 100644 --- a/src/services/backup/dispatcher.rs +++ b/src/services/backup/dispatcher.rs @@ -12,6 +12,7 @@ impl BackupService { method: BackupMethod, storages: &Vec, encrypt: bool, + engine: &str, ) { let Some(cfg) = config .databases @@ -29,10 +30,11 @@ impl BackupService { let db_cfg = cfg.clone(); let storages = storages.clone(); let generated_id = generated_id.clone(); + let engine = engine.to_string(); tokio::spawn(async move { if let Err(e) = service - .execute_backup(generated_id, db_cfg, method, storages, encrypt) + .execute_backup(generated_id, db_cfg, method, storages, encrypt, engine) .await { error!("Backup execution failed: {}", e); diff --git a/src/services/backup/executor.rs b/src/services/backup/executor.rs index 018eed9..c31884b 100644 --- a/src/services/backup/executor.rs +++ b/src/services/backup/executor.rs @@ -1,7 +1,10 @@ use super::logger::JobLogger; +use super::models::BackupResult; use super::service::BackupService; use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::config::DatabaseConfig; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::services::restic; +use crate::services::sync; use crate::utils::common::BackupMethod; use crate::utils::locks::FileLock; @@ -18,6 +21,7 @@ impl BackupService { method: BackupMethod, storages: Vec, encrypt: bool, + engine: String, ) -> Result<()> { let logger = Arc::new(JobLogger::new()); @@ -30,6 +34,31 @@ impl BackupService { let backup = self.create_backup_record(&generated_id, &method).await?; let backup_id = backup.backup.id; + if engine == "restic" || engine == "sync" { + if matches!(db_cfg.db_type, DbType::Files) { + let uploads = if engine == "restic" { + restic::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await + } else { + sync::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await + }; + logger.log("info", "Database backup job finished".to_string()); + let result = BackupResult { + generated_id: generated_id.clone(), + db_type: db_cfg.db_type.clone(), + status: "success".into(), + backup_file: None, + code: None, + }; + let duration_ms = start.elapsed().as_millis() as f64; + let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries(); + // fileSize = average logical size across storages (send_result): restic reports + // the size each snapshot processed; sync reports none, so it stays null. + self.send_result(result, uploads, &backup_id, logs, duration_ms).await?; + return Ok(()); + } + logger.log("warn", format!("The {engine} method only applies to files sources; making an archive")); + } + let temp_dir = TempDir::new()?; let tmp_path = temp_dir.path(); diff --git a/src/services/backup/result.rs b/src/services/backup/result.rs index e005948..5d2b124 100644 --- a/src/services/backup/result.rs +++ b/src/services/backup/result.rs @@ -21,11 +21,9 @@ impl BackupService { "failed" }; - let file_size = upload_results - .iter() - .filter_map(|r| r.total_size) - .reduce(|a, b| a + b) - .map(|sum| sum / upload_results.len() as u64); + // Average over the uploads that reported a size: failed ones have none. + let sizes: Vec = upload_results.iter().filter_map(|r| r.total_size).collect(); + let file_size = (!sizes.is_empty()).then(|| sizes.iter().sum::() / sizes.len() as u64); self.ctx .api diff --git a/src/services/backup/uploader.rs b/src/services/backup/uploader.rs index 64b6325..c20e8f1 100644 --- a/src/services/backup/uploader.rs +++ b/src/services/backup/uploader.rs @@ -49,6 +49,7 @@ impl BackupService { generated_id.clone(), storage_id.clone(), backup_id, + None, ) .await { diff --git a/src/services/config.rs b/src/services/config.rs index 9819632..bb3c3b9 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -28,6 +28,7 @@ pub enum DbType { Mssql, #[serde(rename = "docker-volume")] DockerVolume, + Files, } impl DbType { @@ -44,6 +45,7 @@ impl DbType { DbType::Firebird => "firebird", DbType::Mssql => "mssql", DbType::DockerVolume => "docker-volume", + DbType::Files => "files", } } } @@ -112,6 +114,17 @@ fn optional(opt: &Option) -> T { opt.clone().unwrap_or_default() } +pub const FILES_METHODS: [&str; 3] = ["archive", "snapshot", "sync"]; + +/// The method a files source declares in `options.method` (`archive` when absent). +pub fn files_method(cfg: &DatabaseConfig) -> &'static str { + match cfg.options.get("method").and_then(|v| v.as_str()) { + Some("snapshot") => "snapshot", + Some("sync") => "sync", + _ => "archive", + } +} + pub fn build_config(db: InputDatabaseConfig) -> Result { if Uuid::parse_str(&db.generated_id).is_err() { return Err(format!("Invalid UUID for database '{}'", db.name)); @@ -143,7 +156,7 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { | DbType::Firebird | DbType::Valkey | DbType::Mssql => required(&db.host, &db.name, "host")?, - DbType::Sqlite | DbType::DockerVolume => optional(&db.host), + DbType::Sqlite | DbType::DockerVolume | DbType::Files => optional(&db.host), }; let port = match db.db_type { @@ -155,11 +168,13 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { | DbType::Firebird | DbType::Valkey | DbType::Mssql => required(&db.port, &db.name, "port")?, - DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), + DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume | DbType::Files => { + db.port.unwrap_or(0) + } }; let database_name = match db.db_type { - DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => { + DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume | DbType::Files => { optional(&db.database) } DbType::PostgresqlCluster => db @@ -170,8 +185,28 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { }; let path_val = match db.db_type { DbType::Sqlite => required(&db.path, &db.name, "path")?, + DbType::Files => { + let p = required(&db.path, &db.name, "path")?; + if !p.starts_with('/') || p.trim_end_matches('/').is_empty() { + return Err(format!( + "Path for files source '{}' must be absolute and not '/'", + db.name + )); + } + p + } _ => optional(&db.path), }; + if matches!(db.db_type, DbType::Files) { + if let Some(method) = db.options.as_ref().and_then(|o| o.get("method")) { + if !method.as_str().is_some_and(|m| FILES_METHODS.contains(&m)) { + return Err(format!( + "Unknown method {method} for files source '{}' (expected archive, snapshot or sync)", + db.name + )); + } + } + } let max_packet_size = match db.db_type { DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()), _ => String::new(), diff --git a/src/services/cron.rs b/src/services/cron.rs index 87aa1a3..69d9166 100644 --- a/src/services/cron.rs +++ b/src/services/cron.rs @@ -29,7 +29,8 @@ impl CronService { let encrypt: bool = database.encrypt; let metadata = json!({ "storages": storages, - "encrypt": encrypt + "encrypt": encrypt, + "engine": database.data.backup.engine.as_deref().unwrap_or("archive"), }); check_and_update_cron( diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs index 851a2c8..e5d6b91 100644 --- a/src/services/dashboard_config.rs +++ b/src/services/dashboard_config.rs @@ -2,6 +2,7 @@ use crate::services::api::models::agent::status::PingResult; use crate::services::config::{DatabaseConfig, DatabasesConfig}; +use std::collections::HashSet; use std::path::Path; pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig { @@ -19,6 +20,15 @@ pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> Database DatabasesConfig { databases } } +/// `generated_id`s of local (databases.json) sources that no dashboard config replaces. +pub fn local_only_ids(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> HashSet { + local + .iter() + .filter(|c| !dashboard.iter().any(|d| d.generated_id == c.generated_id)) + .map(|c| c.generated_id.clone()) + .collect() +} + pub fn collect_configs(ping: &PingResult) -> Vec { ping.databases .iter() diff --git a/src/services/mod.rs b/src/services/mod.rs index e6c391a..f6516ed 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -3,6 +3,8 @@ pub mod backup; pub mod config; pub mod cron; pub mod dashboard_config; +pub mod restic; pub mod restore; pub mod status; pub mod storage; +pub mod sync; diff --git a/src/services/restic/backup.rs b/src/services/restic/backup.rs new file mode 100644 index 0000000..c713a17 --- /dev/null +++ b/src/services/restic/backup.rs @@ -0,0 +1,201 @@ +use super::command::{ResticRepo, short}; +use super::excludes::backup_excludes; +use super::json::BackupSummary; +use crate::core::context::Context as CoreContext; +use crate::domain::files::matcher::{exclude_patterns, one_file_system}; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::backup::models::UploadResult; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::{Context, Result, bail}; + +/// One snapshot of `cfg.path`, tagged with its `backup_storage` id. +/// Exit 3 (unreadable files) fails the run and forgets the snapshot: a mirror +/// restore of it would delete the files it is missing (P0 rule R21). On an +/// append-only repository the agent cannot forget it, so the dashboard does. +pub async fn snapshot( + repo: &ResticRepo, + cfg: &DatabaseConfig, + backup_storage_id: &str, + logger: &JobLogger, +) -> Result { + let root = std::fs::canonicalize(&cfg.path) + .with_context(|| format!("cannot read source directory {}", cfg.path))?; + if !root.is_dir() { + bail!("source path {} is not a directory", root.display()); + } + repo.ensure_initialized(logger).await?; + + let mut args: Vec = vec![ + "backup".into(), + root.to_string_lossy().into_owned(), + "--json".into(), + // Stable host: container hostnames change on recreate and would break + // parent-snapshot detection (full rescan every time). + "--host".into(), + format!("portabase-{}", cfg.generated_id), + "--tag".into(), + "portabase".into(), + "--tag".into(), + format!("bs:{backup_storage_id}"), + ]; + for pattern in backup_excludes(&root, &exclude_patterns(cfg))? { + args.push("--exclude".into()); + args.push(pattern); + } + if one_file_system(cfg) { + args.push("--one-file-system".into()); + } + + logger.log("info", format!("Snapshotting {} into {}", root.display(), repo.repository())); + let run = repo.run(&args, logger).await?; + for error in &run.errors { + logger.log("warn", format!("Unreadable: {error}")); + } + let summary: Option = run + .summary + .clone() + .map(serde_json::from_value) + .transpose() + .context("unexpected restic backup summary")?; + + match (run.code, summary) { + (0, Some(s)) => { + logger.log( + "info", + format!( + "Snapshot {}: {} new, {} changed, {} unmodified file(s), {} byte(s) added", + short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.data_added_packed + ), + ); + Ok(s) + } + (0, None) => bail!("restic backup returned no summary"), + (3, summary) => { + let mut fate = "snapshot discarded"; + if let Some(s) = summary { + if repo.is_append_only() { + logger.log( + "warn", + format!("Incomplete snapshot {} left for the dashboard to forget", short(&s.snapshot_id)), + ); + fate = "snapshot not restorable (the dashboard forgets it)"; + } else { + let forget = repo.run(["forget", s.snapshot_id.as_str()], logger).await?; + if forget.code != 0 { + logger.log( + "error", + format!("Could not forget incomplete snapshot {}: {}", s.snapshot_id, forget.error("forget")), + ); + } + } + } + bail!( + "{} file(s) could not be read; {fate} so a mirror restore cannot delete them (fix permissions or add an exclude)", + run.errors.len() + ) + } + _ => Err(run.error("backup")), + } +} + +/// Snapshots every storage in turn (one scan each, v1). Never errors: each +/// storage reports its own `backup_storage` row, like the archive uploader. +pub async fn run( + ctx: &CoreContext, + cfg: &DatabaseConfig, + storages: &[DatabaseStorage], + backup_id: &str, + logger: &JobLogger, +) -> Vec { + if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { + logger.log("error", format!("Snapshot aborted: {e}")); + return Vec::new(); + } + let mut results = Vec::with_capacity(storages.len()); + for storage in storages { + results.push(one_storage(ctx, cfg, storage, backup_id, logger).await); + } + if let Err(e) = FileLock::release(&cfg.generated_id).await { + logger.log("warn", format!("Failed to release the backup lock: {e}")); + } + results +} + +pub(crate) async fn one_storage( + ctx: &CoreContext, + cfg: &DatabaseConfig, + storage: &DatabaseStorage, + backup_id: &str, + logger: &JobLogger, +) -> UploadResult { + let agent_id = ctx.edge_key.agent_id.clone(); + let failed = |error: String| UploadResult { + storage_id: storage.id.clone(), + success: false, + error: Some(error), + remote_file_path: None, + total_size: None, + }; + + let backup_storage_id = match ctx + .api + .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("restic")) + .await + { + Ok(Some(response)) => response.backup_storage.id, + Ok(None) => { + logger.log("error", "Upload init returned empty response"); + return failed("backup_upload_init returned empty response".into()); + } + Err(e) => { + logger.log("error", format!("Upload init failed: {e}")); + return failed("backup_upload_init failed".into()); + } + }; + + let outcome = match ResticRepo::open(storage, &cfg.generated_id, &ctx.edge_key) { + Ok(repo) => snapshot(&repo, cfg, &backup_storage_id, logger).await, + Err(e) => Err(e), + }; + + match outcome { + Ok(summary) => match ctx + .api + .backup_upload_status( + agent_id, + cfg.generated_id.clone(), + backup_storage_id, + "success", + summary.snapshot_id.clone(), + summary.data_added_packed, + backup_id, + ) + .await + { + Ok(_) => UploadResult { + storage_id: storage.id.clone(), + success: true, + error: None, + remote_file_path: Some(summary.snapshot_id), + total_size: Some(summary.total_bytes_processed), + }, + Err(e) => { + logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); + failed(e.to_string()) + } + }, + Err(e) => { + logger.log("error", format!("Snapshot to storage {} failed: {e:#}", storage.id)); + if let Err(err) = ctx + .api + .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) + .await + { + logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); + } + failed(format!("{e:#}")) + } + } +} diff --git a/src/services/restic/command.rs b/src/services/restic/command.rs new file mode 100644 index 0000000..a99ab02 --- /dev/null +++ b/src/services/restic/command.rs @@ -0,0 +1,254 @@ +use super::json::{self, Line}; +use super::password::{derive_password, local_storage_password, LOCAL_STORAGE_USER}; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; +use crate::services::storage::providers::rclone::target::{RcloneTarget, rclone_target}; +use crate::settings::CONFIG; +use crate::utils::edge_key::EdgeKey; +use anyhow::{Context, Result, anyhow}; +use serde_json::Value; +use std::ffi::{OsStr, OsString}; +use std::path::PathBuf; +use std::process::Stdio; +use tempfile::NamedTempFile; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +/// One source's repository on one storage channel: +/// `rclone:://restic/`, or +/// `rest:/storage/restic//` on the dashboard's local storage. +pub struct ResticRepo { + repository: String, + password: String, + /// `None` when the cache directory cannot be created: restic then runs `--no-cache`. + cache_dir: Option, + /// rclone config of `rclone:` repositories; `None` on the dashboard's REST server. + config: Option, + /// HTTP basic auth of `rest:` repositories (the dashboard's local storage). + rest_auth: Option<(String, String)>, + /// Keeps temp files referenced by the config (sftp key) alive. + _target: Option, +} + +/// What a finished restic command left behind. +pub struct ResticRun { + pub code: i32, + /// stdout lines that are neither `status` nor `summary` (e.g. `snapshots --json`). + pub stdout: String, + /// The last `summary` line, if any. + pub summary: Option, + /// `error` lines as "item: message" (unreadable files and the like). + pub errors: Vec, + /// Last plain-text / `exit_error` stderr lines, for failure messages. + pub stderr_tail: String, +} + +impl ResticRun { + pub fn error(&self, op: &str) -> anyhow::Error { + match self.code { + 10 => anyhow!("restic {op}: repository does not exist"), + 11 => anyhow!("restic {op}: repository is locked by another operation (waited 5 minutes)"), + 12 => anyhow!("restic {op}: wrong repository password (did the master key change?)"), + code => anyhow!("restic {op} failed (exit {code}): {}", self.stderr_tail.trim()), + } + } +} + +/// A full snapshot id. Ids from the dashboard reach restic as positional arguments: +/// anything else could be parsed as a flag (e.g. `--password-command=`). +pub fn is_snapshot_id(id: &str) -> bool { + id.len() == 64 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +pub fn short(id: &str) -> &str { + id.get(..8).unwrap_or(id) +} + +/// restic fails hard (not warns) on an unusable cache dir, so probe it up front. +fn usable_cache_dir(cache_dir: PathBuf) -> Option { + match std::fs::create_dir_all(&cache_dir) { + Ok(()) => Some(cache_dir), + Err(e) => { + tracing::warn!("restic cache dir {} unusable ({e}); running without cache", cache_dir.display()); + None + } + } +} + +impl ResticRepo { + pub fn open(storage: &DatabaseStorage, generated_id: &str, edge_key: &EdgeKey) -> Result { + let password = derive_password(&edge_key.master_key_b64, generated_id)?; + let cache_dir = PathBuf::from(&CONFIG.data_path).join("cache/restic"); + if storage.provider == "local" { + return Ok(Self::rest( + &format!("{}/storage/restic", edge_key.server_url.trim_end_matches('/')), + generated_id, + &password, + &local_storage_password(&edge_key.master_key_b64)?, + cache_dir, + )); + } + let target = rclone_target(storage)?; + let folder = storage + .folder_name + .as_deref() + .map(|f| f.trim().trim_matches('/')) + .filter(|f| !f.is_empty()) + .unwrap_or("backups"); + let mut repo = Self::new( + &target.config_text, + &target.remote_name, + &target.base_path, + folder, + generated_id, + &password, + cache_dir, + )?; + repo._target = Some(target); + Ok(repo) + } + + pub fn new( + config_text: &str, + remote_name: &str, + base_path: &str, + folder: &str, + generated_id: &str, + password: &str, + cache_dir: PathBuf, + ) -> Result { + let path = remote_target(remote_name, base_path, &format!("{folder}/restic/{generated_id}")); + Ok(Self { + repository: format!("rclone:{path}"), + password: password.to_string(), + cache_dir: usable_cache_dir(cache_dir), + config: Some(write_config(config_text)?), + rest_auth: None, + _target: None, + }) + } + + /// The dashboard's local storage (`rclone serve restic --append-only`): + /// `rest://`, basic auth only in the child env. + pub fn rest(base_url: &str, generated_id: &str, password: &str, server_password: &str, cache_dir: PathBuf) -> Self { + Self { + repository: format!("rest:{}/{generated_id}/", base_url.trim_end_matches('/')), + password: password.to_string(), + cache_dir: usable_cache_dir(cache_dir), + config: None, + rest_auth: Some((LOCAL_STORAGE_USER.to_string(), server_password.to_string())), + _target: None, + } + } + + /// The dashboard's server refuses deletions: the agent cannot forget there. + pub fn is_append_only(&self) -> bool { + self.rest_auth.is_some() + } + + pub fn repository(&self) -> &str { + &self.repository + } + + /// `restic --retry-lock 5m` (+ `--no-cache` without a cache dir); secrets only in the child env. `status` + /// lines are logged every 10 %. + pub async fn run(&self, args: I, logger: &JobLogger) -> Result + where + I: IntoIterator, + S: AsRef, + { + let args: Vec = args.into_iter().map(|a| a.as_ref().to_owned()).collect(); + let shown: Vec = args.iter().map(|a| a.to_string_lossy().into_owned()).collect(); + logger.log("debug", format!("restic {}", shown.join(" "))); + + let mut cmd = Command::new("restic"); + cmd.args(&args).args(["--retry-lock", "5m"]); + match &self.cache_dir { + Some(dir) => cmd.env("RESTIC_CACHE_DIR", dir), + None => cmd.arg("--no-cache"), + }; + if let Some(config) = &self.config { + cmd.env("RCLONE_CONFIG", config.path()); + } + if let Some((user, pass)) = &self.rest_auth { + cmd.env("RESTIC_REST_USERNAME", user).env("RESTIC_REST_PASSWORD", pass); + } + let mut child = cmd + .env("RESTIC_REPOSITORY", &self.repository) + .env("RESTIC_PASSWORD", &self.password) + // No TTY: without this restic prints no status lines at all. + .env("RESTIC_PROGRESS_FPS", "0.2") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => anyhow!("restic binary not found"), + _ => anyhow!(e).context("failed to start restic"), + })?; + + let stderr = child.stderr.take().context("restic stderr unavailable")?; + let stderr_task = tokio::spawn(async move { + let mut errors = Vec::new(); + let mut tail = Vec::new(); + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + match json::parse(&line) { + Line::Error { message, item } => errors.push(match item { + Some(item) => format!("{item}: {message}"), + None => message, + }), + Line::ExitError { message, .. } => tail.push(message), + Line::Text(text) if !text.trim().is_empty() => tail.push(text), + _ => {} + } + } + (errors, tail) + }); + + let mut run = ResticRun { code: -1, stdout: String::new(), summary: None, errors: Vec::new(), stderr_tail: String::new() }; + let stdout = child.stdout.take().context("restic stdout unavailable")?; + let mut lines = BufReader::new(stdout).lines(); + let mut next_pct = 10u32; + while let Some(line) = lines.next_line().await.context("failed to read restic output")? { + match json::parse(&line) { + Line::Status { percent_done } => { + let pct = (percent_done * 100.0) as u32; + if pct >= next_pct { + logger.log("info", format!("restic: {pct}% done")); + next_pct = pct / 10 * 10 + 10; + } + } + Line::Summary(value) => run.summary = Some(value), + _ => { + run.stdout.push_str(&line); + run.stdout.push('\n'); + } + } + } + + let status = child.wait().await.context("failed to wait for restic")?; + let (errors, tail) = stderr_task.await.unwrap_or_default(); + run.code = status.code().unwrap_or(-1); + run.errors = errors; + run.stderr_tail = tail[tail.len().saturating_sub(10)..].join("\n"); + Ok(run) + } + + /// `restic cat config`; exit 10 (no repository) → `restic init`. Callers hold + /// the source's FileLock, so two inits never race. + pub async fn ensure_initialized(&self, logger: &JobLogger) -> Result<()> { + let cat = self.run(["cat", "config"], logger).await?; + match cat.code { + 0 => Ok(()), + 10 => { + logger.log("info", format!("Initializing restic repository {}", self.repository)); + let init = self.run(["init"], logger).await?; + if init.code == 0 { Ok(()) } else { Err(init.error("init")) } + } + _ => Err(cat.error("cat config")), + } + } +} diff --git a/src/services/restic/excludes.rs b/src/services/restic/excludes.rs new file mode 100644 index 0000000..bc2dbdd --- /dev/null +++ b/src/services/restic/excludes.rs @@ -0,0 +1,79 @@ +use anyhow::{Result, bail}; +use std::path::Path; + +/// Escapes restic/filepath.Match metacharacters so a literal path matches only itself. +pub fn glob_escape(literal: &str) -> String { + let mut out = String::with_capacity(literal.len()); + for c in literal.chars() { + if matches!(c, '\\' | '*' | '?' | '[' | ']') { + out.push('\\'); + } + out.push(c); + } + out +} + +/// globset (the P0 matcher) reads `[!x]` as bracket negation; restic (Go +/// `filepath.Match`) and rclone only know `[^x]` and take `!` literally. +pub(crate) fn bracket_negation(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + let mut in_class = false; + while let Some(c) = chars.next() { + out.push(c); + match c { + '\\' => out.extend(chars.next()), + '[' if !in_class => { + in_class = true; + if chars.as_str().starts_with('!') { + chars.next(); + out.push('^'); + } + } + ']' => in_class = false, + _ => {} + } + } + out +} + +/// P0 patterns, trimmed like the P0 matcher, as `(anchored, pattern)`; a leading `/` marks anchored ones. +fn normalized(patterns: &[String]) -> Result> { + let mut out = Vec::new(); + for raw in patterns { + let pattern = raw.trim(); + if pattern.contains(['{', '}']) { + bail!("exclude pattern '{pattern}': brace patterns are not supported in Snapshots mode"); + } + let (anchored, rest) = match pattern.strip_prefix('/') { + Some(rest) => (true, rest), + None => (false, pattern), + }; + let rest = rest.trim_end_matches('/'); + if !rest.is_empty() { + out.push((anchored, bracket_negation(rest))); + } + } + Ok(out) +} + +/// `restic backup` matches patterns against absolute paths, so every pattern is +/// rooted at the source; unanchored ones would otherwise also match directories +/// above it and exclude the whole source (spike S5). +pub fn backup_excludes(root: &Path, patterns: &[String]) -> Result> { + let root = glob_escape(root.to_string_lossy().trim_end_matches('/')); + Ok(normalized(patterns)? + .into_iter() + .map(|(anchored, p)| { + if anchored { format!("{root}/{p}") } else { format!("{root}/**/{p}") } + }) + .collect()) +} + +/// `restic restore :` matches patterns relative to that subfolder (spike S1). +pub fn restore_excludes(patterns: &[String]) -> Result> { + Ok(normalized(patterns)? + .into_iter() + .map(|(anchored, p)| if anchored { format!("/{p}") } else { format!("/**/{p}") }) + .collect()) +} diff --git a/src/services/restic/json.rs b/src/services/restic/json.rs new file mode 100644 index 0000000..a4bfb9d --- /dev/null +++ b/src/services/restic/json.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; +use serde_json::Value; + +/// One line of `restic --json` output. stdout carries `status` / `summary`, +/// stderr carries `error` / `exit_error` and plain-text fatal messages. +#[derive(Debug, PartialEq)] +pub enum Line { + Status { percent_done: f64 }, + Summary(Value), + Error { message: String, item: Option }, + ExitError { code: i32, message: String }, + Text(String), +} + +pub fn parse(line: &str) -> Line { + let Ok(value) = serde_json::from_str::(line) else { + return Line::Text(line.to_string()); + }; + match value.get("message_type").and_then(Value::as_str) { + Some("status") => Line::Status { + percent_done: value["percent_done"].as_f64().unwrap_or(0.0), + }, + Some("summary") => Line::Summary(value), + Some("error") => Line::Error { + message: value["error"]["message"].as_str().unwrap_or("unknown error").to_string(), + item: value["item"].as_str().map(str::to_string), + }, + Some("exit_error") => Line::ExitError { + code: value["code"].as_i64().unwrap_or(1) as i32, + message: value["message"].as_str().unwrap_or_default().to_string(), + }, + _ => Line::Text(line.to_string()), + } +} + +#[derive(Debug, Deserialize)] +pub struct BackupSummary { + pub snapshot_id: String, + pub data_added_packed: u64, + pub total_bytes_processed: u64, + pub files_new: u64, + pub files_changed: u64, + pub files_unmodified: u64, +} + +/// One entry of `restic snapshots --json`. +#[derive(Debug, Deserialize)] +pub struct Snapshot { + #[allow(dead_code)] // part of the `restic snapshots` shape; restore() only needs `paths` + pub id: String, + pub paths: Vec, +} diff --git a/src/services/restic/mod.rs b/src/services/restic/mod.rs new file mode 100644 index 0000000..8d112b5 --- /dev/null +++ b/src/services/restic/mod.rs @@ -0,0 +1,7 @@ +//! Snapshots mode: restic repositories reached through the P1 rclone mapper. +pub mod backup; +pub mod command; +pub mod excludes; +pub mod json; +pub mod password; +pub mod restore; diff --git a/src/services/restic/password.rs b/src/services/restic/password.rs new file mode 100644 index 0000000..ca08919 --- /dev/null +++ b/src/services/restic/password.rs @@ -0,0 +1,42 @@ +use anyhow::{Context, Result}; +use base64::{Engine as _, engine::general_purpose}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +type HmacSha256 = Hmac; + +/// Repository password: hex(HKDF-SHA256(master key, salt = "", +/// info = "portabase/restic/v1/", 32 bytes)). The dashboard derives +/// the same value (`src/lib/restic/repo.ts`), so it is never stored nor sent. +pub fn derive_password(master_key_b64: &str, generated_id: &str) -> Result { + let master_key = general_purpose::STANDARD + .decode(master_key_b64.trim()) + .context("master key is not valid base64")?; + let info = format!("portabase/restic/v1/{generated_id}"); + Ok(hex::encode(hkdf_sha256_32(&master_key, info.as_bytes()))) +} + +/// User of the dashboard's local-storage servers (`/storage/restic`, `/storage/sync`). +pub const LOCAL_STORAGE_USER: &str = "portabase"; + +/// Their password: hex(HKDF-SHA256(master key, salt = "", info = "portabase/local-storage/v1", +/// 32 bytes)). The dashboard derives the same value (`src/lib/local-storage/credential.ts`). +pub fn local_storage_password(master_key_b64: &str) -> Result { + let master_key = general_purpose::STANDARD + .decode(master_key_b64.trim()) + .context("master key is not valid base64")?; + Ok(hex::encode(hkdf_sha256_32(&master_key, b"portabase/local-storage/v1"))) +} + +/// RFC 5869 with an empty salt and a single expand block (L = HashLen = 32). +pub fn hkdf_sha256_32(ikm: &[u8], info: &[u8]) -> [u8; 32] { + // An empty HMAC key is zero-padded: the RFC's default salt of HashLen zeros. + let mut extract = HmacSha256::new_from_slice(&[]).expect("HMAC takes any key length"); + extract.update(ikm); + let prk = extract.finalize().into_bytes(); + + let mut expand = HmacSha256::new_from_slice(&prk).expect("HMAC takes any key length"); + expand.update(info); + expand.update(&[1]); + expand.finalize().into_bytes().into() +} diff --git a/src/services/restic/restore.rs b/src/services/restic/restore.rs new file mode 100644 index 0000000..12c410f --- /dev/null +++ b/src/services/restic/restore.rs @@ -0,0 +1,124 @@ +use super::command::{ResticRepo, is_snapshot_id, short}; +use super::excludes::{glob_escape, restore_excludes}; +use super::json::Snapshot; +use crate::domain::files::matcher::{exclude_patterns, one_file_system}; +use crate::domain::files::platform::dev; +use crate::domain::files::restore::ensure_restorable; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::utils::edge_key::EdgeKey; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::{Context, Result, bail}; +use std::path::Path; +use walkdir::WalkDir; + +/// Mirror restore of `snapshot_id` into `cfg.path`: files created since are +/// deleted, paths matching the current exclude patterns are kept (spike S1). +pub async fn restore( + repo: &ResticRepo, + cfg: &DatabaseConfig, + snapshot_id: &str, + logger: &JobLogger, +) -> Result<()> { + if !is_snapshot_id(snapshot_id) { + bail!("invalid snapshot id in the restore payload"); + } + let target = ensure_restorable(Path::new(&cfg.path))?; + + let listed = repo.run(["snapshots", snapshot_id, "--json"], logger).await?; + if listed.code != 0 { + return Err(listed.error("snapshots")); + } + let snapshots: Vec = serde_json::from_str(listed.stdout.trim()) + .context("unexpected `restic snapshots` output")?; + // The original path inside the snapshot; the mount point may have moved since. + let Some(original) = snapshots.first().and_then(|s| s.paths.first()) else { + bail!("snapshot {} not found in {}", short(snapshot_id), repo.repository()); + }; + + let mut excludes = restore_excludes(&exclude_patterns(cfg))?; + if one_file_system(cfg) { + excludes.extend(mount_points(&target).into_iter().map(|rel| format!("/{}", glob_escape(&rel)))); + } + let mut args: Vec = vec![ + "restore".into(), + format!("{snapshot_id}:{original}"), + "--target".into(), + target.to_string_lossy().into_owned(), + "--delete".into(), + "--json".into(), + ]; + for pattern in excludes { + args.push("--exclude".into()); + args.push(pattern); + } + + logger.log( + "info", + format!("Restoring snapshot {} ({original}) into {}", short(snapshot_id), target.display()), + ); + let run = repo.run(&args, logger).await?; + for error in &run.errors { + logger.log("warn", format!("Restore error: {error}")); + } + if run.code != 0 { + return Err(run.error("restore")); + } + if let Some(s) = &run.summary { + logger.log( + "info", + format!( + "Restored {} file(s), deleted {} file(s)", + s["files_restored"].as_u64().unwrap_or(0), + s["files_deleted"].as_u64().unwrap_or(0) + ), + ); + } + Ok(()) +} + +/// Directories under `root` on another device. With `one_file_system` backup +/// never entered them, so the mirror restore must not delete their content. +fn mount_points(root: &Path) -> Vec { + let Ok(meta) = std::fs::metadata(root) else { return Vec::new() }; + let root_dev = dev(&meta); + let mut out = Vec::new(); + let mut walker = WalkDir::new(root).min_depth(1).follow_links(false).into_iter(); + while let Some(entry) = walker.next() { + let Ok(entry) = entry else { continue }; + if !entry.file_type().is_dir() { + continue; + } + if matches!(entry.metadata(), Ok(m) if dev(&m) != root_dev) { + if let Ok(rel) = entry.path().strip_prefix(root) { + out.push(rel.to_string_lossy().into_owned()); + } + walker.skip_current_dir(); + } + } + out +} + +/// Snapshot restore job: the source's FileLock is held for the whole restore. +pub async fn run( + edge_key: &EdgeKey, + cfg: &DatabaseConfig, + snapshot_id: &str, + storage: &DatabaseStorage, + logger: &JobLogger, +) -> Result<()> { + if !matches!(cfg.db_type, DbType::Files) { + bail!("snapshot restore needs a files source, got {}", cfg.db_type.as_str()); + } + FileLock::acquire(&cfg.generated_id, DbOpLock::Restore.as_str()) + .await + .context("another backup or restore of this source is running")?; + let result = async { + let repo = ResticRepo::open(storage, &cfg.generated_id, edge_key)?; + restore(&repo, cfg, snapshot_id, logger).await + } + .await; + FileLock::release(&cfg.generated_id).await?; + result +} diff --git a/src/services/restore/archive.rs b/src/services/restore/archive.rs index 3119522..73412ec 100644 --- a/src/services/restore/archive.rs +++ b/src/services/restore/archive.rs @@ -60,6 +60,11 @@ impl RestoreService { archive = decrypted; } + if matches!(db_type, DbType::Files) { + logger.log("info", "Files archive ready for extraction".to_string()); + return Ok(archive); + } + if matches!(db_type, DbType::DockerVolume) { let raw_tar = tmp_path.join("volume.tar"); crate::utils::compress::gunzip_to_file(archive.as_path(), &raw_tar).await?; diff --git a/src/services/restore/dispatcher.rs b/src/services/restore/dispatcher.rs index 61f35fd..2cf7b2f 100644 --- a/src/services/restore/dispatcher.rs +++ b/src/services/restore/dispatcher.rs @@ -15,6 +15,19 @@ impl RestoreService { return; }; + if db.data.restore.engine.as_deref() == Some("restic") { + let snapshot_id = db.data.restore.snapshot_id.clone(); + let storage = db.data.restore.storage.clone(); + let service = Self { ctx: self.ctx.clone() }; + let db_cfg = cfg.clone(); + tokio::spawn(async move { + if let Err(e) = service.execute_restic_restore(db_cfg, snapshot_id, storage).await { + error!("Snapshot restore failed: {}", e); + } + }); + return; + } + let Some(file_to_restore) = db.data.restore.file.clone() else { error!("restore file not found"); return; diff --git a/src/services/restore/executor.rs b/src/services/restore/executor.rs index 49fbcd2..a38597b 100644 --- a/src/services/restore/executor.rs +++ b/src/services/restore/executor.rs @@ -1,6 +1,9 @@ +use super::models::RestoreResult; use super::service::RestoreService; +use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; +use crate::services::restic; use anyhow::Result; use std::sync::Arc; use std::time::Instant; @@ -43,4 +46,40 @@ impl RestoreService { Ok(()) } + + pub async fn execute_restic_restore( + &self, + cfg: DatabaseConfig, + snapshot_id: Option, + storage: Option, + ) -> Result<()> { + let logger = Arc::new(JobLogger::new()); + let start = Instant::now(); + logger.log("info", "Snapshot restore job started".to_string()); + + let outcome = match (snapshot_id, storage) { + (Some(id), Some(storage)) => { + restic::restore::run(&self.ctx.edge_key, &cfg, &id, &storage, &logger).await + } + _ => Err(anyhow::anyhow!( + "incomplete snapshot restore payload (snapshotId or storage missing)" + )), + }; + let status = match outcome { + Ok(()) => { + logger.log("info", "Snapshot restore job finished".to_string()); + "success" + } + Err(e) => { + logger.log("error", format!("Snapshot restore failed: {e:#}")); + "failed" + } + }; + + let duration_ms = start.elapsed().as_millis() as f64; + let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries(); + let result = RestoreResult { generated_id: cfg.generated_id.clone(), status: status.into() }; + self.send_result(result, logs, duration_ms).await?; + Ok(()) + } } diff --git a/src/services/status.rs b/src/services/status.rs index 8779adb..5e9ab9f 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -6,11 +6,12 @@ use crate::services::api::endpoints::status::DatabasePayload; use crate::services::api::models::agent::status::DatabaseStatus; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::api::models::agent::status::PingResult; -use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig}; +use crate::services::config::{build_config, files_method, DatabaseConfig, DbType, InputDatabaseConfig}; use crate::settings::CONFIG; use crate::utils::file::decrypt_json_gcm; use futures_util::future::try_join_all; use reqwest::Client; +use std::collections::HashSet; use std::error::Error; use std::sync::Arc; use tracing::info; @@ -35,6 +36,29 @@ pub fn resolve_dashboard_config( Ok(()) } +/// A storage channel sent encrypted as a one-element JSON array. +fn decrypt_storage(ciphertext: &str, master_key_b64: &str, what: &str) -> Result, String> { + let plaintext = decrypt_json_gcm(ciphertext, master_key_b64) + .map_err(|e| format!("Failed to decrypt {what} storage: {e}"))?; + let storages: Vec = serde_json::from_slice(&plaintext) + .map_err(|e| format!("Failed to parse {what} storage: {e}"))?; + Ok(storages.into_iter().next()) +} + +/// Decrypts the storage channel of a snapshot restore (`restore.storageCiphertext`). +pub fn resolve_restore_storage(status: &mut DatabaseStatus, master_key_b64: &str) -> Result<(), String> { + if let Some(ciphertext) = status.data.restore.storage_ciphertext.clone() { + status.data.restore.storage = decrypt_storage(&ciphertext, master_key_b64, "restore")?; + } + Ok(()) +} + +/// `method` reported in the ping for a files source declared in databases.json; +/// `None` for dashboard-managed sources (the dashboard owns their method) and other dbms. +pub fn payload_method(db: &DatabaseConfig, local_ids: &HashSet) -> Option<&'static str> { + (matches!(db.db_type, DbType::Files) && local_ids.contains(&db.generated_id)).then(|| files_method(db)) +} + pub struct StatusService { ctx: Arc, client: Client, @@ -48,7 +72,11 @@ impl StatusService { } } - pub async fn ping(&self, databases: &[DatabaseConfig]) -> Result> { + pub async fn ping( + &self, + databases: &[DatabaseConfig], + local_ids: &HashSet, + ) -> Result> { let edge_key = &self.ctx.edge_key; let databases_payload: Vec = @@ -63,6 +91,7 @@ impl StatusService { dbms: &db.db_type.as_str(), generated_id: &db.generated_id, ping_status: reachable, + method: payload_method(db, local_ids), }) })) .await?; @@ -92,6 +121,9 @@ impl StatusService { if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) { tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id); } + if let Err(e) = resolve_restore_storage(db, &edge_key.master_key_b64) { + tracing::warn!("Snapshot restore storage unreadable for {}: {e}", db.generated_id); + } } Ok(result) } diff --git a/src/services/storage/providers/google_cloud_storage/helpers.rs b/src/services/storage/providers/google_cloud_storage/helpers.rs index 431cdba..eee1689 100644 --- a/src/services/storage/providers/google_cloud_storage/helpers.rs +++ b/src/services/storage/providers/google_cloud_storage/helpers.rs @@ -8,25 +8,26 @@ use google_cloud_storage::client::Storage; use google_cloud_storage::streaming_source::{SizeHint, StreamingSource}; use std::pin::Pin; -pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result { +/// Service-account key JSON (keys in alphabetical order so the rclone mapper +/// emits byte-identical output to the dashboard's). +pub fn service_account_key(cfg: &GoogleCloudStorageProviderConfig) -> serde_json::Value { // Service-account JSON stores the PEM with `\n` escape sequences. When the key is // carried through config as a JSON string those can arrive as literal two-char `\n` - // sequences rather than real newlines, so the PEM parser finds no `-----BEGIN-----` - // line ("no items found"). Normalize them back to real newlines. A PEM that already - // has real newlines contains no literal `\n` pairs, so this is a no-op for it. + // sequences rather than real newlines. Normalize them back to real newlines. let private_key = cfg.private_key.replace("\\n", "\n"); - - let key = serde_json::json!({ - "type": "service_account", - "project_id": cfg.project_id, + serde_json::json!({ "client_email": cfg.client_email, "private_key": private_key, "private_key_id": "", + "project_id": cfg.project_id, "token_uri": "https://oauth2.googleapis.com/token", + "type": "service_account", "universe_domain": "googleapis.com", - }); + }) +} - google_cloud_auth::credentials::service_account::Builder::new(key) +pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result { + google_cloud_auth::credentials::service_account::Builder::new(service_account_key(cfg)) .build() .context("failed to build GCS service account credentials") } diff --git a/src/services/storage/providers/google_cloud_storage/mod.rs b/src/services/storage/providers/google_cloud_storage/mod.rs index 0220a09..fa76cc5 100644 --- a/src/services/storage/providers/google_cloud_storage/mod.rs +++ b/src/services/storage/providers/google_cloud_storage/mod.rs @@ -1,5 +1,5 @@ pub mod helpers; -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/google_drive/mod.rs b/src/services/storage/providers/google_drive/mod.rs index e127414..c8089e6 100644 --- a/src/services/storage/providers/google_drive/mod.rs +++ b/src/services/storage/providers/google_drive/mod.rs @@ -1,5 +1,5 @@ mod helpers; -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/rclone/helpers.rs b/src/services/storage/providers/rclone/helpers.rs index 41458d6..98a1aeb 100644 --- a/src/services/storage/providers/rclone/helpers.rs +++ b/src/services/storage/providers/rclone/helpers.rs @@ -10,7 +10,9 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; use tracing::info; -const BLOCKED_BACKEND_TYPES: [&str; 13] = [ +/// Entries must not contain spaces: the type is compared with its spaces removed (rclone resolves a +/// backend by name, registered prefix, or name without spaces: `google photos` = `googlephotos`). +const BLOCKED_BACKEND_TYPES: [&str; 14] = [ "local", "alias", "crypt", @@ -24,67 +26,133 @@ const BLOCKED_BACKEND_TYPES: [&str; 13] = [ "memory", "http", "googlephotos", + "gphotos", ]; -fn sections(config_text: &str) -> Vec<(String, Option)> { - let mut out: Vec<(String, Option)> = Vec::new(); - - for line in config_text.lines() { - let line = line.trim(); - - if line.starts_with('[') && line.ends_with(']') && line.len() > 2 { - out.push((line[1..line.len() - 1].trim().to_string(), None)); - continue; - } - - let Some((key, value)) = line.split_once('=') else { - continue; - }; - - if key.trim().eq_ignore_ascii_case("type") - && let Some(current) = out.last_mut() - && current.1.is_none() - { - current.1 = Some(value.trim().to_ascii_lowercase()); - } - } +fn valid_remote_name(name: &str) -> bool { + let mut chars = name.chars(); + chars.next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '+' | '@' | '-')) +} - out +fn valid_key(key: &str) -> bool { + !key.is_empty() && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') } +fn forbidden_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + matches!( + key.as_str(), + "ssh" | "env_auth" | "use_msi" | "use_az" | "use_kerberos" | "kerberos_ccache" | "key_use_agent" | "set_env" | "unix_socket" + ) || key.ends_with("_command") + || key.ends_with("_file") + || key.ends_with("_path") +} +/// Strict check for user-pasted rclone configs (never for configs Portabase builds itself). +/// Exactly one `[remote]` of plain `key = value` lines: rclone and a naive parser must read it +/// identically, and nothing may run a local command or read host credentials. pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> { - let sections = sections(config_text); + let mut names = Vec::new(); + let mut pairs = Vec::new(); + let mut before_header = None; - if sections.is_empty() { - bail!("rclone config contains no remote sections"); + for (i, line) in config_text.split('\n').enumerate() { + let (n, line) = (i + 1, line.trim()); + if line.is_empty() || line.starts_with(['#', ';']) { + continue; + } + if line.starts_with('[') && line.ends_with(']') { + names.push(line[1..line.len() - 1].trim()); + } else if let Some((key, value)) = line.split_once('=').filter(|(k, _)| !k.is_empty()) { + if names.is_empty() { + before_header.get_or_insert(n); + } + pairs.push((key.trim(), value.trim())); + } else { + bail!("rclone config line {n} is not a [section] header or a 'key = value' pair"); + } } - for (name, backend) in §ions { - let Some(backend) = backend else { continue }; - if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) { - bail!("rclone backend type '{backend}' is not allowed (remote '{name}')"); + if let Some(n) = before_header { + bail!("rclone config line {n} appears before any [section]"); + } + if names.len() != 1 { + bail!("rclone config must contain exactly one [section] (found {})", names.len()); + } + let name = names[0]; + if !valid_remote_name(name) { + bail!("invalid rclone remote name '{name}'"); + } + for (key, _) in &pairs { + if !valid_key(key) { + bail!("invalid rclone config key '{key}' (remote '{name}')"); } } - - if !sections.iter().any(|(name, _)| name == remote_name) { - let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect(); - bail!( - "remote '{remote_name}' is not defined in the rclone config (available: {})", - available.join(", ") - ); + // rclone (goconfig) unquotes `value` and `"""value"""` and drops what follows the closing quote. + for (key, value) in &pairs { + if value.starts_with('`') || value.starts_with("\"\"\"") { + bail!("rclone config value for key '{key}' must not be quoted (remote '{name}')"); + } + } + let mut seen = std::collections::HashSet::new(); + for (key, _) in &pairs { + if !seen.insert(key.to_ascii_lowercase()) { + bail!("duplicate rclone config key '{key}' (remote '{name}')"); + } + } + for (key, _) in &pairs { + if forbidden_key(key) { + bail!("rclone config key '{key}' is not allowed (remote '{name}')"); + } + } + let Some((_, backend)) = pairs.iter().find(|(k, _)| k.eq_ignore_ascii_case("type")) else { + bail!("rclone remote '{name}' has no type"); + }; + let backend = backend.to_ascii_lowercase(); + if backend.is_empty() || !backend.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == ' ') { + bail!("invalid rclone backend type '{backend}' (remote '{name}')"); + } + let compact = backend.replace(' ', ""); + if BLOCKED_BACKEND_TYPES.contains(&compact.as_str()) { + bail!("rclone backend type '{backend}' is not allowed (remote '{name}')"); + } + // The other OCI providers read the host's OCI config or instance identity. Exact match on + // purpose: rclone 1.75.1 compares the key and the value case-sensitively, and falls back to + // host credentials for `PROVIDER = no_auth` (key ignored) and `provider = NO_AUTH` (unknown value). + if compact == "oracleobjectstorage" { + if !pairs.contains(&("provider", "no_auth")) { + bail!("rclone oracleobjectstorage remotes must use provider = no_auth (remote '{name}')"); + } + } + if remote_name != name { + bail!("remote '{remote_name}' is not defined in the rclone config (available: {name})"); } Ok(()) } +/// `rclone obscure -` reads the password from stdin, so it never shows up in argv. pub fn obscure_password(password: &str) -> Result { - let out = std::process::Command::new("rclone") + let mut child = std::process::Command::new("rclone") .arg("obscure") - .arg(password) - .output() + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .context("failed to spawn rclone (is the binary installed in this image?)")?; + // Dropping the handle closes stdin, which is what ends rclone's read. + child + .stdin + .take() + .context("rclone stdin unavailable")? + .write_all(password.as_bytes()) + .context("failed to write the password to rclone obscure")?; + + let out = child.wait_with_output().context("failed to wait for rclone obscure")?; + if !out.status.success() { bail!( "rclone obscure failed: {}", diff --git a/src/services/storage/providers/rclone/mod.rs b/src/services/storage/providers/rclone/mod.rs index de6df63..0004651 100644 --- a/src/services/storage/providers/rclone/mod.rs +++ b/src/services/storage/providers/rclone/mod.rs @@ -1,5 +1,6 @@ pub mod helpers; pub mod models; +pub mod target; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/rclone/target.rs b/src/services/storage/providers/rclone/target.rs new file mode 100644 index 0000000..9f13292 --- /dev/null +++ b/src/services/storage/providers/rclone/target.rs @@ -0,0 +1,137 @@ +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::storage::providers::azure_blob::models::AzureBlobProviderConfig; +use crate::services::storage::providers::google_cloud_storage::helpers::service_account_key; +use crate::services::storage::providers::google_cloud_storage::models::GoogleCloudStorageProviderConfig; +use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; +use crate::services::storage::providers::rclone::helpers::{build_rclone_config, validate_config}; +use crate::services::storage::providers::rclone::models::RcloneProviderConfig; +use crate::services::storage::providers::s3::models::S3ProviderConfig; +use crate::services::storage::providers::sftp::helpers::build_sftp_config; +use crate::services::storage::providers::sftp::models::SftpProviderConfig; +use anyhow::{Context, Result, bail}; +use tempfile::TempPath; + +/// A storage channel expressed as an rclone remote. Temp files the config refers +/// to (e.g. an sftp key) live as long as this value. +pub struct RcloneTarget { + pub config_text: String, + pub remote_name: String, + pub base_path: String, + _guards: Vec, +} + +impl RcloneTarget { + fn new(config_text: String, remote_name: &str, base_path: String) -> Self { + Self { config_text, remote_name: remote_name.to_string(), base_path, _guards: Vec::new() } + } +} + +pub fn rclone_target(storage: &DatabaseStorage) -> Result { + let config = storage.config.clone(); + match storage.provider.as_str() { + "s3" => s3(config.try_into().context("invalid s3 storage config")?), + "blob" => blob(config.try_into().context("invalid azure blob storage config")?), + "google-cloud-storage" => gcs(config.try_into().context("invalid google cloud storage config")?), + "google-drive" => drive(config.try_into().context("invalid google drive storage config")?), + "rclone" => user_rclone(config.try_into().context("invalid rclone storage config")?), + "sftp" => sftp(config.try_into().context("invalid sftp storage config")?), + "local" => bail!("a local storage channel cannot be used as an rclone target"), + other => bail!("unknown storage provider '{other}'"), + } +} + +fn s3(c: S3ProviderConfig) -> Result { + let scheme = if c.ssl { "https" } else { "http" }; + let endpoint = match c.port.as_deref().map(str::trim).filter(|p| !p.is_empty()) { + Some(port) => format!("{scheme}://{}:{port}", c.end_point_url), + None => format!("{scheme}://{}", c.end_point_url), + }; + let region = c.region.as_deref().map(str::trim).filter(|r| !r.is_empty()).unwrap_or("us-east-1"); + let fields = [ + ("type", "s3".to_string()), + ("provider", "Other".to_string()), + ("access_key_id", c.access_key), + ("secret_access_key", c.secret_key), + ("endpoint", endpoint), + ("region", region.to_string()), + ("force_path_style", "true".to_string()), + ("no_check_bucket", "true".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("s3", &fields)?, "s3", c.bucket_name)) +} + +fn blob(c: AzureBlobProviderConfig) -> Result { + let resolved = c.resolve().context("invalid azure blob storage config")?; + if resolved.account_key.trim().is_empty() { + bail!("azure blob needs an account key to be used as an rclone target (SAS-only connection strings are not supported)"); + } + let fields = [ + ("type", "azureblob".to_string()), + ("account", resolved.account_name), + ("key", resolved.account_key), + ("endpoint", resolved.blob_endpoint), + // Native uploads never create the container; neither should rclone. + ("no_check_container", "true".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("blob", &fields)?, "blob", c.container_name)) +} + +fn gcs(c: GoogleCloudStorageProviderConfig) -> Result { + let mut fields = vec![ + ("type", "google cloud storage".to_string()), + ("project_number", c.project_id.clone()), + ]; + match c.api_endpoint.as_deref().map(str::trim).filter(|e| !e.is_empty()) { + // A custom endpoint means the fake-gcs emulator, which does not verify credentials. + // rclone's `endpoint` is the JSON API base path, not the bare host. + Some(endpoint) => { + fields.push(("anonymous", "true".to_string())); + fields.push(("endpoint", format!("{}/storage/v1/", endpoint.trim_end_matches('/')))); + } + None => fields.push(( + "service_account_credentials", + serde_json::to_string(&service_account_key(&c))?, + )), + } + // Uniform bucket-level access (the GCS default) rejects rclone's default + // `predefinedAcl=private`; and native uploads never create the bucket. + fields.push(("bucket_policy_only", "true".to_string())); + fields.push(("no_check_bucket", "true".to_string())); + Ok(RcloneTarget::new( + build_rclone_config("google-cloud-storage", &fields)?, + "google-cloud-storage", + c.bucket_name, + )) +} + +fn drive(c: GoogleDriveProviderConfig) -> Result { + let token = serde_json::json!({ + "access_token": "", + "expiry": "0001-01-01T00:00:00Z", + "refresh_token": c.refresh_token, + "token_type": "Bearer", + }); + let fields = [ + ("type", "drive".to_string()), + ("client_id", c.client_id), + ("client_secret", c.client_secret), + ("scope", "drive.file".to_string()), + ("token", serde_json::to_string(&token)?), + ("root_folder_id", c.folder_id), + // Deletes (retention, restic prune) must free space, not fill the Drive trash. + ("use_trash", "false".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("google-drive", &fields)?, "google-drive", String::new())) +} + +fn user_rclone(c: RcloneProviderConfig) -> Result { + validate_config(&c.config_text, &c.remote_name)?; + Ok(RcloneTarget::new(c.config_text, &c.remote_name, c.remote_path)) +} + +fn sftp(c: SftpProviderConfig) -> Result { + let (config_text, key_file) = build_sftp_config(&c)?; + let mut target = RcloneTarget::new(config_text, "sftp", c.remote_path); + target._guards.extend(key_file.map(|f| f.into_temp_path())); + Ok(target) +} diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index 4d79f24..653f30f 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -1,4 +1,4 @@ -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/sftp/mod.rs b/src/services/storage/providers/sftp/mod.rs index 322da3c..4e166af 100644 --- a/src/services/storage/providers/sftp/mod.rs +++ b/src/services/storage/providers/sftp/mod.rs @@ -6,8 +6,7 @@ use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::services::storage::providers::rclone::helpers::{rcat, remote_target, write_config}; -use crate::services::storage::providers::sftp::helpers::build_sftp_config; -use crate::services::storage::providers::sftp::models::SftpProviderConfig; +use crate::services::storage::providers::rclone::target::rclone_target; use crate::utils::common::BackupMethod; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; @@ -53,19 +52,13 @@ impl StorageProvider for SftpProvider { } }; - let config: SftpProviderConfig = match storage.clone().config.try_into() { - Ok(c) => c, + // `target` owns the sftp key file: keep it alive until `rcat` returns. + let target = match rclone_target(storage) { + Ok(t) => t, Err(e) => { - error!("sftp config deserialization failed: {}", e); - return failed(&storage_id, e, Some(total_size)); - } - }; - - let (config_text, _key_file) = match build_sftp_config(&config) { - Ok(v) => v, - Err(e) => { - error!("sftp config build failed: {}", e); - return failed(&storage_id, e, Some(total_size)); + // `{:#}` keeps the whole anyhow chain (the serde cause), which is what the user sees. + error!("sftp config build failed: {e:#}"); + return failed(&storage_id, format!("{e:#}"), Some(total_size)); } }; @@ -82,7 +75,7 @@ impl StorageProvider for SftpProvider { let file_name = full_file_name(encrypt); let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref()); - let config_file = match write_config(&config_text) { + let config_file = match write_config(&target.config_text) { Ok(f) => f, Err(e) => { error!("sftp config write failed: {}", e); @@ -90,11 +83,11 @@ impl StorageProvider for SftpProvider { } }; - let target = remote_target("sftp", &config.remote_path, &remote_file_path); + let remote = remote_target(&target.remote_name, &target.base_path, &remote_file_path); - info!("Starting sftp (rclone) upload to {}", target); + info!("Starting sftp (rclone) upload to {}", remote); - match rcat(config_file.path(), &target, upload.stream).await { + match rcat(config_file.path(), &remote, upload.stream).await { Ok(()) => { info!("sftp upload successful: {}", remote_file_path); UploadResult { diff --git a/src/services/sync/backup.rs b/src/services/sync/backup.rs new file mode 100644 index 0000000..a708d95 --- /dev/null +++ b/src/services/sync/backup.rs @@ -0,0 +1,147 @@ +//! Sync as a backup method: one `rclone sync` per storage channel, each reported +//! as its own `backup_storage` row (`engine = sync`), like restic snapshots. +use super::rclone::{replica_target, sync_dir}; +use super::stats::SyncStats; +use crate::core::context::Context as CoreContext; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::backup::models::UploadResult; +use crate::services::config::DatabaseConfig; +use crate::services::restic::password::{LOCAL_STORAGE_USER, local_storage_password}; +use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password}; +use crate::services::storage::providers::rclone::target::rclone_target; +use crate::utils::edge_key::EdgeKey; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::Result; + +/// Syncs every storage in turn, under the source's backup lock. Never errors: +/// each storage reports its own `backup_storage` row. +pub async fn run( + ctx: &CoreContext, + cfg: &DatabaseConfig, + storages: &[DatabaseStorage], + backup_id: &str, + logger: &JobLogger, +) -> Vec { + if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { + logger.log("error", format!("Sync aborted: {e}")); + return Vec::new(); + } + let mut results = Vec::with_capacity(storages.len()); + for storage in storages { + results.push(one_storage(ctx, cfg, storage, backup_id, logger).await); + } + if let Err(e) = FileLock::release(&cfg.generated_id).await { + logger.log("warn", format!("Failed to release the backup lock: {e}")); + } + results +} + +fn folder(storage: &DatabaseStorage) -> &str { + storage + .folder_name + .as_deref() + .map(|f| f.trim().trim_matches('/')) + .filter(|f| !f.is_empty()) + .unwrap_or("backups") +} + +/// `/sync//current`, relative to the channel's base path. +pub fn replica_path(storage: &DatabaseStorage, generated_id: &str) -> String { + format!("{}/sync/{generated_id}/current", folder(storage)) +} + +const LOCAL_REMOTE: &str = "pblocal"; + +/// rclone config of the dashboard's local storage (`rclone serve webdav` at `/storage/sync`). +pub fn local_sync_config(edge_key: &EdgeKey) -> Result { + build_rclone_config( + LOCAL_REMOTE, + &[ + ("type", "webdav".into()), + ("url", format!("{}/storage/sync", edge_key.server_url.trim_end_matches('/'))), + // rclone's own WebDAV extensions keep modification times: unchanged files are skipped. + ("vendor", "rclone".into()), + ("user", LOCAL_STORAGE_USER.into()), + ("pass", obscure_password(&local_storage_password(&edge_key.master_key_b64)?)?), + ], + ) +} + +async fn replicate(ctx: &CoreContext, cfg: &DatabaseConfig, storage: &DatabaseStorage, logger: &JobLogger) -> Result { + if storage.provider == "local" { + let dest = format!("{LOCAL_REMOTE}:{}/current", cfg.generated_id); + return sync_dir(&local_sync_config(&ctx.edge_key)?, &dest, cfg, logger).await; + } + let target = rclone_target(storage)?; + let dest = replica_target(&target.remote_name, &target.base_path, folder(storage), &cfg.generated_id); + sync_dir(&target.config_text, &dest, cfg, logger).await +} + +pub(crate) async fn one_storage( + ctx: &CoreContext, + cfg: &DatabaseConfig, + storage: &DatabaseStorage, + backup_id: &str, + logger: &JobLogger, +) -> UploadResult { + let agent_id = ctx.edge_key.agent_id.clone(); + let failed = |error: String| UploadResult { + storage_id: storage.id.clone(), + success: false, + error: Some(error), + remote_file_path: None, + total_size: None, + }; + + let backup_storage_id = match ctx + .api + .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("sync")) + .await + { + Ok(Some(response)) => response.backup_storage.id, + Ok(None) => { + logger.log("error", "Upload init returned empty response"); + return failed("backup_upload_init returned empty response".into()); + } + Err(e) => { + logger.log("error", format!("Upload init failed: {e}")); + return failed("backup_upload_init failed".into()); + } + }; + + match replicate(ctx, cfg, storage, logger).await { + Ok(stats) => { + let path = replica_path(storage, &cfg.generated_id); + match ctx + .api + .backup_upload_sync_status(agent_id, cfg.generated_id.clone(), backup_storage_id, path.clone(), &stats, backup_id) + .await + { + Ok(_) => UploadResult { + storage_id: storage.id.clone(), + success: true, + error: None, + remote_file_path: Some(path), + // No total size: rclone does not report the replica's size (backups.file_size stays null). + total_size: None, + }, + Err(e) => { + logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); + failed(e.to_string()) + } + } + } + Err(e) => { + logger.log("error", format!("Sync to storage {} failed: {e:#}", storage.id)); + if let Err(err) = ctx + .api + .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) + .await + { + logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); + } + failed(format!("{e:#}")) + } + } +} diff --git a/src/services/sync/excludes.rs b/src/services/sync/excludes.rs new file mode 100644 index 0000000..fc24e90 --- /dev/null +++ b/src/services/sync/excludes.rs @@ -0,0 +1,65 @@ +use crate::services::restic::excludes::bracket_negation; + +/// P0 exclude patterns → `rclone sync --exclude` values. rclone filters are +/// relative to the sync root (spike 2026-10-03: 19/19 patterns give the P0 file set). +pub fn sync_excludes(patterns: &[String]) -> Vec { + let mut out = Vec::new(); + for raw in patterns { + let pattern = raw.trim(); + let (anchored, rest) = match pattern.strip_prefix('/') { + Some(rest) => (true, rest), + None => (false, pattern), + }; + let mut rest = rest.trim_end_matches('/'); + if !anchored { + // Unanchored rclone patterns already match at any depth; a leading + // `**/` would stop them from matching at the root. + while let Some(stripped) = rest.strip_prefix("**/") { + rest = stripped; + } + } + if rest.is_empty() { + continue; + } + let rest = bracket_negation(rest); + // Collapse repeated /**/ into a single /**/ + let rest = collapse_double_stars(&rest); + let full = if anchored { format!("/{rest}") } else { rest }; + for variant in zero_dir_variants(&full) { + out.push(variant.clone()); + out.push(format!("{variant}/**")); + } + } + out +} + +/// Collapse repeated `/**/` sequences into a single `/**/`. +/// E.g., `a/**/**/b` becomes `a/**/b`. +fn collapse_double_stars(pattern: &str) -> String { + let mut result = pattern.to_string(); + while result.contains("/**/**/") { + result = result.replace("/**/**/", "/**/"); + } + result +} + +/// rclone's `**` between slashes needs at least one directory; globset's also +/// matches none. Emit every combination of each `/**/` kept or collapsed to `/`. +fn zero_dir_variants(pattern: &str) -> Vec { + let parts: Vec<&str> = pattern.split("/**/").collect(); + let gaps = parts.len() - 1; + // ponytail: 2^gaps variants; past 4 gaps only "all kept" and "all collapsed". + if gaps > 4 { + return vec![pattern.to_string(), pattern.replace("/**/", "/")]; + } + (0..1u32 << gaps) + .map(|mask| { + let mut variant = parts[0].to_string(); + for (i, part) in parts[1..].iter().enumerate() { + variant.push_str(if mask & (1 << i) == 0 { "/**/" } else { "/" }); + variant.push_str(part); + } + variant + }) + .collect() +} diff --git a/src/services/sync/mod.rs b/src/services/sync/mod.rs new file mode 100644 index 0000000..b8501ae --- /dev/null +++ b/src/services/sync/mod.rs @@ -0,0 +1,6 @@ +//! Sync mode: a destination kept identical to a files source (`rclone sync`). +//! Replication, not backup: deletions propagate. +pub mod backup; +pub mod excludes; +pub mod stats; +pub mod rclone; diff --git a/src/services/sync/rclone.rs b/src/services/sync/rclone.rs new file mode 100644 index 0000000..1532b3a --- /dev/null +++ b/src/services/sync/rclone.rs @@ -0,0 +1,86 @@ +use super::excludes::sync_excludes; +use super::stats::{SyncStats, parse_log}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; +use anyhow::{Context, Result, anyhow, bail}; +use std::process::Stdio; +use tokio::process::Command; + +/// `://sync//current` +pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generated_id: &str) -> String { + remote_target(remote_name, base_path, &format!("{folder}/sync/{generated_id}/current")) +} + +/// Mirrors `cfg.path` onto `dest` (`:`). rclone skips its deletions when an error +/// happens before the delete phase ("not deleting files as there were IO errors"); +/// an error during the delete phase can leave the replica partially updated. +/// An empty source is refused so an unmounted volume cannot wipe the replica. +pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logger: &JobLogger) -> Result { + let root = std::fs::canonicalize(&cfg.path) + .with_context(|| format!("cannot read source directory {}", cfg.path))?; + if !root.is_dir() { + bail!("source path {} is not a directory", root.display()); + } + if std::fs::read_dir(&root) + .with_context(|| format!("cannot read source directory {}", root.display()))? + .next() + .is_none() + { + bail!( + "source directory {} is empty; refusing to sync so an unmounted volume cannot wipe the replica", + root.display() + ); + } + let config = write_config(config_text)?; + + let mut cmd = Command::new("rclone"); + cmd.arg("--config") + .arg(config.path()) + .arg("sync") + .arg(&root) + .arg(dest) + .args([ + "--links", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1", + // An unreachable endpoint fails in ~1–2 min instead of blocking the next runs. + "--contimeout", "30s", "--low-level-retries", "3", + ]); + if one_file_system(cfg) { + cmd.arg("--one-file-system"); + } + for pattern in sync_excludes(&exclude_patterns(cfg)) { + cmd.arg("--exclude").arg(pattern); + } + + logger.log("info", format!("Syncing {} to {dest}", root.display())); + let out = cmd + .stdin(Stdio::null()) + .kill_on_drop(true) + .output() + .await + .map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => anyhow!("rclone binary not found"), + _ => anyhow!(e).context("failed to start rclone"), + })?; + + let log = parse_log(&String::from_utf8_lossy(&out.stderr)); + for error in &log.errors { + logger.log("warn", format!("rclone: {error}")); + } + if !out.status.success() { + let reason = log + .stats + .as_ref() + .and_then(|s| s.last_error.clone()) + .or_else(|| log.errors.last().cloned()) + .unwrap_or_else(|| "no error message".to_string()); + bail!("rclone sync failed ({}): {reason}; the replica may be partially updated", out.status); + } + let stats = log.stats.unwrap_or_default(); + logger.log( + "info", + format!("Synced: {} file(s) transferred ({} bytes), {} deleted", stats.transfers, stats.bytes, stats.deletes), + ); + Ok(stats) +} diff --git a/src/services/sync/stats.rs b/src/services/sync/stats.rs new file mode 100644 index 0000000..6c1246d --- /dev/null +++ b/src/services/sync/stats.rs @@ -0,0 +1,56 @@ +use serde::Deserialize; +use serde_json::Value; + +/// Final `stats` object of `rclone --use-json-log --stats 1h --stats-log-level NOTICE`. +#[derive(Debug, Default, Deserialize, PartialEq)] +pub struct SyncStats { + #[serde(default)] + pub bytes: u64, + #[serde(default)] + pub transfers: u64, + #[serde(default)] + pub deletes: u64, + #[serde(default)] + pub errors: u64, + #[serde(default, rename = "lastError")] + pub last_error: Option, +} + +/// What `rclone sync` printed on stderr: its last stats object and its error +/// messages (JSON `level: error|critical|alert|emergency` lines and any plain-text line). +#[derive(Debug, Default)] +pub struct SyncLog { + pub stats: Option, + pub errors: Vec, +} + +pub fn parse_log(stderr: &str) -> SyncLog { + let mut log = SyncLog::default(); + for line in stderr.lines().map(str::trim).filter(|l| !l.is_empty()) { + let Ok(value) = serde_json::from_str::(line) else { + if !log.errors.contains(&line.to_string()) { + log.errors.push(line.to_string()); + } + continue; + }; + if let Some(stats) = value.get("stats").and_then(|s| serde_json::from_value(s.clone()).ok()) { + log.stats = Some(stats); + } + if let Some(level) = value.get("level").and_then(Value::as_str) { + if matches!(level, "error" | "critical" | "alert" | "emergency") { + if let Some(msg) = value.get("msg").and_then(Value::as_str) { + let msg_str = msg.to_string(); + if !log.errors.contains(&msg_str) { + log.errors.push(msg_str); + } + } + } + } + } + // Keep only the last 20 messages + if log.errors.len() > 20 { + log.errors = log.errors.into_iter().rev().take(20).collect::>(); + log.errors.reverse(); + } + log +} diff --git a/src/tests/domain/files.rs b/src/tests/domain/files.rs new file mode 100644 index 0000000..d4695ea --- /dev/null +++ b/src/tests/domain/files.rs @@ -0,0 +1,616 @@ +use crate::domain::files::matcher::ExcludeMatcher; +use std::path::Path; + +fn matcher(patterns: &[&str]) -> ExcludeMatcher { + let owned: Vec = patterns.iter().map(|p| p.to_string()).collect(); + ExcludeMatcher::new(&owned).unwrap() +} + +#[test] +fn matcher_unanchored_matches_any_depth() { + let m = matcher(&["node_modules", "*.log"]); + assert!(m.is_excluded(Path::new("node_modules"))); + assert!(m.is_excluded(Path::new("app/node_modules"))); + assert!(m.is_excluded(Path::new("x.log"))); + assert!(m.is_excluded(Path::new("a/b/x.log"))); + assert!(!m.is_excluded(Path::new("src/main.rs"))); +} + +#[test] +fn matcher_anchored_matches_root_only() { + let m = matcher(&["/cache", "/build/"]); + assert!(m.is_excluded(Path::new("cache"))); + assert!(!m.is_excluded(Path::new("app/cache"))); + assert!(m.is_excluded(Path::new("build"))); +} + +#[test] +fn matcher_star_does_not_cross_separator_but_double_star_does() { + let m = matcher(&["/logs/*.txt", "/data/**/tmp"]); + assert!(m.is_excluded(Path::new("logs/a.txt"))); + assert!(!m.is_excluded(Path::new("logs/sub/a.txt"))); + assert!(m.is_excluded(Path::new("data/tmp"))); + assert!(m.is_excluded(Path::new("data/a/b/tmp"))); +} + +#[test] +fn matcher_ignores_blank_patterns_and_rejects_invalid_ones() { + let m = matcher(&["", " "]); + assert!(!m.is_excluded(Path::new("anything"))); + assert!(ExcludeMatcher::new(&["[".to_string()]).is_err()); +} + +use crate::domain::files::backup; +use crate::services::backup::logger::JobLogger; +use crate::services::config::{DatabaseConfig, DbType}; +use flate2::read::GzDecoder; +use std::collections::HashMap; +use std::fs; +use std::io::Read; +use std::sync::Arc; + +pub(crate) fn files_config(root: &Path, exclude: &[&str]) -> DatabaseConfig { + let mut options = HashMap::new(); + options.insert("exclude".to_string(), serde_json::json!(exclude)); + DatabaseConfig { + name: "files-test".to_string(), + database: String::new(), + db_type: DbType::Files, + username: String::new(), + password: String::new(), + port: 0, + host: String::new(), + generated_id: uuid::Uuid::new_v4().to_string(), + path: root.to_string_lossy().into_owned(), + max_packet_size: String::new(), + volume_name: String::new(), + container_name: None, + options, + } +} + +fn sample_tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::create_dir_all(root.join("app/node_modules")).unwrap(); + fs::write(root.join("app/node_modules/dep.js"), "dep").unwrap(); + fs::write(root.join("app/index.js"), "app").unwrap(); + std::os::unix::fs::symlink("docs/a.txt", root.join("link")).unwrap(); +} + +#[tokio::test] +async fn backup_writes_tar_gz_without_excluded_paths() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let cfg = files_config(src.path(), &["node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + + let archive = backup::run( + cfg.clone(), + out.path().to_path_buf(), + Arc::new(JobLogger::new()), + ) + .await + .unwrap(); + + assert_eq!( + archive.file_name().unwrap().to_string_lossy(), + format!("{}.tar.gz", cfg.generated_id) + ); + + let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(&archive).unwrap())); + let mut files = Vec::new(); + let mut link_is_symlink = false; + let mut a_txt = String::new(); + for entry in ar.entries().unwrap() { + let mut entry = entry.unwrap(); + let name = entry + .path() + .unwrap() + .to_string_lossy() + .trim_end_matches('/') + .to_string(); + if name == "link" { + link_is_symlink = entry.header().entry_type().is_symlink(); + } + if name == "docs/a.txt" { + entry.read_to_string(&mut a_txt).unwrap(); + } + files.push(name); + } + assert!(files.contains(&"docs/a.txt".to_string())); + assert!(files.contains(&"app/index.js".to_string())); + assert!( + !files.iter().any(|n| n.contains("node_modules")), + "excluded path archived: {files:?}" + ); + assert!(link_is_symlink, "symlink must be stored as a link"); + assert_eq!(a_txt, "v1", "file content must be archived"); +} + +fn entry_names(archive: &Path) -> Vec { + let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(archive).unwrap())); + ar.entries() + .unwrap() + .map(|e| { + e.unwrap() + .path() + .unwrap() + .to_string_lossy() + .trim_end_matches('/') + .to_string() + }) + .collect() +} + +#[tokio::test] +async fn backup_does_not_archive_its_own_output() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let out_dir = src.path().join("backups"); + fs::create_dir_all(&out_dir).unwrap(); + let cfg = files_config(src.path(), &[]); + + let archive = backup::run(cfg, out_dir.clone(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + assert!(archive.exists()); + let names = entry_names(&archive); + assert!(names.contains(&"docs/a.txt".to_string())); + assert!( + !names.iter().any(|n| n.starts_with("backups")), + "own output archived: {names:?}" + ); +} + +#[tokio::test] +async fn backup_skips_sockets_with_a_warning() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let _listener = std::os::unix::net::UnixListener::bind(src.path().join("sock")).unwrap(); + let cfg = files_config(src.path(), &[]); + let out = tempfile::TempDir::new().unwrap(); + + let archive = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + let names = entry_names(&archive); + assert!(names.contains(&"docs/a.txt".to_string())); + assert!( + !names.iter().any(|n| n == "sock"), + "socket archived: {names:?}" + ); +} + +#[tokio::test] +async fn backup_fails_when_source_is_not_a_directory() { + let src = tempfile::TempDir::new().unwrap(); + let file = src.path().join("plain.txt"); + fs::write(&file, "x").unwrap(); + let cfg = files_config(&file, &[]); + let out = tempfile::TempDir::new().unwrap(); + + let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await; + + assert!( + result.is_err(), + "a file path must not produce an empty archive" + ); +} + +use crate::domain::files::restore; + +#[tokio::test] +async fn restore_is_a_mirror_that_keeps_excluded_paths() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &["node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + fs::create_dir_all(root.join("newdir/sub")).unwrap(); + fs::write(root.join("app/node_modules/dep.js"), "changed").unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert_eq!( + fs::read_to_string(root.join("app/index.js")).unwrap(), + "app" + ); + assert!( + !root.join("new.txt").exists(), + "file created after backup must be deleted" + ); + assert!( + !root.join("newdir").exists(), + "dir created after backup must be deleted" + ); + assert_eq!( + fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(), + "changed", + "excluded path must be left untouched" + ); + assert!( + fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +#[test] +fn ensure_restorable_rejects_dangerous_targets() { + assert!(restore::ensure_restorable(Path::new("/")).is_err()); + assert!(restore::ensure_restorable(Path::new("")).is_err()); + assert!(restore::ensure_restorable(Path::new("relative/dir")).is_err()); + assert!(restore::ensure_restorable(Path::new("/definitely/not/here")).is_err()); + let dir = tempfile::TempDir::new().unwrap(); + assert!(restore::ensure_restorable(dir.path()).is_ok()); + let escapes_to_root = dir.path().join("../../../../../../../../../.."); + assert!(restore::ensure_restorable(&escapes_to_root).is_err()); +} + +#[tokio::test] +async fn restore_refuses_corrupt_archive_without_touching_data() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let len = fs::metadata(&archive).unwrap().len(); + fs::OpenOptions::new() + .write(true) + .open(&archive) + .unwrap() + .set_len(len / 2) + .unwrap(); + fs::write(root.join("keep.txt"), "keep").unwrap(); + + let result = restore::run(cfg, archive, logger).await; + + assert!(result.is_err(), "a truncated archive must be refused"); + assert!( + root.join("keep.txt").exists(), + "data must survive a refused restore" + ); + assert!( + root.join("docs/a.txt").exists(), + "data must survive a refused restore" + ); +} + +#[tokio::test] +async fn ping_reports_directory_reachability() { + let dir = tempfile::TempDir::new().unwrap(); + let ok = files_config(dir.path(), &[]); + assert!(crate::domain::files::ping::run(ok).await.unwrap()); + let missing = files_config(Path::new("/definitely/not/here"), &[]); + assert!(!crate::domain::files::ping::run(missing).await.unwrap()); +} + +#[tokio::test] +async fn restore_survives_archive_stored_inside_the_source() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let tmp_dir = root.join(".tmp-restore"); + fs::create_dir_all(&tmp_dir).unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), tmp_dir.clone(), logger.clone()) + .await + .unwrap(); + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + + restore::run(cfg, archive.clone(), logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert_eq!( + fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(), + "dep" + ); + assert!(!root.join("new.txt").exists()); + assert!( + archive.exists(), + "the directory holding the archive must not be wiped" + ); +} + +#[tokio::test] +async fn restore_survives_archive_stored_directly_in_the_source() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let outside = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let archive = root.join("snapshot.tar.gz"); + fs::copy(&outside, &archive).unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + + restore::run(cfg, archive.clone(), logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert!(!root.join("new.txt").exists()); + assert!(archive.exists(), "the archive itself must not be wiped"); +} + +#[tokio::test] +async fn restore_never_follows_symlinked_directories() { + let outside = tempfile::TempDir::new().unwrap(); + fs::write(outside.path().join("secret.txt"), "outside").unwrap(); + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + std::os::unix::fs::symlink(outside.path(), root.join("old_link")).unwrap(); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + std::os::unix::fs::symlink(outside.path(), root.join("late_link")).unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!( + fs::read_to_string(outside.path().join("secret.txt")).unwrap(), + "outside" + ); + assert!( + fs::symlink_metadata(root.join("late_link")).is_err(), + "link created after backup must be removed" + ); + assert!( + fs::symlink_metadata(root.join("old_link")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +#[tokio::test] +async fn restore_refuses_archive_with_corrupt_gzip_trailer() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + // The last 8 bytes are CRC32 + ISIZE; tar itself never reads them. + let mut bytes = fs::read(&archive).unwrap(); + let idx = bytes.len() - 6; + bytes[idx] ^= 0xff; + fs::write(&archive, bytes).unwrap(); + fs::write(root.join("keep.txt"), "keep").unwrap(); + + assert!(restore::run(cfg, archive, logger).await.is_err()); + assert_eq!(fs::read_to_string(root.join("keep.txt")).unwrap(), "keep"); + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); +} + +#[tokio::test] +async fn restore_anchored_exclude_applies_through_a_symlinked_path() { + let base = tempfile::TempDir::new().unwrap(); + let real = base.path().join("real"); + fs::create_dir_all(real.join("app/node_modules")).unwrap(); + fs::create_dir_all(real.join("node_modules")).unwrap(); + fs::write(real.join("app/node_modules/x"), "x1").unwrap(); + fs::write(real.join("node_modules/y"), "y1").unwrap(); + let link = base.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let cfg = files_config(&link, &["/app/node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + fs::write(real.join("app/node_modules/x"), "x2").unwrap(); + fs::write(real.join("node_modules/y"), "y2").unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!( + fs::read_to_string(real.join("app/node_modules/x")).unwrap(), + "x2", + "anchored exclude must be kept" + ); + assert_eq!( + fs::read_to_string(real.join("node_modules/y")).unwrap(), + "y1", + "top-level node_modules is not excluded" + ); +} + +#[test] +fn wipe_keeps_only_excluded_paths_on_a_single_filesystem() { + for one_file_system in [false, true] { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let m = matcher(&["node_modules"]); + + let failures = + restore::wipe_except_excluded(root, &m, one_file_system, &JobLogger::new()).unwrap(); + + assert_eq!(failures, 0); + + assert!(root.join("app/node_modules/dep.js").exists()); + assert!(!root.join("app/index.js").exists()); + assert!(!root.join("docs").exists()); + assert!(fs::symlink_metadata(root.join("link")).is_err()); + } +} + +use crate::domain::files::platform::is_root; +use std::os::unix::fs::PermissionsExt; + +#[test] +fn unpack_retries_without_metadata_when_ownership_cannot_be_set() { + if is_root() { + return; // root may chown to uid 0, so the first pass would not fail + } + let dir = tempfile::TempDir::new().unwrap(); + let archive = dir.path().join("root-owned.tar.gz"); + let gz = flate2::write::GzEncoder::new( + fs::File::create(&archive).unwrap(), + flate2::Compression::default(), + ); + let mut builder = tar::Builder::new(gz); + for (name, body) in [("a.txt", "alpha"), ("sub/b.txt", "beta")] { + let mut header = tar::Header::new_gnu(); + header.set_path(name).unwrap(); + header.set_uid(0); + header.set_gid(0); + header.set_mode(0o644); + header.set_size(body.len() as u64); + header.set_cksum(); + builder.append(&header, body.as_bytes()).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + let logger = JobLogger::new(); + + restore::unpack( + &fs::File::open(&archive).unwrap(), + target.path(), + true, + &logger, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(target.path().join("a.txt")).unwrap(), + "alpha" + ); + assert_eq!( + fs::read_to_string(target.path().join("sub/b.txt")).unwrap(), + "beta" + ); + assert!( + logger + .into_entries() + .iter() + .any(|e| e.level == "warn" && e.message.contains("retrying without metadata")), + "the chown failure must go through the fallback" + ); +} + +#[tokio::test] +async fn restore_extracts_even_when_some_entries_cannot_be_removed() { + if is_root() { + return; // root ignores directory permissions + } + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + let locked = root.join("locked"); + fs::create_dir(&locked).unwrap(); + fs::write(locked.join("stale.txt"), "stale").unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o555)).unwrap(); + + let result = restore::run(cfg, archive, logger).await; + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap(); + + let err = result.expect_err("an entry that could not be removed must fail the restore"); + assert!( + err.to_string().contains("could not be removed"), + "unexpected error: {err:#}" + ); + assert_eq!( + fs::read_to_string(root.join("docs/a.txt")).unwrap(), + "v1", + "the archive must still be extracted" + ); + assert_eq!( + fs::read_to_string(root.join("app/index.js")).unwrap(), + "app" + ); + assert!( + !root.join("new.txt").exists(), + "removable entries must still be wiped" + ); + assert!(locked.join("stale.txt").exists()); +} + +#[tokio::test] +async fn backup_fails_when_an_entry_cannot_be_read() { + if is_root() { + return; // root reads mode 0o000 files + } + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let secret = src.path().join("secret.txt"); + fs::write(&secret, "secret").unwrap(); + fs::set_permissions(&secret, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(src.path(), &[]); + let out = tempfile::TempDir::new().unwrap(); + + let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await; + fs::set_permissions(&secret, fs::Permissions::from_mode(0o644)).unwrap(); + + let err = result.expect_err("an unreadable entry must fail the backup"); + assert!( + err.to_string().contains("1 entry could not be read"), + "unexpected error: {err:#}" + ); +} + +#[tokio::test] +async fn restore_keeps_special_files() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let _listener = std::os::unix::net::UnixListener::bind(root.join("sock")).unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert!( + fs::symlink_metadata(root.join("sock")).is_ok(), + "socket must survive a restore" + ); + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); +} diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index e467fdd..61f114b 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -8,3 +8,4 @@ mod valkey; mod firebird; mod mssql; mod docker_volume; +pub(crate) mod files; diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index a4f8e87..aa892a4 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -228,6 +228,72 @@ fn resolve_dashboard_config_noop_when_not_encrypted() { assert!(status.resolved_config.is_none()); } +#[test] +fn ping_without_engine_fields_defaults_to_none() { + let status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": false, "file": null, "metaFile": null, "size": null } } + })) + .unwrap(); + assert!(status.data.backup.engine.is_none()); + assert!(status.data.restore.engine.is_none()); + assert!(status.data.restore.snapshot_id.is_none()); + assert!(status.data.restore.storage.is_none()); +} + +#[test] +fn resolve_restore_storage_decrypts_the_snapshot_channel() { + use crate::services::status::resolve_restore_storage; + use base64::{engine::general_purpose, Engine}; + + let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]); + let channel = r#"[{"id":"ch-1","provider":"s3","folderName":"backups","config":{"endPointUrl":"s3.example.com","bucketName":"b"}}]"#; + let snapshot_id = "ab".repeat(32); + + let mut status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { + "backup": { "action": false, "cron": "0 * * * *", "engine": "restic" }, + "restore": { "action": true, "file": null, "metaFile": null, "size": null, + "engine": "restic", "snapshotId": snapshot_id, + "storageCiphertext": encrypt_json_gcm(channel.as_bytes(), &master_key_b64) } + } + })) + .unwrap(); + + assert_eq!(status.data.backup.engine.as_deref(), Some("restic")); + assert_eq!(status.data.restore.snapshot_id.as_deref(), Some(snapshot_id.as_str())); + + resolve_restore_storage(&mut status, &master_key_b64).unwrap(); + + let storage = status.data.restore.storage.expect("decrypted"); + assert_eq!(storage.id, "ch-1"); + assert_eq!(storage.provider, "s3"); + assert_eq!(storage.folder_name.as_deref(), Some("backups")); +} + +#[test] +fn resolve_restore_storage_rejects_a_bad_ciphertext() { + use crate::services::status::resolve_restore_storage; + use base64::{engine::general_purpose, Engine}; + + let mut status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": true, "engine": "restic", "storageCiphertext": "bm9wZQ==" } } + })) + .unwrap(); + assert!(resolve_restore_storage(&mut status, &general_purpose::STANDARD.encode([7u8; 32])).is_err()); + assert!(status.data.restore.storage.is_none()); +} + fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { use aes_gcm::aead::{Aead, KeyInit}; use aes_gcm::{Aes256Gcm, Key, Nonce}; @@ -243,3 +309,39 @@ fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { data.extend_from_slice(&ct); general_purpose::STANDARD.encode(data) } + +#[test] +fn ping_payload_sends_method_only_when_set() { + use crate::services::api::endpoints::status::DatabasePayload; + let payload = |method: Option<&'static str>| DatabasePayload { + name: "docs", + dbms: "files", + generated_id: "g", + ping_status: true, + method, + }; + let with = serde_json::to_value(payload(Some("sync"))).unwrap(); + assert_eq!(with["method"], "sync"); + let without = serde_json::to_value(payload(None)).unwrap(); + assert!(without.get("method").is_none(), "{without}"); +} + +#[test] +fn upload_status_sends_sync_counters_only_when_set() { + use crate::services::api::endpoints::agent::backup::upload::status::StatusUploadRequest; + let request = |files_transferred: Option, files_deleted: Option| StatusUploadRequest { + generated_id: "g".into(), + backup_storage_id: "bs".into(), + status: "success".into(), + path: "p".into(), + size: 3, + backup_id: "b".into(), + files_transferred, + files_deleted, + }; + let with = serde_json::to_value(request(Some(2), Some(1))).unwrap(); + assert_eq!(with["filesTransferred"], 2); + assert_eq!(with["filesDeleted"], 1); + let without = serde_json::to_value(request(None, None)).unwrap(); + assert!(without.get("filesTransferred").is_none() && without.get("filesDeleted").is_none(), "{without}"); +} diff --git a/src/tests/services/backup_uploader_tests.rs b/src/tests/services/backup_uploader_tests.rs index f23b99f..3588d48 100644 --- a/src/tests/services/backup_uploader_tests.rs +++ b/src/tests/services/backup_uploader_tests.rs @@ -20,7 +20,7 @@ use tempfile::NamedTempFile; use wiremock::matchers::{body_partial_json, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; -fn ctx_pointing_at(base_url: String) -> Context { +pub(crate) fn ctx_pointing_at(base_url: String) -> Context { Context { edge_key: EdgeKey { server_url: String::new(), diff --git a/src/tests/services/config_tests.rs b/src/tests/services/config_tests.rs index 6ceb172..6c29465 100644 --- a/src/tests/services/config_tests.rs +++ b/src/tests/services/config_tests.rs @@ -1,7 +1,7 @@ use crate::core::context::Context; use crate::services::api::ApiClient; use crate::services::config::ConfigService; -use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig}; +use crate::services::config::{DatabasesConfig, InputDatabaseConfig, build_config, files_method}; use crate::utils::edge_key::EdgeKey; use std::io::Write; use std::sync::Arc; @@ -176,24 +176,39 @@ fn keep_ownership_extraction_logic() { // true → keep ownership let mut opts: HashMap = HashMap::new(); opts.insert("keep_ownership".to_string(), Value::Bool(true)); - let keep = opts.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep = opts + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(keep, "should keep ownership when flag is true"); // false → strip let mut opts2: HashMap = HashMap::new(); opts2.insert("keep_ownership".to_string(), Value::Bool(false)); - let keep2 = opts2.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep2 = opts2 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep2, "should strip when flag is false"); // missing → strip let opts3: HashMap = HashMap::new(); - let keep3 = opts3.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep3 = opts3 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep3, "should strip when key absent"); // wrong type → strip let mut opts4: HashMap = HashMap::new(); - opts4.insert("keep_ownership".to_string(), Value::String("yes".to_string())); - let keep4 = opts4.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + opts4.insert( + "keep_ownership".to_string(), + Value::String("yes".to_string()), + ); + let keep4 = opts4 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep4, "should strip when value is not bool"); } @@ -258,7 +273,9 @@ fn docker_volume_requires_volume_name() { ); let service = ConfigService::new(test_context()); - let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err(); + let err = service + .load(Some(file.path().to_str().unwrap())) + .unwrap_err(); assert!(err.contains("volume_name"), "error was: {err}"); } @@ -323,7 +340,9 @@ fn databases_config_roundtrips_through_serde() { ) .unwrap(); let cfg = build_config(input).unwrap(); - let wrapped = DatabasesConfig { databases: vec![cfg] }; + let wrapped = DatabasesConfig { + databases: vec![cfg], + }; let json = serde_json::to_string(&wrapped).unwrap(); let back: DatabasesConfig = serde_json::from_str(&json).unwrap(); @@ -331,3 +350,77 @@ fn databases_config_roundtrips_through_serde() { assert_eq!(back.databases[0].db_type.as_str(), "postgresql"); assert_eq!(back.databases[0].password, "secret"); } + +#[test] +fn files_type_parses_with_options() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ + "name": "shared", + "type": "files", + "path": "/data/files", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681", + "options": { "exclude": ["node_modules", "/cache"], "one_file_system": true } + }"#, + ) + .unwrap(); + + let cfg = build_config(input).unwrap(); + assert_eq!(cfg.db_type.as_str(), "files"); + assert_eq!(cfg.path, "/data/files"); + assert_eq!( + cfg.options["exclude"], + serde_json::json!(["node_modules", "/cache"]) + ); + assert_eq!(cfg.options["one_file_system"], serde_json::json!(true)); +} + +#[test] +fn files_type_requires_path() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ "name": "shared", "type": "files", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }"#, + ) + .unwrap(); + let err = build_config(input).unwrap_err(); + assert!(err.contains("path"), "unexpected error: {err}"); +} + +#[test] +fn files_type_rejects_relative_and_root_paths() { + for bad in ["data/files", "/", "//"] { + let json = format!( + r#"{{ "name": "shared", "type": "files", "path": "{bad}", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }}"# + ); + let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap(); + let err = build_config(input).unwrap_err(); + assert!(err.contains("absolute"), "path {bad:?} gave: {err}"); + } +} + +fn files_input(options: &str) -> InputDatabaseConfig { + serde_json::from_str(&format!( + r#"{{ "name": "docs", "type": "files", "path": "/data/files", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"{options} }}"# + )) + .unwrap() +} + +#[test] +fn files_method_defaults_to_archive() { + let cfg = build_config(files_input("")).unwrap(); + assert_eq!(files_method(&cfg), "archive"); +} + +#[test] +fn files_method_reads_snapshot_and_sync() { + let snap = build_config(files_input(r#", "options": { "method": "snapshot" }"#)).unwrap(); + assert_eq!(files_method(&snap), "snapshot"); + let sync = build_config(files_input(r#", "options": { "method": "sync" }"#)).unwrap(); + assert_eq!(files_method(&sync), "sync"); +} + +#[test] +fn unknown_files_method_refuses_the_source() { + let err = build_config(files_input(r#", "options": { "method": "mirror" }"#)).unwrap_err(); + assert!(err.contains("Unknown method"), "{err}"); + assert!(err.contains("docs"), "{err}"); +} diff --git a/src/tests/services/dashboard_config_tests.rs b/src/tests/services/dashboard_config_tests.rs index 68027ad..decb577 100644 --- a/src/tests/services/dashboard_config_tests.rs +++ b/src/tests/services/dashboard_config_tests.rs @@ -82,3 +82,38 @@ fn persist_cache_leaves_no_tmp_file() { assert!(!tmp.exists(), "temp file should have been renamed away"); assert!(path.exists()); } + +use crate::services::dashboard_config::local_only_ids; +use crate::services::status::payload_method; + +fn files_cfg(gen_id: &str, method: Option<&str>) -> DatabaseConfig { + let options = method + .map(|m| format!(r#", "options": {{ "method": "{m}" }}"#)) + .unwrap_or_default(); + let json = format!( + r#"{{ "name": "docs", "type": "files", "path": "/data/files", "generated_id": "{gen_id}"{options} }}"# + ); + build_config(serde_json::from_str::(&json).unwrap()).unwrap() +} + +#[test] +fn local_only_ids_skips_sources_replaced_by_the_dashboard() { + let local = vec![cfg("local-a", ID_A, "h"), cfg("local-b", ID_B, "h")]; + let dashboard = vec![cfg("dash-b", ID_B, "h")]; + let ids = local_only_ids(&local, &dashboard); + assert!(ids.contains(ID_A)); + assert!(!ids.contains(ID_B)); +} + +#[test] +fn payload_method_is_sent_for_local_files_sources_only() { + let local = files_cfg(ID_A, Some("sync")); + let ids = local_only_ids(&[local.clone()], &[]); + assert_eq!(payload_method(&local, &ids), Some("sync")); + assert_eq!(payload_method(&files_cfg(ID_A, None), &ids), Some("archive")); + // Replaced by a dashboard config: the dashboard owns the method. + let replaced = local_only_ids(&[local.clone()], &[local.clone()]); + assert_eq!(payload_method(&local, &replaced), None); + // Not a files source. + assert_eq!(payload_method(&cfg("pg", ID_A, "h"), &ids), None); +} diff --git a/src/tests/services/local_storage_tests.rs b/src/tests/services/local_storage_tests.rs new file mode 100644 index 0000000..1c46d71 --- /dev/null +++ b/src/tests/services/local_storage_tests.rs @@ -0,0 +1,190 @@ +//! Spec B: the dashboard's local storage, served here by `rclone serve restic +//! --append-only` and `rclone serve webdav` on free ports. The restic tests need +//! restic on PATH (test container); the sync test only needs rclone. + +use crate::domain::files::platform::is_root; +use crate::services::backup::logger::JobLogger; +use crate::services::restic::backup::snapshot; +use crate::services::restic::command::ResticRepo; +use crate::services::restic::restore::restore; +use crate::tests::domain::files::files_config; +use std::fs; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +pub(crate) const MASTER_KEY_B64: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + +pub(crate) struct RcloneServer { + child: Child, + pub base_url: String, +} + +impl Drop for RcloneServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// `rclone serve --baseurl /storage/ ` on a free port, user +/// `portabase`, credentials through the env like the dashboard does. +pub(crate) fn serve(kind: &str, prefix: &str, root: &Path, extra: &[&str], password: &str) -> RcloneServer { + let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + let child = Command::new("rclone") + .args(["serve", kind, "--addr", &format!("127.0.0.1:{port}"), "--baseurl", &format!("/storage/{prefix}")]) + .args(extra) + .arg(root) + .env("RCLONE_USER", "portabase") + .env("RCLONE_PASS", password) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("rclone binary not found"); + for _ in 0..100 { + if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + RcloneServer { child, base_url: format!("http://127.0.0.1:{port}") } +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); +} + +fn local_repo(server: &RcloneServer, generated_id: &str, cache: &Path) -> ResticRepo { + ResticRepo::rest(&format!("{}/storage/restic", server.base_url), generated_id, "test-password", "s3cret", cache.to_path_buf()) +} + +#[tokio::test] +async fn local_snapshot_and_restore_go_through_the_append_only_server() { + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + let repo_dir = store.path().join(&cfg.generated_id); + assert!(repo_dir.join("config").exists(), "repository written on the served disk"); + assert_eq!(fs::read_dir(repo_dir.join("locks")).unwrap().count(), 0, "lock removed despite --append-only"); + + fs::write(src.join("a.txt"), "changed").unwrap(); + fs::write(src.join("extra.txt"), "x").unwrap(); + restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap(); + assert_eq!(fs::read_to_string(src.join("a.txt")).unwrap(), "v1"); + assert!(!src.join("extra.txt").exists()); +} + +#[tokio::test] +async fn the_agent_cannot_forget_on_the_local_server() { + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + let forget = repo.run(["forget", snap.snapshot_id.as_str()], &logger).await.unwrap(); + + assert_ne!(forget.code, 0, "append-only must refuse forget"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert!(listed.stdout.contains(&snap.snapshot_id), "{}", listed.stdout); +} + +#[tokio::test] +async fn unreadable_file_on_a_local_repo_keeps_the_snapshot_for_the_dashboard() { + if is_root() { + return; // root reads a 000 file anyway + } + use std::os::unix::fs::PermissionsExt; + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let locked = src.join("docs/b.txt"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + + let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("the dashboard forgets it"), "{err:#}"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert!(listed.stdout.contains("bs:bs-1"), "{}", listed.stdout); +} + +#[tokio::test] +async fn local_sync_mirrors_through_the_webdav_server() { + use crate::core::context::Context; + use crate::services::api::ApiClient; + use crate::services::api::models::agent::status::DatabaseStorage; + use crate::services::restic::password::local_storage_password; + use crate::services::sync::backup::one_storage; + use crate::utils::edge_key::EdgeKey; + use serde_json::json; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let password = local_storage_password(MASTER_KEY_B64).unwrap(); + let server = serve("webdav", "sync", store.path(), &[], &password); + + let api = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/agent/agent-1/backup/upload/init")) + .and(body_partial_json(json!({ "engine": "sync" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&api) + .await; + Mock::given(method("PATCH")) + .and(path("/agent/agent-1/backup/upload/status")) + .and(body_partial_json(json!({ "status": "success" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&api) + .await; + + let ctx = Context { + edge_key: EdgeKey { + server_url: server.base_url.clone(), + agent_id: "agent-1".into(), + master_key_b64: MASTER_KEY_B64.into(), + }, + api: ApiClient::new(api.uri()), + }; + let storage: DatabaseStorage = + serde_json::from_value(json!({ "id": "ch-local", "provider": "local", "folderName": "backups", "config": {} })).unwrap(); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let replica = store.path().join(&cfg.generated_id).join("current"); + + let first = one_storage(&ctx, &cfg, &storage, "backup-1", &JobLogger::new()).await; + assert!(first.success, "{:?}", first.error); + assert_eq!(first.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); + assert_eq!(fs::read_to_string(replica.join("docs/b.txt")).unwrap(), "b"); + + fs::remove_file(src.join("docs/b.txt")).unwrap(); + let second = one_storage(&ctx, &cfg, &storage, "backup-2", &JobLogger::new()).await; + assert!(second.success, "{:?}", second.error); + assert!(!replica.join("docs/b.txt").exists(), "deletion propagated"); + assert!(replica.join("a.txt").exists()); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index aa4be3a..65ec678 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -1,6 +1,9 @@ mod api_models_tests; mod backup_runner_tests; -mod backup_uploader_tests; +pub(crate) mod backup_uploader_tests; mod config_tests; mod dashboard_config_tests; +pub(crate) mod local_storage_tests; +mod restic_tests; mod restore_downloader_tests; +mod sync_tests; diff --git a/src/tests/services/restic_tests.rs b/src/tests/services/restic_tests.rs new file mode 100644 index 0000000..c4cd0c2 --- /dev/null +++ b/src/tests/services/restic_tests.rs @@ -0,0 +1,348 @@ +//! Snapshots mode (restic). Pure helpers first; the integration tests further +//! down need `restic` and `rclone` on PATH. + +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::restic::command::is_snapshot_id; +use crate::services::restic::excludes::{backup_excludes, glob_escape, restore_excludes}; +use crate::services::restic::json::{self, BackupSummary, Line}; +use crate::services::restic::password::{derive_password, hkdf_sha256_32}; +use crate::tests::domain::files::files_config; +use serde_json::json; +use std::path::Path; + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|s| s.to_string()).collect() +} + +#[test] +fn snapshot_ids_are_full_lowercase_hex() { + assert!(is_snapshot_id(&"a1".repeat(32))); + for bad in ["--password-command=touch /tmp/pwned", &"A".repeat(64), &"a".repeat(63), &"a".repeat(65), ""] { + assert!(!is_snapshot_id(bad), "{bad}"); + } +} + +#[test] +fn hkdf_matches_rfc5869_case_3_first_block() { + let okm = hkdf_sha256_32(&[0x0b; 22], b""); + assert_eq!( + hex::encode(okm), + "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d" + ); +} + +#[test] +fn password_matches_the_dashboard_vector() { + // Same vector as src/lib/restic/repo.ts in the dashboard (spike S4). + let password = derive_password( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "test-generated-id", + ) + .unwrap(); + assert_eq!( + password, + "577e00efe6c953e8583d90e14d78109d805b616164279b8182282db76033b090" + ); +} + +#[test] +fn password_rejects_a_non_base64_master_key() { + assert!(derive_password("not base64!", "x").is_err()); +} + +#[test] +fn backup_excludes_are_rooted_at_the_source() { + let got = backup_excludes( + Path::new("/data/files"), + &strings(&["*.log", "node_modules/", "/build/", "/docs/**/*.md", " ", "/"]), + ) + .unwrap(); + assert_eq!( + got, + strings(&[ + "/data/files/**/*.log", + "/data/files/**/node_modules", + "/data/files/build", + "/data/files/docs/**/*.md", + ]) + ); +} + +#[test] +fn backup_excludes_escape_glob_characters_in_the_root() { + let got = backup_excludes(Path::new("/data/src[1]"), &strings(&["tmp"])).unwrap(); + assert_eq!(got, strings(&["/data/src\\[1\\]/**/tmp"])); +} + +#[test] +fn restore_excludes_are_relative_to_the_snapshot_subfolder() { + let got = restore_excludes(&strings(&["*.log", "/build/", "a/b"])).unwrap(); + assert_eq!(got, strings(&["/**/*.log", "/build", "/**/a/b"])); +} + +#[test] +fn bracket_negation_is_rewritten_for_restic() { + // globset reads `[!a]`; restic (Go filepath.Match) only knows `[^a]`. + assert_eq!( + backup_excludes(Path::new("/data"), &strings(&["[!a]*.log", "/x/[!0-9]", r"\[!a]", "[a[!]"])).unwrap(), + strings(&["/data/**/[^a]*.log", "/data/x/[^0-9]", r"/data/**/\[!a]", "/data/**/[a[!]"]) + ); + assert_eq!(restore_excludes(&strings(&["/[!a]*", "[!a]"])).unwrap(), strings(&["/[^a]*", "/**/[^a]"])); +} + +#[test] +fn brace_patterns_are_rejected_on_both_sides() { + let patterns = strings(&["{x,y}.dat"]); + let err = backup_excludes(Path::new("/data"), &patterns).unwrap_err(); + assert!(err.to_string().contains("brace patterns are not supported in Snapshots mode")); + assert!(restore_excludes(&patterns).is_err()); +} + +#[test] +fn glob_escape_escapes_every_metacharacter() { + assert_eq!(glob_escape(r"a*b?c[d]e\f"), r"a\*b\?c\[d\]e\\f"); +} + +#[test] +fn json_lines_are_classified() { + // Recorded from restic 0.19.1 (paths shortened). + let status = r#"{"message_type":"status","percent_done":0.9999997777778271,"total_files":4,"files_done":2,"total_bytes":9000002,"bytes_done":9000000,"error_count":1}"#; + let error = r#"{"message_type":"error","error":{"message":"open /src/u: permission denied"},"during":"archival","item":"/src/u"}"#; + let exit = r#"{"message_type":"exit_error","code":3,"message":"Warning: at least one source file could not be read"}"#; + + assert_eq!(json::parse(status), Line::Status { percent_done: 0.9999997777778272 }); + assert_eq!( + json::parse(error), + Line::Error { message: "open /src/u: permission denied".into(), item: Some("/src/u".into()) } + ); + assert_eq!( + json::parse(exit), + Line::ExitError { code: 3, message: "Warning: at least one source file could not be read".into() } + ); + assert_eq!(json::parse("Fatal: wrong password"), Line::Text("Fatal: wrong password".into())); + assert_eq!(json::parse("[]"), Line::Text("[]".into())); +} + +#[test] +fn backup_summary_deserializes_from_a_recorded_line() { + let line = r#"{"message_type":"summary","files_new":3,"files_changed":0,"files_unmodified":0,"dirs_new":9,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":8,"tree_blobs":10,"data_added":9005859,"data_added_packed":9004808,"total_files_processed":3,"total_bytes_processed":9000000,"total_duration":0.78928,"backup_start":"2026-10-03T16:59:06.569467+02:00","backup_end":"2026-10-03T16:59:07.359124+02:00","snapshot_id":"05ba7db8d08e4e097b869dbc511a8b8936298b2fcb01b221115789491204a470"}"#; + let Line::Summary(value) = json::parse(line) else { panic!("not a summary") }; + let summary: BackupSummary = serde_json::from_value(value).unwrap(); + assert_eq!(summary.snapshot_id.len(), 64); + assert_eq!(summary.data_added_packed, 9004808); + assert_eq!(summary.total_bytes_processed, 9000000); + assert_eq!(summary.files_new, 3); +} + +#[test] +fn a_local_channel_opens_the_dashboard_rest_repository() { + use crate::utils::edge_key::EdgeKey; + let storage: DatabaseStorage = + serde_json::from_value(json!({ "id": "storage-1", "provider": "local", "config": {} })).unwrap(); + let edge_key = EdgeKey { + server_url: "http://dash:8887/".into(), + agent_id: "agent-1".into(), + master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(), + }; + let repo = ResticRepo::open(&storage, "g1", &edge_key).unwrap(); + assert_eq!(repo.repository(), "rest:http://dash:8887/storage/restic/g1/"); + assert!(repo.is_append_only()); +} + +#[test] +fn local_storage_password_matches_the_dashboard_vector() { + use crate::services::restic::password::local_storage_password; + // Same vector as src/lib/local-storage/credential.ts in the dashboard. + assert_eq!( + local_storage_password("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=").unwrap(), + "8abfb788701b8c502658154746af6af768ea08f579381ad73b0f1e7dd43caea2" + ); +} + +// ---------- integration: restic + rclone on PATH ---------- + +use crate::domain::files::platform::is_root; +use crate::services::restic::backup::snapshot; +use crate::services::restic::command::ResticRepo; +use crate::services::restic::restore::restore; +use std::fs; +use tempfile::TempDir; + +/// A repository on an rclone `alias` remote pointing at a temp dir (test-only: +/// production channels always go through `rclone_target`). +fn local_repo(dir: &Path, generated_id: &str) -> ResticRepo { + fs::create_dir_all(dir).unwrap(); + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + ResticRepo::new(&config, "pb", "", "backups", generated_id, "test-password", dir.join("cache")).unwrap() +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); + fs::create_dir_all(root.join("build")).unwrap(); + fs::write(root.join("build/out.bin"), "out").unwrap(); + fs::write(root.join("app.log"), "log").unwrap(); +} + +/// `relative/path=content` for every regular file, sorted. +fn files(root: &Path) -> Vec { + let mut out: Vec = walkdir::WalkDir::new(root) + .min_depth(1) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| { + let rel = e.path().strip_prefix(root).unwrap().display().to_string(); + format!("{rel}={}", fs::read_to_string(e.path()).unwrap()) + }) + .collect(); + out.sort(); + out +} + +#[tokio::test] +async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_parent() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + + let first = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + assert_eq!(first.snapshot_id.len(), 64); + assert_eq!(first.files_new, 4); + assert!(base.path().join("repo/backups/restic").join(&cfg.generated_id).join("config").exists()); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + let second = snapshot(&repo, &cfg, "bs-2", &logger).await.unwrap(); + assert_eq!(second.files_changed, 1); + assert_eq!(second.files_unmodified, 3); + + let tagged = repo.run(["snapshots", "--json", "--tag", "bs:bs-2"], &logger).await.unwrap(); + // The tag selects only the second snapshot, and its `parent` is the first + // (a plain `contains(first)` check would match that `parent` field). + let listed: serde_json::Value = serde_json::from_str(tagged.stdout.trim()).unwrap(); + let listed = listed.as_array().unwrap(); + assert_eq!(listed.len(), 1, "{}", tagged.stdout); + assert_eq!(listed[0]["id"], second.snapshot_id.as_str()); + assert_eq!(listed[0]["parent"], first.snapshot_id.as_str()); +} + +#[tokio::test] +async fn restore_is_a_mirror_that_keeps_excluded_paths() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + fs::write(src.path().join("docs/new.log"), "kept").unwrap(); + fs::write(src.path().join("build/new.bin"), "kept").unwrap(); + + restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap(); + + assert_eq!( + files(src.path()), + vec![ + "app.log=log", + "build/new.bin=kept", + "build/out.bin=out", + "docs/a.txt=v1", + "docs/b.txt=b", + "docs/new.log=kept", + ] + ); +} + +#[tokio::test] +async fn unreadable_file_fails_the_snapshot_and_forgets_it() { + if is_root() { + return; // root reads a 000 file anyway + } + use std::os::unix::fs::PermissionsExt; + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let locked = src.path().join("docs/b.txt"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + + let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("could not be read"), "{err:#}"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert_eq!(listed.stdout.trim(), "[]"); +} + +#[tokio::test] +async fn unanchored_exclude_never_matches_directories_above_the_source() { + // Regression for spike S5: the source lives under a directory named like the pattern. + let base = TempDir::new().unwrap(); + let src = base.path().join("tmp/src"); + fs::create_dir_all(src.join("tmp")).unwrap(); + fs::write(src.join("kept.txt"), "k").unwrap(); + fs::write(src.join("tmp/dropped.txt"), "d").unwrap(); + let cfg = files_config(&src, &["tmp"]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + + let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + assert_eq!(snap.files_new, 1); +} + +#[tokio::test] +async fn restoring_an_unknown_snapshot_fails_without_touching_the_target() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + let before = files(src.path()); + + let err = restore(&repo, &cfg, &"deadbeef".repeat(8), &logger).await.unwrap_err(); + + assert!(err.to_string().contains("not found"), "{err:#}"); + assert_eq!(files(src.path()), before); +} + +#[tokio::test] +async fn wrong_password_is_reported_explicitly() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let dir = base.path().join("repo"); + snapshot(&local_repo(&dir, &cfg.generated_id), &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + let other = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "other", dir.join("cache")).unwrap(); + let err = snapshot(&other, &cfg, "bs-2", &JobLogger::new()).await.unwrap_err(); + assert!(err.to_string().contains("wrong repository password"), "{err:#}"); +} + +#[tokio::test] +async fn an_unusable_cache_dir_runs_restic_without_cache() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let dir = base.path().join("repo"); + fs::create_dir_all(&dir).unwrap(); + let blocker = tempfile::NamedTempFile::new().unwrap(); // a regular file: nothing can be created below it + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + let repo = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "test-password", blocker.path().join("cache")).unwrap(); + snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); +} diff --git a/src/tests/services/sync_tests.rs b/src/tests/services/sync_tests.rs new file mode 100644 index 0000000..98f408c --- /dev/null +++ b/src/tests/services/sync_tests.rs @@ -0,0 +1,250 @@ +//! Sync mode (rclone replica). Pure helpers first; integration tests need `rclone` on PATH. + +use crate::services::sync::excludes::sync_excludes; +use crate::services::sync::stats::{SyncStats, parse_log}; + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|s| s.to_string()).collect() +} + +#[test] +fn sync_excludes_translate_p0_patterns() { + let got = sync_excludes(&strings(&[ + "*.log", "node_modules/", "/build/", "**/tmp", "/docs/**/*.md", "[!a].txt", "{x,y}.dat", " ", "/", + ])); + assert_eq!( + got, + strings(&[ + "*.log", "*.log/**", + "node_modules", "node_modules/**", + "/build", "/build/**", + "tmp", "tmp/**", + "/docs/**/*.md", "/docs/**/*.md/**", "/docs/*.md", "/docs/*.md/**", + "[^a].txt", "[^a].txt/**", + "{x,y}.dat", "{x,y}.dat/**", + ]) + ); +} + +#[test] +fn every_double_star_gap_gets_a_zero_directory_variant() { + assert_eq!( + sync_excludes(&strings(&["a/**/b/**/c"])), + strings(&[ + "a/**/b/**/c", "a/**/b/**/c/**", + "a/b/**/c", "a/b/**/c/**", + "a/**/b/c", "a/**/b/c/**", + "a/b/c", "a/b/c/**", + ]) + ); +} + +#[test] +fn anchored_leading_double_star_also_matches_the_root() { + assert_eq!(sync_excludes(&strings(&["/**/x"])), strings(&["/**/x", "/**/x/**", "/x", "/x/**"])); +} + +#[test] +fn escaped_bracket_is_left_alone() { + assert_eq!(sync_excludes(&strings(&[r"\[!a]"])), strings(&[r"\[!a]", r"\[!a]/**"])); +} + +#[test] +fn parse_log_reads_the_final_stats_and_error_lines() { + let stderr = concat!( + r#"{"level":"error","msg":"Failed to copy: couldn't copy from /src/secret: errno -1","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"not deleting files as there were IO errors","source":"sync/sync.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"notice","msg":"\nTransferred: 0 B\n","source":"accounting/stats.go:1","stats":{"bytes":0,"checks":12,"deletes":0,"errors":1,"fatalError":false,"lastError":"couldn't copy from /src/secret: errno -1","transfers":0},"time":"2026-10-03T19:44:59+02:00"}"#, "\n", + ); + let log = parse_log(stderr); + assert_eq!( + log.stats, + Some(SyncStats { bytes: 0, transfers: 0, deletes: 0, errors: 1, last_error: Some("couldn't copy from /src/secret: errno -1".into()) }) + ); + assert_eq!(log.errors.len(), 2); + assert_eq!(log.errors[1], "not deleting files as there were IO errors"); +} + +#[test] +fn parse_log_reads_a_successful_run() { + let stderr = r#"{"level":"notice","msg":"\nTransferred: 2 B\n","source":"accounting/stats.go:1","stats":{"bytes":2,"checks":5,"deletes":1,"errors":0,"fatalError":false,"transfers":1},"time":"2026-10-03T19:45:10+02:00"}"#; + let log = parse_log(stderr); + assert_eq!(log.stats, Some(SyncStats { bytes: 2, transfers: 1, deletes: 1, errors: 0, last_error: None })); + assert!(log.errors.is_empty()); +} + +#[test] +fn parse_log_keeps_plain_text_lines_as_errors() { + let log = parse_log("Error: unknown flag: --bogus\n"); + assert!(log.stats.is_none()); + assert_eq!(log.errors, strings(&["Error: unknown flag: --bogus"])); +} + +#[test] +fn parse_log_accepts_critical_error_level() { + let stderr = r#"{"level":"critical","msg":"Failed to create file system for \":sftp,host=127.0.0.1,port=1,user=x:/tmp/dst\": NewFs: couldn't connect SSH: dial tcp 127.0.0.1:1: connect: connection refused","source":"cmd/cmd.go:148"}"#; + let log = parse_log(stderr); + assert!(log.stats.is_none()); + assert_eq!(log.errors.len(), 1); + assert!(log.errors[0].starts_with("Failed to create file system")); +} + +#[test] +fn parse_log_deduplicates_repeated_error_lines() { + let stderr = concat!( + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + ); + let log = parse_log(stderr); + assert_eq!(log.errors.len(), 1); + assert_eq!(log.errors[0], "connection timeout"); +} + +#[test] +fn sync_excludes_collapses_repeated_double_stars() { + assert_eq!( + sync_excludes(&strings(&["a/**/**/b"])), + strings(&["a/**/b", "a/**/b/**", "a/b", "a/b/**"]) + ); +} + +// ---------- integration: rclone on PATH, `alias` remote under a temp dir ---------- + +use crate::domain::files::platform::is_root; +use crate::services::backup::logger::JobLogger; +use crate::services::sync::rclone::{replica_target, sync_dir}; +use crate::tests::domain::files::files_config; +use std::fs; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; + +fn alias(dir: &Path) -> String { + fs::create_dir_all(dir).unwrap(); + format!("[pb]\ntype = alias\nremote = {}\n", dir.display()) +} + +fn replica(store: &Path, generated_id: &str) -> PathBuf { + store.join("backups/sync").join(generated_id).join("current") +} + +/// `relative/path=content` for every regular file, sorted. +fn files(root: &Path) -> Vec { + let mut out: Vec = walkdir::WalkDir::new(root) + .min_depth(1) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| format!("{}={}", e.path().strip_prefix(root).unwrap().display(), fs::read_to_string(e.path()).unwrap())) + .collect(); + out.sort(); + out +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); + fs::create_dir_all(root.join("build")).unwrap(); + fs::write(root.join("build/out.bin"), "out").unwrap(); + fs::write(root.join("app.log"), "log").unwrap(); +} + +#[test] +fn replica_target_follows_the_path_convention() { + assert_eq!(replica_target("s3", "bucket", "backups", "g1"), "s3:bucket/backups/sync/g1/current"); + assert_eq!(replica_target("pb", "", "backups", "g1"), "pb:backups/sync/g1/current"); +} + +#[tokio::test] +async fn sync_mirrors_the_source_and_propagates_changes() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let config = alias(store.path()); + let logger = JobLogger::new(); + + let first = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); + assert_eq!(first.transfers, 2); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1", "docs/b.txt=b"]); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + + let second = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); + assert_eq!(second.transfers, 2); + assert_eq!(second.deletes, 1); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v2", "new.txt=new"]); +} + +#[tokio::test] +async fn failed_sync_keeps_files_the_failed_run_would_delete() { + if is_root() { + return; // root reads a 000 file anyway + } + use std::os::unix::fs::PermissionsExt; + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let config = alias(store.path()); + sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap(); + + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + let locked = src.path().join("docs/a.txt"); + // rclone only opens a file whose size/mtime changed, so the unreadable file must differ from the replica's copy. + fs::write(&locked, "changed and now unreadable").unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let err = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("rclone sync failed"), "{err:#}"); + assert!(replica(store.path(), &cfg.generated_id).join("docs/b.txt").exists()); +} + +#[tokio::test] +async fn empty_source_is_refused_and_the_replica_is_untouched() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let cfg = files_config(src.path(), &[]); + let err = sync_dir(&alias(store.path()), &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err(); + assert!(err.to_string().contains("is empty"), "{err:#}"); + assert!(!store.path().join("backups").exists()); +} + +// ---------- sync as a backup method ---------- + +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::sync::backup::replica_path; +use serde_json::json; + +#[test] +fn replica_path_uses_the_channel_folder() { + let custom: DatabaseStorage = + serde_json::from_value(json!({ "id": "s", "provider": "s3", "folderName": "/pb/", "config": {} })).unwrap(); + assert_eq!(replica_path(&custom, "g1"), "pb/sync/g1/current"); + let default: DatabaseStorage = + serde_json::from_value(json!({ "id": "s", "provider": "s3", "config": {} })).unwrap(); + assert_eq!(replica_path(&default, "g1"), "backups/sync/g1/current"); +} + +#[test] +fn local_sync_config_points_at_the_dashboard_webdav_server() { + use crate::services::sync::backup::local_sync_config; + use crate::utils::edge_key::EdgeKey; + let edge_key = EdgeKey { + server_url: "http://dash:8887/".into(), + agent_id: "agent-1".into(), + master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(), + }; + let text = local_sync_config(&edge_key).unwrap(); + assert!( + text.starts_with( + "[pblocal]\ntype = webdav\nurl = http://dash:8887/storage/sync\nvendor = rclone\nuser = portabase\npass = " + ), + "{text}" + ); + assert!(!text.contains("8abfb788"), "the password must be obscured: {text}"); +} diff --git a/src/tests/storage/fixtures/rclone-target/blob-account-key.ini b/src/tests/storage/fixtures/rclone-target/blob-account-key.ini new file mode 100644 index 0000000..6a11d0b --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-account-key.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = myaccount +key = a2V5LWJhc2U2NA== +endpoint = https://myaccount.blob.core.windows.net +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini b/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini new file mode 100644 index 0000000..5c90e5d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = devstoreaccount1 +key = a2V5 +endpoint = http://127.0.0.1:10000/devstoreaccount1 +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini b/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini new file mode 100644 index 0000000..5c90e5d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = devstoreaccount1 +key = a2V5 +endpoint = http://127.0.0.1:10000/devstoreaccount1 +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini b/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini new file mode 100644 index 0000000..4a08d5f --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini @@ -0,0 +1,7 @@ +[google-cloud-storage] +type = google cloud storage +project_number = test +anonymous = true +endpoint = http://127.0.0.1:4443/storage/v1/ +bucket_policy_only = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini b/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini new file mode 100644 index 0000000..fdca8af --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini @@ -0,0 +1,6 @@ +[google-cloud-storage] +type = google cloud storage +project_number = my-project +service_account_credentials = {"client_email":"svc@my-project.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEexample\n-----END PRIVATE KEY-----\n","private_key_id":"","project_id":"my-project","token_uri":"https://oauth2.googleapis.com/token","type":"service_account","universe_domain":"googleapis.com"} +bucket_policy_only = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/google-drive.ini b/src/tests/storage/fixtures/rclone-target/google-drive.ini new file mode 100644 index 0000000..d28fe0d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-drive.ini @@ -0,0 +1,8 @@ +[google-drive] +type = drive +client_id = cid.apps.googleusercontent.com +client_secret = csecret +scope = drive.file +token = {"access_token":"","expiry":"0001-01-01T00:00:00Z","refresh_token":"1//refresh","token_type":"Bearer"} +root_folder_id = folder123 +use_trash = false diff --git a/src/tests/storage/fixtures/rclone-target/inputs.json b/src/tests/storage/fixtures/rclone-target/inputs.json new file mode 100644 index 0000000..4d85c98 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/inputs.json @@ -0,0 +1,52 @@ +{ + "s3": { + "provider": "s3", + "config": { "endPointUrl": "s3.example.com", "accessKey": "AKIAEXAMPLE", "secretKey": "secretEXAMPLE", "bucketName": "my-bucket", "ssl": true, "region": "eu-west-1" }, + "expected": { "remoteName": "s3", "basePath": "my-bucket" } + }, + "s3-port-no-ssl": { + "provider": "s3", + "config": { "endPointUrl": "minio.local", "port": 9000, "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "backups", "ssl": false, "region": " " }, + "expected": { "remoteName": "s3", "basePath": "backups" } + }, + "blob-account-key": { + "provider": "blob", + "config": { "authMode": "accountKey", "accountName": "myaccount", "accountKey": "a2V5LWJhc2U2NA==", "containerName": "backups" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "blob-endpoint-url": { + "provider": "blob", + "config": { "authMode": "accountKey", "accountName": "devstoreaccount1", "accountKey": "a2V5", "containerName": "backups", "endpointUrl": "http://127.0.0.1:10000" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "blob-connection-string": { + "provider": "blob", + "config": { "authMode": "connectionString", "connectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=a2V5;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", "containerName": "backups" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "google-cloud-storage": { + "provider": "google-cloud-storage", + "config": { "projectId": "my-project", "bucketName": "my-gcs-bucket", "clientEmail": "svc@my-project.iam.gserviceaccount.com", "privateKey": "-----BEGIN PRIVATE KEY-----\\nMIIEexample\\n-----END PRIVATE KEY-----\\n", "apiEndpoint": "" }, + "expected": { "remoteName": "google-cloud-storage", "basePath": "my-gcs-bucket" } + }, + "google-cloud-storage-emulator": { + "provider": "google-cloud-storage", + "config": { "projectId": "test", "bucketName": "bucket", "clientEmail": "x@test", "privateKey": "", "apiEndpoint": "http://127.0.0.1:4443" }, + "expected": { "remoteName": "google-cloud-storage", "basePath": "bucket" } + }, + "google-drive": { + "provider": "google-drive", + "config": { "clientId": "cid.apps.googleusercontent.com", "clientSecret": "csecret", "refreshToken": "1//refresh", "folderId": "folder123" }, + "expected": { "remoteName": "google-drive", "basePath": "" } + }, + "rclone": { + "provider": "rclone", + "config": { "configText": "[ovh]\ntype = s3\nprovider = OVHcloud\naccess_key_id = a\nsecret_access_key = b\nendpoint = s3.gra.io.cloud.ovh.net\n", "remoteName": "ovh", "remotePath": "my-bucket" }, + "expected": { "remoteName": "ovh", "basePath": "my-bucket" } + }, + "sftp-key": { + "provider": "sftp", + "config": { "host": "sftp.example.com", "port": 2222, "username": "backup", "privateKey": "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----\n", "remotePath": "/srv/backups" }, + "expected": { "remoteName": "sftp", "basePath": "/srv/backups" } + } +} diff --git a/src/tests/storage/fixtures/rclone-target/rclone.ini b/src/tests/storage/fixtures/rclone-target/rclone.ini new file mode 100644 index 0000000..1d57741 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/rclone.ini @@ -0,0 +1,6 @@ +[ovh] +type = s3 +provider = OVHcloud +access_key_id = a +secret_access_key = b +endpoint = s3.gra.io.cloud.ovh.net diff --git a/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini b/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini new file mode 100644 index 0000000..7a5c1eb --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini @@ -0,0 +1,9 @@ +[s3] +type = s3 +provider = Other +access_key_id = minioadmin +secret_access_key = minioadmin +endpoint = http://minio.local:9000 +region = us-east-1 +force_path_style = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/s3.ini b/src/tests/storage/fixtures/rclone-target/s3.ini new file mode 100644 index 0000000..e9fd105 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/s3.ini @@ -0,0 +1,9 @@ +[s3] +type = s3 +provider = Other +access_key_id = AKIAEXAMPLE +secret_access_key = secretEXAMPLE +endpoint = https://s3.example.com +region = eu-west-1 +force_path_style = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/sftp-key.ini b/src/tests/storage/fixtures/rclone-target/sftp-key.ini new file mode 100644 index 0000000..f7afe12 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/sftp-key.ini @@ -0,0 +1,6 @@ +[sftp] +type = sftp +host = sftp.example.com +port = 2222 +user = backup +key_file = diff --git a/src/tests/storage/fixtures/rclone-validation/cases.json b/src/tests/storage/fixtures/rclone-validation/cases.json new file mode 100644 index 0000000..69fe4c1 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-validation/cases.json @@ -0,0 +1,345 @@ +[ + { + "name": "ovh-s3", + "configText": "[ovhcloud-rbx]\ntype = s3\nprovider = OVHcloud\naccess_key_id = my_access\nsecret_access_key = my_secret\nregion = rbx\nendpoint = s3.rbx.io.cloud.ovh.net\nacl = private\n", + "remoteName": "ovhcloud-rbx", + "ok": true, + "errorContains": null + }, + { + "name": "comments-crlf", + "configText": "# comment\r\n[r]\r\ntype = s3\r\n; note\r\nprovider = AWS\r\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "empty-value", + "configText": "[r]\ntype = webdav\nurl = https://dav.example.com\nuser =\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "duplicate-type", + "configText": "[x]\ntype = s3\ntype = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'type'" + }, + { + "name": "duplicate-type-case", + "configText": "[x]\ntype = s3\nTYPE = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'TYPE'" + }, + { + "name": "colon-separator", + "configText": "[x]\ntype: local\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 2 is not a [section] header" + }, + { + "name": "on-the-fly", + "configText": "[:local]\ntype = s3\n", + "remoteName": ":local", + "ok": false, + "errorContains": "invalid rclone remote name ':local'" + }, + { + "name": "sftp-ssh", + "configText": "[x]\ntype = sftp\nhost = h\nssh = touch /tmp/pwned\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'ssh' is not allowed" + }, + { + "name": "webdav-command", + "configText": "[x]\ntype = webdav\nurl = http://h\nbearer_token_command = id\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'bearer_token_command' is not allowed" + }, + { + "name": "env-auth", + "configText": "[x]\ntype = s3\nenv_auth = true\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'env_auth' is not allowed" + }, + { + "name": "use-msi", + "configText": "[x]\ntype = azureblob\naccount = a\nuse_msi = true\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_msi' is not allowed" + }, + { + "name": "service-account-file", + "configText": "[x]\ntype = google cloud storage\nservice_account_file = /etc/passwd\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'service_account_file' is not allowed" + }, + { + "name": "blocked-local", + "configText": "[x]\ntype = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed" + }, + { + "name": "two-sections", + "configText": "[a]\ntype = s3\n[b]\ntype = s3\n", + "remoteName": "a", + "ok": false, + "errorContains": "exactly one [section] (found 2)" + }, + { + "name": "key-before-section", + "configText": "type = s3\n[x]\ntype = s3\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 1 appears before any [section]" + }, + { + "name": "no-type", + "configText": "[x]\nprovider = AWS\n", + "remoteName": "x", + "ok": false, + "errorContains": "has no type" + }, + { + "name": "remote-mismatch", + "configText": "[a]\ntype = s3\n", + "remoteName": "b", + "ok": false, + "errorContains": "remote 'b' is not defined" + }, + { + "name": "bad-key", + "configText": "[x]\ntype = s3\nbad key = 1\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone config key 'bad key'" + }, + { + "name": "empty-config", + "configText": "", + "remoteName": "r", + "ok": false, + "errorContains": "exactly one [section] (found 0)" + }, + { + "name": "empty-section-name", + "configText": "[]\ntype = s3\n", + "remoteName": "", + "ok": false, + "errorContains": "invalid rclone remote name ''" + }, + { + "name": "name-with-space", + "configText": "[a b]\ntype = s3\n", + "remoteName": "a b", + "ok": false, + "errorContains": "invalid rclone remote name 'a b'" + }, + { + "name": "blocked-type-uppercase", + "configText": "[x]\ntype = LOCAL\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed" + }, + { + "name": "key-file-uppercase", + "configText": "[x]\ntype = sftp\nKEY_FILE = /etc/passwd\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'KEY_FILE' is not allowed" + }, + { + "name": "equals-at-line-start", + "configText": "[x]\n= v\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 2 is not a [section] header" + }, + { + "name": "whitespace-tolerant", + "configText": " [r] \n type=s3 \n\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "quoted-type-backtick", + "configText": "[x]\ntype = `local`\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-type-triple", + "configText": "[x]\ntype = \"\"\"local\"\"\"\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-type-trailing-junk", + "configText": "[x]\ntype = `alias` junk\nremote = /\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-other-value", + "configText": "[x]\ntype = s3\nendpoint = `x`\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'endpoint' must not be quoted (remote 'x')" + }, + { + "name": "blocked-google-photos", + "configText": "[x]\ntype = google photos\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'google photos' is not allowed" + }, + { + "name": "blocked-gphotos", + "configText": "[x]\ntype = gphotos\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'gphotos' is not allowed" + }, + { + "name": "invalid-type-syntax", + "configText": "[x]\ntype = s3!\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone backend type 's3!' (remote 'x')" + }, + { + "name": "forbidden-use-az", + "configText": "[x]\ntype = azureblob\nuse_az = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_az' is not allowed (remote 'x')" + }, + { + "name": "forbidden-use-kerberos", + "configText": "[x]\ntype = hdfs\nuse_kerberos = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_kerberos' is not allowed (remote 'x')" + }, + { + "name": "forbidden-kerberos-ccache", + "configText": "[x]\ntype = hdfs\nkerberos_ccache = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'kerberos_ccache' is not allowed (remote 'x')" + }, + { + "name": "forbidden-key-use-agent", + "configText": "[x]\ntype = sftp\nkey_use_agent = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'key_use_agent' is not allowed (remote 'x')" + }, + { + "name": "forbidden-set-env", + "configText": "[x]\ntype = sftp\nset_env = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'set_env' is not allowed (remote 'x')" + }, + { + "name": "forbidden-unix-socket", + "configText": "[x]\ntype = sftp\nunix_socket = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'unix_socket' is not allowed (remote 'x')" + }, + { + "name": "forbidden-client-certificate-path", + "configText": "[x]\ntype = azureblob\nclient_certificate_path = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'client_certificate_path' is not allowed (remote 'x')" + }, + { + "name": "oracle-env-auth", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = env_auth\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-missing-provider", + "configText": "[x]\ntype = oracleobjectstorage\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-spaced-type", + "configText": "[x]\ntype = oracle object storage\nprovider = instance_principal_auth\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-provider-uppercase-value", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = NO_AUTH\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-provider-uppercase-key", + "configText": "[x]\ntype = oracleobjectstorage\nPROVIDER = no_auth\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-no-auth", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = no_auth\nnamespace = n\nregion = eu-frankfurt-1\n", + "remoteName": "x", + "ok": true, + "errorContains": null + }, + { + "name": "nel-in-type", + "configText": "[x]\ntype = local\u0085\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed (remote 'x')" + }, + { + "name": "order-duplicate-before-forbidden", + "configText": "[x]\ntype = s3\nssh = a\nssh = b\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'ssh' (remote 'x')" + }, + { + "name": "order-bad-line-before-pair-before-header", + "configText": "type = s3\n[x]\nfoo\n", + "remoteName": "x", + "ok": false, + "errorContains": "rclone config line 3 is not a [section] header" + }, + { + "name": "order-invalid-key-before-duplicate", + "configText": "[x]\ntype = s3\nbad key = 1\nbad key = 2\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone config key 'bad key' (remote 'x')" + } +] diff --git a/src/tests/storage/google_cloud_storage.rs b/src/tests/storage/google_cloud_storage.rs index 9a5142d..0b01da6 100644 --- a/src/tests/storage/google_cloud_storage.rs +++ b/src/tests/storage/google_cloud_storage.rs @@ -12,7 +12,7 @@ use testcontainers::{GenericImage, ImageExt}; const BUCKET: &str = "portabase"; -async fn start_fake_gcs() -> (testcontainers::ContainerAsync, String) { +pub(super) async fn start_fake_gcs() -> (testcontainers::ContainerAsync, String) { // Natural random host port (no port-80 pin). The provider forces a single-shot // upload for custom endpoints, which issues one request to this endpoint and never // follows a server-built `Location` — so it works on any port, unlike the resumable diff --git a/src/tests/storage/mod.rs b/src/tests/storage/mod.rs index f6c9152..96840a0 100644 --- a/src/tests/storage/mod.rs +++ b/src/tests/storage/mod.rs @@ -1,4 +1,5 @@ mod azure_blob; mod google_cloud_storage; mod rclone; +mod rclone_target; mod sftp; diff --git a/src/tests/storage/rclone.rs b/src/tests/storage/rclone.rs index 6827cf7..9d40389 100644 --- a/src/tests/storage/rclone.rs +++ b/src/tests/storage/rclone.rs @@ -63,6 +63,41 @@ fn validate_config_accepts_the_target_remote() { assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok()); } +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct ValidationCase { + name: String, + config_text: String, + remote_name: String, + ok: bool, + error_contains: Option, +} + +/// The same file is verified against the dashboard validator: keep both in sync. +#[test] +fn validate_config_matches_the_shared_test_vectors() { + let cases: Vec = + serde_json::from_str(include_str!("fixtures/rclone-validation/cases.json")).unwrap(); + assert!(!cases.is_empty()); + + let mut failures = Vec::new(); + for case in &cases { + match (validate_config(&case.config_text, &case.remote_name), case.ok) { + (Ok(()), true) => {} + (Ok(()), false) => failures.push(format!("[{}] expected an error, got Ok", case.name)), + (Err(e), true) => failures.push(format!("[{}] expected Ok, got: {e:#}", case.name)), + (Err(e), false) => { + let msg = format!("{e:#}"); + let want = case.error_contains.as_deref().expect("ok:false needs errorContains"); + if !msg.contains(want) { + failures.push(format!("[{}] error {msg:?} does not contain {want:?}", case.name)); + } + } + } + } + assert!(failures.is_empty(), "{} case(s) failed:\n{}", failures.len(), failures.join("\n")); +} + #[test] fn validate_config_rejects_an_unknown_remote_name() { let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string(); @@ -85,23 +120,28 @@ fn validate_config_rejects_alias_backend() { } #[test] -fn validate_config_rejects_a_blocked_backend_in_a_chained_section() { +fn validate_config_rejects_chained_sections() { + // A blocked backend hidden behind an allowed (or wrapping) remote can no longer be expressed: + // anything beyond a single [section] is refused outright. let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n"; let err = validate_config(cfg, "secret").unwrap_err().to_string(); - assert!(err.contains("crypt"), "unexpected error: {err}"); - assert!(err.contains("secret"), "error should name the offending remote: {err}"); + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n"; let err = validate_config(cfg, "outer").unwrap_err().to_string(); - assert!(err.contains("local"), "unexpected error: {err}"); - assert!(err.contains("disk"), "error should name the offending remote: {err}"); -} + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); -#[test] -fn validate_config_rejects_crypt_even_over_an_allowed_remote() { let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}"); let err = validate_config(&cfg, "secret").unwrap_err().to_string(); + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); +} + +#[test] +fn validate_config_rejects_crypt_over_an_allowed_remote_name() { + let cfg = "[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n"; + let err = validate_config(cfg, "secret").unwrap_err().to_string(); assert!(err.contains("crypt"), "unexpected error: {err}"); + assert!(err.contains("secret"), "error should name the offending remote: {err}"); } #[test] @@ -179,7 +219,7 @@ use testcontainers::{GenericImage, ImageExt}; const BUCKET: &str = "portabase"; -async fn start_minio() -> (testcontainers::ContainerAsync, String) { +pub(super) async fn start_minio() -> (testcontainers::ContainerAsync, String) { let container = GenericImage::new("coollabsio/minio", "latest") .with_exposed_port(9000.tcp()) .with_wait_for(WaitFor::message_on_stderr("API:")) @@ -208,7 +248,7 @@ fn minio_config(endpoint: &str) -> String { ) } -fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec { +pub(super) fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec { let out = Command::new("rclone") .arg("--config") .arg(config_path) diff --git a/src/tests/storage/rclone_target.rs b/src/tests/storage/rclone_target.rs new file mode 100644 index 0000000..11e99ef --- /dev/null +++ b/src/tests/storage/rclone_target.rs @@ -0,0 +1,301 @@ +use crate::services::api::models::agent::status::DatabaseStorage; +use super::google_cloud_storage::start_fake_gcs; +use super::rclone::{rclone_ok, start_minio}; +use crate::services::storage::providers::rclone::helpers::{obscure_password, write_config}; +use crate::services::storage::providers::rclone::target::rclone_target; +use serde_json::{Value, json}; + +const INPUTS: &str = include_str!("fixtures/rclone-target/inputs.json"); + +pub(super) fn expected_ini(name: &str) -> &'static str { + match name { + "s3" => include_str!("fixtures/rclone-target/s3.ini"), + "s3-port-no-ssl" => include_str!("fixtures/rclone-target/s3-port-no-ssl.ini"), + "blob-account-key" => include_str!("fixtures/rclone-target/blob-account-key.ini"), + "blob-endpoint-url" => include_str!("fixtures/rclone-target/blob-endpoint-url.ini"), + "blob-connection-string" => include_str!("fixtures/rclone-target/blob-connection-string.ini"), + "google-cloud-storage" => include_str!("fixtures/rclone-target/google-cloud-storage.ini"), + "google-cloud-storage-emulator" => { + include_str!("fixtures/rclone-target/google-cloud-storage-emulator.ini") + } + "google-drive" => include_str!("fixtures/rclone-target/google-drive.ini"), + "rclone" => include_str!("fixtures/rclone-target/rclone.ini"), + "sftp-key" => include_str!("fixtures/rclone-target/sftp-key.ini"), + other => panic!("no expected fixture for {other}"), + } +} + +pub(super) fn storage(provider: &str, config: Value) -> DatabaseStorage { + serde_json::from_value(json!({ + "id": "storage-1", + "provider": provider, + "folderName": "backups", + "config": config, + })) + .unwrap() +} + +/// Temp-file paths differ per run; normalize them before comparing. Keeps the exact +/// trailing newline so a missing final `\n` fails the golden test. +pub(super) fn normalize(config_text: &str) -> String { + config_text + .split('\n') + .map(|l| { + if l.starts_with("key_file = ") { + "key_file = ".to_string() + } else { + l.to_string() + } + }) + .collect::>() + .join("\n") +} + +#[test] +fn rclone_target_matches_golden_fixtures() { + let inputs: Value = serde_json::from_str(INPUTS).unwrap(); + for (name, case) in inputs.as_object().unwrap() { + let target = rclone_target(&storage( + case["provider"].as_str().unwrap(), + case["config"].clone(), + )) + .unwrap_or_else(|e| panic!("{name}: {e:#}")); + + assert_eq!(normalize(&target.config_text), expected_ini(name), "{name}: config_text"); + assert_eq!(target.remote_name, case["expected"]["remoteName"].as_str().unwrap(), "{name}: remote_name"); + assert_eq!(target.base_path, case["expected"]["basePath"].as_str().unwrap(), "{name}: base_path"); + } +} + +#[test] +fn rclone_target_rejects_local_and_unknown_providers() { + let err = rclone_target(&storage("local", json!({}))).err().unwrap(); + assert!(format!("{err:#}").contains("a local storage channel cannot be used as an rclone target")); + + let err = rclone_target(&storage("ftp-of-doom", json!({}))).err().unwrap(); + assert!(format!("{err:#}").contains("ftp-of-doom")); +} + +#[test] +fn rclone_target_rejects_azure_without_account_key() { + let err = rclone_target(&storage( + "blob", + json!({ + "authMode": "connectionString", + "connectionString": "BlobEndpoint=https://acct.blob.core.windows.net;SharedAccessSignature=sv=2022&sig=x", + "containerName": "c" + }), + )) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("account key"), "{err:#}"); +} + +#[test] +fn rclone_target_applies_the_rclone_backend_blocklist() { + let err = rclone_target(&storage( + "rclone", + json!({ "configText": "[disk]\ntype = local\n", "remoteName": "disk", "remotePath": "" }), + )) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("not allowed"), "{err:#}"); +} + +#[test] +fn sftp_key_file_lives_exactly_as_long_as_the_target() { + let target = rclone_target(&storage( + "sftp", + json!({ "host": "h", "username": "u", "privateKey": "KEY", "remotePath": "" }), + )) + .unwrap(); + let key_path = target + .config_text + .lines() + .find_map(|l| l.strip_prefix("key_file = ")) + .unwrap() + .to_string(); + assert_eq!(std::fs::read_to_string(&key_path).unwrap(), "KEY"); + drop(target); + assert!(!std::path::Path::new(&key_path).exists(), "key file must be removed with the target"); +} + +#[test] +fn sftp_config_error_keeps_the_root_cause() { + let err = rclone_target(&storage("sftp", json!({ "username": "u", "password": "p" }))) + .err() + .unwrap(); + let text = format!("{err:#}"); + assert!(text.contains("invalid sftp storage config"), "{text}"); + assert!(text.contains("host"), "{text}"); +} + +#[test] +fn sftp_password_only_has_pass_and_no_key_file() { + let target = rclone_target(&storage( + "sftp", + json!({ "host": "h", "username": "u", "password": "secret", "remotePath": "" }), + )) + .unwrap(); + assert!(!target.config_text.contains("key_file"), "{}", target.config_text); + assert!(target.config_text.lines().any(|l| l.starts_with("pass = ")), "{}", target.config_text); +} + +#[test] +fn obscured_password_round_trips_through_rclone_reveal() { + let obscured = obscure_password("s3cr3t").unwrap(); + assert_ne!(obscured, "s3cr3t"); + + let out = std::process::Command::new("rclone").args(["reveal", &obscured]).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "s3cr3t"); +} + +#[tokio::test] +async fn s3_target_reaches_minio_through_rclone() { + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + + let target = rclone_target(&storage( + "s3", + json!({ "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }), + )) + .unwrap(); + let config = write_config(&target.config_text).unwrap(); + + // The mapper sets `no_check_bucket = true`, so create the bucket with a per-call override. + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap(); + assert!(listing.contains("portabase"), "lsd output: {listing}"); +} + +#[tokio::test] +async fn gcs_emulator_target_reaches_fake_gcs_through_rclone() { + let (_container, endpoint) = start_fake_gcs().await; // "http://host:port" + + let target = rclone_target(&storage( + "google-cloud-storage", + json!({ "projectId": "test", "bucketName": "bucket1", "clientEmail": "x@test", + "privateKey": "", "apiEndpoint": endpoint }), + )) + .unwrap(); + let config = write_config(&target.config_text).unwrap(); + + // Same per-call override as the s3 test: the mapper never creates buckets on its own. + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap(); + assert!(listing.contains("bucket1"), "lsd output: {listing}"); +} + +#[tokio::test] +async fn s3_target_hosts_a_restic_repository() { + use crate::services::backup::logger::JobLogger; + use crate::services::restic::backup::snapshot; + use crate::services::restic::command::ResticRepo; + use base64::{Engine as _, engine::general_purpose}; + + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + let channel = storage( + "s3", + json!({ "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }), + ); + let target = rclone_target(&channel).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + + let src = tempfile::TempDir::new().unwrap(); + std::fs::write(src.path().join("a.txt"), "a").unwrap(); + let cfg = crate::tests::domain::files::files_config(src.path(), &[]); + let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]); + + let edge_key = crate::utils::edge_key::EdgeKey { + server_url: String::new(), + agent_id: "agent-1".into(), + master_key_b64: master_key_b64.clone(), + }; + let repo = ResticRepo::open(&channel, &cfg.generated_id, &edge_key).unwrap(); + assert_eq!(repo.repository(), format!("rclone:s3:portabase/backups/restic/{}", cfg.generated_id)); + let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + + let listed = String::from_utf8(rclone_ok( + config.path(), + &["lsf", &format!("s3:portabase/backups/restic/{}/snapshots", cfg.generated_id)], + )) + .unwrap(); + assert!(listed.contains(&snap.snapshot_id), "{listed}"); +} + +#[tokio::test] +async fn s3_targets_host_one_sync_replica_per_channel() { + use crate::services::backup::logger::JobLogger; + use crate::services::sync::backup::one_storage; + use crate::tests::services::backup_uploader_tests::ctx_pointing_at; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + let channel = |id: &str, bucket: &str| -> DatabaseStorage { + serde_json::from_value(json!({ + "id": id, "provider": "s3", "folderName": "backups", + "config": { "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": bucket } + })) + .unwrap() + }; + let channels = [channel("ch-a", "portabase"), channel("ch-b", "portabase2")]; + for c in &channels { + let target = rclone_target(c).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + } + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/agent/agent-1/backup/upload/init")) + .and(body_partial_json(json!({ "engine": "sync" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&server) + .await; + Mock::given(method("PATCH")) + .and(path("/agent/agent-1/backup/upload/status")) + .and(body_partial_json(json!({ "status": "success", "filesTransferred": 2, "filesDeleted": 0 }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&server) + .await; + + let src = tempfile::TempDir::new().unwrap(); + std::fs::create_dir_all(src.path().join("docs")).unwrap(); + std::fs::write(src.path().join("a.txt"), "a").unwrap(); + std::fs::write(src.path().join("docs/b.txt"), "b").unwrap(); + let cfg = crate::tests::domain::files::files_config(src.path(), &[]); + let ctx = ctx_pointing_at(server.uri()); + + for c in &channels { + let result = one_storage(&ctx, &cfg, c, "backup-1", &JobLogger::new()).await; + assert!(result.success, "{:?}", result.error); + assert_eq!(result.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); + } + for (c, bucket) in channels.iter().zip(["portabase", "portabase2"]) { + let target = rclone_target(c).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let listed = String::from_utf8(rclone_ok( + config.path(), + &["lsf", "-R", &format!("s3:{bucket}/backups/sync/{}/current", cfg.generated_id)], + )) + .unwrap(); + assert!(listed.contains("a.txt") && listed.contains("docs/b.txt"), "{bucket}: {listed}"); + } +} diff --git a/src/utils/task_manager/scheduler.rs b/src/utils/task_manager/scheduler.rs index c1d248c..8e59b01 100644 --- a/src/utils/task_manager/scheduler.rs +++ b/src/utils/task_manager/scheduler.rs @@ -32,10 +32,8 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) { let mut conn_clone = conn.clone(); tokio::spawn(async move { - info!( - "Executing task={} args={:?} metadata={:?}", - task_clone.task, task_clone.args, task_clone.metadata - ); + // Never log metadata: it carries decrypted storage channels (secrets). + info!("Executing task={} args={:?}", task_clone.task, task_clone.args); if let Err(e) = execute_task( task_clone.task.as_str(), task_clone.args, @@ -65,6 +63,14 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) { } } +/// Local config merged with the dashboard-pushed sources, as the agent loop sees them. +fn merged_config(ctx: &Arc) -> crate::services::config::DatabasesConfig { + let local = ConfigService::new(ctx.clone()).load_optional(None); + let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path).join("dashboard_databases.json"); + let dashboard = crate::services::dashboard_config::load_cache(&cache_path); + crate::services::dashboard_config::merge(&local.databases, &dashboard) +} + pub async fn execute_task( task: &str, args: Vec, @@ -77,14 +83,9 @@ pub async fn execute_task( info!("{} | {}", generated_id, dbms); let ctx = Arc::new(Context::new()); - let config_service = ConfigService::new(ctx.clone()); let backup_service = BackupService::new(ctx.clone()); - let local = config_service.load_optional(None); - let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path) - .join("dashboard_databases.json"); - let dashboard = crate::services::dashboard_config::load_cache(&cache_path); - let config = crate::services::dashboard_config::merge(&local.databases, &dashboard); + let config = merged_config(&ctx); let metadata_obj = metadata .into_iter() @@ -101,6 +102,8 @@ pub async fn execute_task( let storages: Vec = serde_json::from_value(storages_value.clone())?; let encrypt: bool = serde_json::from_value(encrypt_value.clone())?; + // Tasks stored before P2 have no engine: archive. + let engine = metadata_obj.get("engine").and_then(Value::as_str).unwrap_or("archive"); backup_service .dispatch( @@ -109,6 +112,7 @@ pub async fn execute_task( BackupMethod::Automatic, &storages, encrypt, + engine, ) .await; From d1fc31c0ba0600c937c5c238cad1e765e458016b Mon Sep 17 00:00:00 2001 From: Charles Gauthereau Date: Fri, 9 Oct 2026 15:33:03 +0200 Subject: [PATCH 02/11] fix: files --- .gitignore | 2 + Cargo.lock | 2 +- Cargo.toml | 2 +- databases.json | 32 ++++++++++++++ docker-compose.yml | 5 ++- justfile | 3 ++ .../endpoints/agent/backup/upload/status.rs | 24 +++++------ src/services/restic/backup.rs | 41 ++++++++++++++---- src/services/restic/json.rs | 42 ++++++++++++++++++- src/services/sync/backup.rs | 6 +-- src/services/sync/rclone.rs | 39 +++++++++++++++-- src/services/sync/stats.rs | 16 ++++++- src/tests/services/api_models_tests.rs | 20 +++++---- src/tests/services/restic_tests.rs | 26 ++++++++++++ src/tests/services/sync_tests.rs | 24 ++++++++++- src/tests/storage/rclone_target.rs | 3 +- 16 files changed, 240 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 675c67b..1e2220f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ /docs .superpowers + +/test-files diff --git a/Cargo.lock b/Cargo.lock index 6a4b2dc..954596a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3665,7 +3665,7 @@ dependencies = [ [[package]] name = "portabase-agent" -version = "1.21.3" +version = "1.22.0" dependencies = [ "aes", "aes-gcm", diff --git a/Cargo.toml b/Cargo.toml index ab3bded..90f164c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "portabase-agent" -version = "1.21.3" +version = "1.22.0" edition = "2024" license = "Apache-2.0" diff --git a/databases.json b/databases.json index 3b66894..f480f42 100644 --- a/databases.json +++ b/databases.json @@ -131,6 +131,38 @@ "volume_name": "databases_sqlite-data", "generated_id": "16706126-ff7e-4c97-8c83-0adeff214690", "container_name": "db-sqlite" + }, + { + "name": "Files - Archive", + "type": "files", + "path": "/data/files-archive", + "generated_id": "64aeaf80-e03b-4503-a5c1-00a1b829f0e5", + "options": { + "method": "archive", + "exclude": ["node_modules", "*.tmp", "/cache"], + "one_file_system": false + } + }, + { + "name": "Files - Snapshots", + "type": "files", + "path": "/data/files-snapshot", + "generated_id": "9093ad85-3f67-489f-9759-aa369572b30d", + "options": { + "method": "snapshot", + "exclude": ["node_modules", "*.log", "/build"], + "one_file_system": true + } + }, + { + "name": "Files - Sync", + "type": "files", + "path": "/data/files-sync", + "generated_id": "353ecc44-7c75-40e7-803e-ac07683c2c05", + "options": { + "method": "sync", + "exclude": ["*.tmp", "/cache"] + } } ] } diff --git a/docker-compose.yml b/docker-compose.yml index 699ed22..aae6a57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,12 +16,15 @@ services: - databases_sqlite-data:/sqlite-data/workspace/data - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data # - /bigdisk:/scratch + - ./test-files/archive:/data/files-archive + - ./test-files/snapshot:/data/files-snapshot + - ./test-files/sync:/data/files-sync environment: APP_ENV: development LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiYTQyNjQzNTQtZGE3Ni00OWFkLWJkYjctZDVjMjMwYzhjYmViIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiODBkZTY1NDktNzZjYS00NGZhLWIyNTItMTY0MWI2YTQzZjJkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 diff --git a/justfile b/justfile index 3063c15..ac701de 100644 --- a/justfile +++ b/justfile @@ -77,3 +77,6 @@ test: docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync" echo "Down volumes tests databases" docker compose -f docker-compose.test.yml down --volumes + +seed-files: + for d in archive snapshot sync; do mkdir -p test-files/$d && echo one > test-files/$d/a.txt && echo two > test-files/$d/b.txt; done \ No newline at end of file diff --git a/src/services/api/endpoints/agent/backup/upload/status.rs b/src/services/api/endpoints/agent/backup/upload/status.rs index 9ce4e00..441b49d 100644 --- a/src/services/api/endpoints/agent/backup/upload/status.rs +++ b/src/services/api/endpoints/agent/backup/upload/status.rs @@ -1,9 +1,9 @@ use crate::services::api::models::agent::backup::BackupUploadResponse; use crate::services::api::{ApiClient, ApiError}; -use crate::services::sync::stats::SyncStats; use anyhow::Result; use reqwest::Method; use serde::Serialize; +use serde_json::Value; #[derive(Serialize)] pub struct StatusUploadRequest { @@ -16,10 +16,9 @@ pub struct StatusUploadRequest { pub size: u64, #[serde(rename = "backupId")] pub backup_id: String, - #[serde(rename = "filesTransferred", skip_serializing_if = "Option::is_none")] - pub files_transferred: Option, - #[serde(rename = "filesDeleted", skip_serializing_if = "Option::is_none")] - pub files_deleted: Option, + /// Counters of a successful snapshot or sync run, shown by the dashboard. + #[serde(skip_serializing_if = "Option::is_none")] + pub stats: Option, } impl ApiClient { @@ -40,20 +39,20 @@ impl ApiClient { path: remote_path.into(), size: total_size.into(), backup_id: backup_id.into(), - files_transferred: None, - files_deleted: None, + stats: None, }; self.send_upload_status(agent_id.into(), &body).await } - /// Success status of one sync replica, with rclone's counters (`size` = bytes transferred). - pub async fn backup_upload_sync_status( + /// Success status of one snapshot or sync run, with its counters. + pub async fn backup_upload_success( &self, agent_id: impl Into, generated_id: impl Into, backup_storage_id: impl Into, path: String, - stats: &SyncStats, + size: u64, + stats: Value, backup_id: impl Into, ) -> Result, ApiError> { let body = StatusUploadRequest { @@ -61,10 +60,9 @@ impl ApiClient { backup_storage_id: backup_storage_id.into(), status: "success".into(), path, - size: stats.bytes, + size, backup_id: backup_id.into(), - files_transferred: Some(stats.transfers), - files_deleted: Some(stats.deletes), + stats: Some(stats), }; self.send_upload_status(agent_id.into(), &body).await } diff --git a/src/services/restic/backup.rs b/src/services/restic/backup.rs index c713a17..401fb83 100644 --- a/src/services/restic/backup.rs +++ b/src/services/restic/backup.rs @@ -1,6 +1,6 @@ use super::command::{ResticRepo, short}; use super::excludes::backup_excludes; -use super::json::BackupSummary; +use super::json::{BackupSummary, Snapshot, files_removed}; use crate::core::context::Context as CoreContext; use crate::domain::files::matcher::{exclude_patterns, one_file_system}; use crate::services::api::models::agent::status::DatabaseStorage; @@ -27,14 +27,15 @@ pub async fn snapshot( } repo.ensure_initialized(logger).await?; + // Stable host: container hostnames change on recreate and would break + // parent-snapshot detection (full rescan every time). + let host = format!("portabase-{}", cfg.generated_id); let mut args: Vec = vec![ "backup".into(), root.to_string_lossy().into_owned(), "--json".into(), - // Stable host: container hostnames change on recreate and would break - // parent-snapshot detection (full rescan every time). "--host".into(), - format!("portabase-{}", cfg.generated_id), + host.clone(), "--tag".into(), "portabase".into(), "--tag".into(), @@ -61,12 +62,22 @@ pub async fn snapshot( .context("unexpected restic backup summary")?; match (run.code, summary) { - (0, Some(s)) => { + (0, Some(mut s)) => { + s.files_removed = match removed(repo, &host, &s, logger).await { + Ok(removed) => removed, + Err(e) => { + logger.log("warn", format!("Could not count removed files: {e:#}")); + None + } + }; + let removed = s.files_removed.map(|n| format!(", {n} removed")).unwrap_or_default(); logger.log( "info", + // data_added_packed: new file and directory blobs, compressed and encrypted. The + // repository grows a little more (pack headers, index and snapshot files). format!( - "Snapshot {}: {} new, {} changed, {} unmodified file(s), {} byte(s) added", - short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.data_added_packed + "Snapshot {}: {} new, {} changed, {} unmodified{removed} file(s); {} byte(s) scanned, {} byte(s) of new data", + short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.total_bytes_processed, s.data_added_packed ), ); Ok(s) @@ -100,6 +111,18 @@ pub async fn snapshot( } } +/// Removed files of snapshot `s`, against the parent restic actually used (one listing of +/// the source's snapshots, no index load). +async fn removed(repo: &ResticRepo, host: &str, s: &BackupSummary, logger: &JobLogger) -> Result> { + let listed = repo.run(["snapshots", "--json", "--host", host], logger).await?; + if listed.code != 0 { + return Err(listed.error("snapshots")); + } + let snapshots: Vec = + serde_json::from_str(listed.stdout.trim()).context("unexpected `restic snapshots` output")?; + Ok(files_removed(&snapshots, &s.snapshot_id, s.files_changed, s.files_unmodified)) +} + /// Snapshots every storage in turn (one scan each, v1). Never errors: each /// storage reports its own `backup_storage` row, like the archive uploader. pub async fn run( @@ -163,13 +186,13 @@ pub(crate) async fn one_storage( match outcome { Ok(summary) => match ctx .api - .backup_upload_status( + .backup_upload_success( agent_id, cfg.generated_id.clone(), backup_storage_id, - "success", summary.snapshot_id.clone(), summary.data_added_packed, + summary.report(), backup_id, ) .await diff --git a/src/services/restic/json.rs b/src/services/restic/json.rs index a4bfb9d..e933830 100644 --- a/src/services/restic/json.rs +++ b/src/services/restic/json.rs @@ -1,5 +1,5 @@ use serde::Deserialize; -use serde_json::Value; +use serde_json::{Value, json}; /// One line of `restic --json` output. stdout carries `status` / `summary`, /// stderr carries `error` / `exit_error` and plain-text fatal messages. @@ -41,12 +41,50 @@ pub struct BackupSummary { pub files_new: u64, pub files_changed: u64, pub files_unmodified: u64, + /// Files of the parent snapshot this one no longer has: restic does not count them + /// (see `files_removed`). None when unknown. + #[serde(skip)] + pub files_removed: Option, +} + +impl BackupSummary { + /// Counters the dashboard shows for this snapshot (`size` carries `data_added_packed`). + pub fn report(&self) -> Value { + json!({ + "filesNew": self.files_new, + "filesChanged": self.files_changed, + "filesUnmodified": self.files_unmodified, + "filesRemoved": self.files_removed, + }) + } } /// One entry of `restic snapshots --json`. #[derive(Debug, Deserialize)] pub struct Snapshot { - #[allow(dead_code)] // part of the `restic snapshots` shape; restore() only needs `paths` pub id: String, pub paths: Vec, + #[serde(default)] + pub parent: Option, + /// Written by restic >= 0.17. + #[serde(default)] + pub summary: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SnapshotSummary { + pub total_files_processed: u64, +} + +/// Files of the parent missing from `snapshot_id`, from a `restic snapshots --json` listing: +/// restic counts a file as changed or unmodified only when the parent has it, so the parent's +/// file count minus those is what was removed. Some(0) without a parent; None when the parent +/// is not listed or has no summary. +pub fn files_removed(snapshots: &[Snapshot], snapshot_id: &str, changed: u64, unmodified: u64) -> Option { + let find = |id: &str| snapshots.iter().find(|s| s.id == id); + let Some(parent) = find(snapshot_id)?.parent.as_deref() else { + return Some(0); + }; + let total = find(parent)?.summary.as_ref()?.total_files_processed; + Some(total.saturating_sub(changed + unmodified)) } diff --git a/src/services/sync/backup.rs b/src/services/sync/backup.rs index a708d95..33fbd3b 100644 --- a/src/services/sync/backup.rs +++ b/src/services/sync/backup.rs @@ -115,7 +115,7 @@ pub(crate) async fn one_storage( let path = replica_path(storage, &cfg.generated_id); match ctx .api - .backup_upload_sync_status(agent_id, cfg.generated_id.clone(), backup_storage_id, path.clone(), &stats, backup_id) + .backup_upload_success(agent_id, cfg.generated_id.clone(), backup_storage_id, path.clone(), stats.bytes, stats.report(), backup_id) .await { Ok(_) => UploadResult { @@ -123,8 +123,8 @@ pub(crate) async fn one_storage( success: true, error: None, remote_file_path: Some(path), - // No total size: rclone does not report the replica's size (backups.file_size stays null). - total_size: None, + // backups.file_size: the replica's size, like an archive's file size. + total_size: stats.replica_bytes, }, Err(e) => { logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); diff --git a/src/services/sync/rclone.rs b/src/services/sync/rclone.rs index 1532b3a..6d361a8 100644 --- a/src/services/sync/rclone.rs +++ b/src/services/sync/rclone.rs @@ -5,6 +5,7 @@ use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; use anyhow::{Context, Result, anyhow, bail}; +use std::path::Path; use std::process::Stdio; use tokio::process::Command; @@ -13,7 +14,8 @@ pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generate remote_target(remote_name, base_path, &format!("{folder}/sync/{generated_id}/current")) } -/// Mirrors `cfg.path` onto `dest` (`:`). rclone skips its deletions when an error +/// Mirrors `cfg.path` onto `dest` (`:`); files matching an exclude are removed +/// from the replica too (`--delete-excluded`). rclone skips its deletions when an error /// happens before the delete phase ("not deleting files as there were IO errors"); /// an error during the delete phase can leave the replica partially updated. /// An empty source is refused so an unmounted volume cannot wipe the replica. @@ -42,7 +44,7 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge .arg(&root) .arg(dest) .args([ - "--links", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1", + "--links", "--delete-excluded", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1", // An unreachable endpoint fails in ~1–2 min instead of blocking the next runs. "--contimeout", "30s", "--low-level-retries", "3", ]); @@ -77,10 +79,39 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge .unwrap_or_else(|| "no error message".to_string()); bail!("rclone sync failed ({}): {reason}; the replica may be partially updated", out.status); } - let stats = log.stats.unwrap_or_default(); + let mut stats = log.stats.unwrap_or_default(); + // Measured on the replica itself, so it is what is stored, not inferred from the source. + stats.replica_bytes = match remote_size(config.path(), dest).await { + Ok(bytes) => Some(bytes), + Err(e) => { + logger.log("warn", format!("Could not measure the replica size: {e:#}")); + None + } + }; + let replica = stats.replica_bytes.map(|b| format!("; replica holds {b} bytes")).unwrap_or_default(); logger.log( "info", - format!("Synced: {} file(s) transferred ({} bytes), {} deleted", stats.transfers, stats.bytes, stats.deletes), + format!("Synced: {} file(s) transferred ({} bytes), {} deleted{replica}", stats.transfers, stats.bytes, stats.deletes), ); Ok(stats) } + +/// Bytes stored under `target` (`:`), per `rclone size`. +async fn remote_size(config: &Path, target: &str) -> Result { + let out = Command::new("rclone") + .args(["--contimeout", "30s", "--low-level-retries", "3", "--config"]) + .arg(config) + .arg("size") + .arg(target) + .arg("--json") + .stdin(Stdio::null()) + .kill_on_drop(true) + .output() + .await + .context("failed to start rclone")?; + if !out.status.success() { + bail!("rclone size failed ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim()); + } + let size: serde_json::Value = serde_json::from_slice(&out.stdout).context("unexpected rclone size output")?; + size["bytes"].as_u64().context("rclone size returned no byte count") +} diff --git a/src/services/sync/stats.rs b/src/services/sync/stats.rs index 6c1246d..465c8c8 100644 --- a/src/services/sync/stats.rs +++ b/src/services/sync/stats.rs @@ -1,5 +1,5 @@ use serde::Deserialize; -use serde_json::Value; +use serde_json::{Value, json}; /// Final `stats` object of `rclone --use-json-log --stats 1h --stats-log-level NOTICE`. #[derive(Debug, Default, Deserialize, PartialEq)] @@ -14,6 +14,20 @@ pub struct SyncStats { pub errors: u64, #[serde(default, rename = "lastError")] pub last_error: Option, + /// Replica size after a successful run, from `rclone size` on the destination (not an rclone stat). + #[serde(skip)] + pub replica_bytes: Option, +} + +impl SyncStats { + /// Counters the dashboard shows for this run (`size` carries `bytes`). + pub fn report(&self) -> Value { + json!({ + "filesTransferred": self.transfers, + "filesDeleted": self.deletes, + "replicaBytes": self.replica_bytes, + }) + } } /// What `rclone sync` printed on stderr: its last stats object and its error diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index aa892a4..f09c24e 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -327,21 +327,23 @@ fn ping_payload_sends_method_only_when_set() { } #[test] -fn upload_status_sends_sync_counters_only_when_set() { +fn upload_status_sends_stats_only_when_set() { use crate::services::api::endpoints::agent::backup::upload::status::StatusUploadRequest; - let request = |files_transferred: Option, files_deleted: Option| StatusUploadRequest { + use crate::services::sync::stats::SyncStats; + let request = |stats: Option| StatusUploadRequest { generated_id: "g".into(), backup_storage_id: "bs".into(), status: "success".into(), path: "p".into(), size: 3, backup_id: "b".into(), - files_transferred, - files_deleted, + stats, }; - let with = serde_json::to_value(request(Some(2), Some(1))).unwrap(); - assert_eq!(with["filesTransferred"], 2); - assert_eq!(with["filesDeleted"], 1); - let without = serde_json::to_value(request(None, None)).unwrap(); - assert!(without.get("filesTransferred").is_none() && without.get("filesDeleted").is_none(), "{without}"); + let sync = SyncStats { transfers: 2, deletes: 1, replica_bytes: Some(8), ..Default::default() }; + let with = serde_json::to_value(request(Some(sync.report()))).unwrap(); + assert_eq!(with["stats"], serde_json::json!({"filesTransferred": 2, "filesDeleted": 1, "replicaBytes": 8})); + let unmeasured = SyncStats::default().report(); + assert!(unmeasured["replicaBytes"].is_null(), "{unmeasured}"); + let without = serde_json::to_value(request(None)).unwrap(); + assert!(without.get("stats").is_none(), "{without}"); } diff --git a/src/tests/services/restic_tests.rs b/src/tests/services/restic_tests.rs index c4cd0c2..d9046a3 100644 --- a/src/tests/services/restic_tests.rs +++ b/src/tests/services/restic_tests.rs @@ -135,6 +135,24 @@ fn backup_summary_deserializes_from_a_recorded_line() { assert_eq!(summary.files_new, 3); } +#[test] +fn removed_files_come_from_the_parent_summary() { + use crate::services::restic::json::{Snapshot, files_removed}; + let listing: Vec = serde_json::from_value(serde_json::json!([ + {"id": "p", "paths": ["/src"], "summary": {"total_files_processed": 3, "files_new": 3}}, + {"id": "c", "paths": ["/src"], "parent": "p"}, + {"id": "orphan", "paths": ["/src"], "parent": "forgotten"}, + {"id": "old", "paths": ["/src"]}, + {"id": "child-of-old", "paths": ["/src"], "parent": "old"}, + ])) + .unwrap(); + assert_eq!(files_removed(&listing, "c", 0, 2), Some(1)); + assert_eq!(files_removed(&listing, "p", 0, 0), Some(0), "no parent"); + assert_eq!(files_removed(&listing, "orphan", 0, 2), None, "parent not listed"); + assert_eq!(files_removed(&listing, "child-of-old", 0, 2), None, "parent without summary"); + assert_eq!(files_removed(&listing, "missing", 0, 2), None); +} + #[test] fn a_local_channel_opens_the_dashboard_rest_repository() { use crate::utils::edge_key::EdgeKey; @@ -214,12 +232,14 @@ async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_par let first = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); assert_eq!(first.snapshot_id.len(), 64); assert_eq!(first.files_new, 4); + assert_eq!(first.files_removed, Some(0)); assert!(base.path().join("repo/backups/restic").join(&cfg.generated_id).join("config").exists()); fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); let second = snapshot(&repo, &cfg, "bs-2", &logger).await.unwrap(); assert_eq!(second.files_changed, 1); assert_eq!(second.files_unmodified, 3); + assert_eq!(second.files_removed, Some(0)); let tagged = repo.run(["snapshots", "--json", "--tag", "bs:bs-2"], &logger).await.unwrap(); // The tag selects only the second snapshot, and its `parent` is the first @@ -229,6 +249,12 @@ async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_par assert_eq!(listed.len(), 1, "{}", tagged.stdout); assert_eq!(listed[0]["id"], second.snapshot_id.as_str()); assert_eq!(listed[0]["parent"], first.snapshot_id.as_str()); + + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + let third = snapshot(&repo, &cfg, "bs-3", &logger).await.unwrap(); + assert_eq!((third.files_new, third.files_changed, third.files_unmodified), (0, 0, 3)); + assert_eq!(third.files_removed, Some(1)); + assert_eq!(third.report()["filesRemoved"], 1); } #[tokio::test] diff --git a/src/tests/services/sync_tests.rs b/src/tests/services/sync_tests.rs index 98f408c..6c6a2e6 100644 --- a/src/tests/services/sync_tests.rs +++ b/src/tests/services/sync_tests.rs @@ -59,7 +59,7 @@ fn parse_log_reads_the_final_stats_and_error_lines() { let log = parse_log(stderr); assert_eq!( log.stats, - Some(SyncStats { bytes: 0, transfers: 0, deletes: 0, errors: 1, last_error: Some("couldn't copy from /src/secret: errno -1".into()) }) + Some(SyncStats { errors: 1, last_error: Some("couldn't copy from /src/secret: errno -1".into()), ..Default::default() }) ); assert_eq!(log.errors.len(), 2); assert_eq!(log.errors[1], "not deleting files as there were IO errors"); @@ -69,7 +69,7 @@ fn parse_log_reads_the_final_stats_and_error_lines() { fn parse_log_reads_a_successful_run() { let stderr = r#"{"level":"notice","msg":"\nTransferred: 2 B\n","source":"accounting/stats.go:1","stats":{"bytes":2,"checks":5,"deletes":1,"errors":0,"fatalError":false,"transfers":1},"time":"2026-10-03T19:45:10+02:00"}"#; let log = parse_log(stderr); - assert_eq!(log.stats, Some(SyncStats { bytes: 2, transfers: 1, deletes: 1, errors: 0, last_error: None })); + assert_eq!(log.stats, Some(SyncStats { bytes: 2, transfers: 1, deletes: 1, ..Default::default() })); assert!(log.errors.is_empty()); } @@ -167,6 +167,7 @@ async fn sync_mirrors_the_source_and_propagates_changes() { let first = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); assert_eq!(first.transfers, 2); + assert_eq!(first.replica_bytes, Some(3), "excluded files are not counted"); assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1", "docs/b.txt=b"]); fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); @@ -176,9 +177,28 @@ async fn sync_mirrors_the_source_and_propagates_changes() { let second = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); assert_eq!(second.transfers, 2); assert_eq!(second.deletes, 1); + assert_eq!(second.replica_bytes, Some(5)); assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v2", "new.txt=new"]); } +#[tokio::test] +async fn a_later_exclude_removes_files_from_the_replica() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let config = alias(store.path()); + let dest = replica_target("pb", "", "backups", &cfg.generated_id); + sync_dir(&config, &dest, &cfg, &JobLogger::new()).await.unwrap(); + + let mut later = cfg.clone(); + later.options = files_config(src.path(), &["*.log", "/build", "b.txt"]).options; + let stats = sync_dir(&config, &dest, &later, &JobLogger::new()).await.unwrap(); + assert_eq!(stats.deletes, 1); + assert_eq!(stats.replica_bytes, Some(2)); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1"]); +} + #[tokio::test] async fn failed_sync_keeps_files_the_failed_run_would_delete() { if is_root() { diff --git a/src/tests/storage/rclone_target.rs b/src/tests/storage/rclone_target.rs index 11e99ef..4812bf1 100644 --- a/src/tests/storage/rclone_target.rs +++ b/src/tests/storage/rclone_target.rs @@ -270,7 +270,7 @@ async fn s3_targets_host_one_sync_replica_per_channel() { .await; Mock::given(method("PATCH")) .and(path("/agent/agent-1/backup/upload/status")) - .and(body_partial_json(json!({ "status": "success", "filesTransferred": 2, "filesDeleted": 0 }))) + .and(body_partial_json(json!({ "status": "success", "size": 2, "stats": { "filesTransferred": 2, "filesDeleted": 0, "replicaBytes": 2 } }))) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) .expect(2) .mount(&server) @@ -287,6 +287,7 @@ async fn s3_targets_host_one_sync_replica_per_channel() { let result = one_storage(&ctx, &cfg, c, "backup-1", &JobLogger::new()).await; assert!(result.success, "{:?}", result.error); assert_eq!(result.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); + assert_eq!(result.total_size, Some(2), "backups.file_size is the replica size"); } for (c, bucket) in channels.iter().zip(["portabase", "portabase2"]) { let target = rclone_target(c).unwrap(); From 7c262ce14ef54361325fccfa1ed480996348513f Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Sun, 18 Oct 2026 19:01:54 +0200 Subject: [PATCH 03/11] fix: files refactoring --- docker-compose.yml | 2 +- justfile | 6 +- scripts/README.md | 1 + src/domain/files/backup.rs | 26 +- src/domain/files/matcher.rs | 39 ++- src/domain/files/mod.rs | 2 + src/domain/files/ping.rs | 1 - src/domain/files/platform.rs | 1 - src/domain/files/restic/backup.rs | 106 +++++++++ .../files}/restic/excludes.rs | 31 +-- src/{services => domain/files}/restic/json.rs | 11 - src/domain/files/restic/mod.rs | 6 + src/domain/files/restic/password.rs | 7 + .../files/restic/repo.rs} | 35 +-- .../files}/restic/restore.rs | 35 +-- src/domain/files/restore.rs | 43 +--- src/domain/files/sync/backup.rs | 45 ++++ .../files}/sync/excludes.rs | 12 +- src/domain/files/sync/mod.rs | 4 + src/{services => domain/files}/sync/rclone.rs | 17 +- src/{services => domain/files}/sync/stats.rs | 6 - src/services/backup/executor.rs | 13 +- src/services/backup/files_engine.rs | 166 +++++++++++++ src/services/backup/mod.rs | 1 + src/services/dashboard_config.rs | 1 - src/services/mod.rs | 2 - src/services/restic/backup.rs | 224 ------------------ src/services/restic/mod.rs | 7 - src/services/restic/password.rs | 42 ---- src/services/restore/executor.rs | 34 ++- src/services/status.rs | 4 - src/services/storage/providers/local.rs | 8 + src/services/sync/backup.rs | 147 ------------ src/services/sync/mod.rs | 6 - src/tests/domain/{files.rs => files/mod.rs} | 3 + .../files/restic.rs} | 45 +--- .../sync_tests.rs => domain/files/sync.rs} | 20 +- src/tests/services/api_models_tests.rs | 2 +- src/tests/services/local_storage_tests.rs | 22 +- src/tests/services/mod.rs | 2 - src/tests/storage/rclone_target.rs | 8 +- src/utils/edge_key.rs | 21 ++ src/utils/file.rs | 13 +- 43 files changed, 503 insertions(+), 724 deletions(-) create mode 100644 src/domain/files/restic/backup.rs rename src/{services => domain/files}/restic/excludes.rs (54%) rename src/{services => domain/files}/restic/json.rs (75%) create mode 100644 src/domain/files/restic/mod.rs create mode 100644 src/domain/files/restic/password.rs rename src/{services/restic/command.rs => domain/files/restic/repo.rs} (80%) rename src/{services => domain/files}/restic/restore.rs (68%) create mode 100644 src/domain/files/sync/backup.rs rename src/{services => domain/files}/sync/excludes.rs (68%) create mode 100644 src/domain/files/sync/mod.rs rename src/{services => domain/files}/sync/rclone.rs (80%) rename src/{services => domain/files}/sync/stats.rs (80%) create mode 100644 src/services/backup/files_engine.rs delete mode 100644 src/services/restic/backup.rs delete mode 100644 src/services/restic/mod.rs delete mode 100644 src/services/restic/password.rs delete mode 100644 src/services/sync/backup.rs delete mode 100644 src/services/sync/mod.rs rename src/tests/domain/{files.rs => files/mod.rs} (99%) rename src/tests/{services/restic_tests.rs => domain/files/restic.rs} (87%) rename src/tests/{services/sync_tests.rs => domain/files/sync.rs} (93%) diff --git a/docker-compose.yml b/docker-compose.yml index ab4b9b7..c8a0fc2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiODBkZTY1NDktNzZjYS00NGZhLWIyNTItMTY0MWI2YTQzZjJkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGUyMzgxYzgtNzI4Mi00MzIxLTk1ZjQtYTU3NTRiZTc2YmFlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 diff --git a/justfile b/justfile index ac701de..390e720 100644 --- a/justfile +++ b/justfile @@ -79,4 +79,8 @@ test: docker compose -f docker-compose.test.yml down --volumes seed-files: - for d in archive snapshot sync; do mkdir -p test-files/$d && echo one > test-files/$d/a.txt && echo two > test-files/$d/b.txt; done \ No newline at end of file + for d in archive snapshot sync; do mkdir -p test-files/$d && echo one > test-files/$d/a.txt && echo two > test-files/$d/b.txt; done + +seed-files-1gb: + echo "Seeding files sources (1GB each)..." + bash ./scripts/files/seed-1gb.sh diff --git a/scripts/README.md b/scripts/README.md index f94a41e..4b4b58a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -11,6 +11,7 @@ make seed-postgres-1gb make seed-all make seed-sqlite make seed-sqlite SEED=big +just seed-files-1gb ``` ## Verify commands diff --git a/src/domain/files/backup.rs b/src/domain/files/backup.rs index 4169eac..153afbd 100644 --- a/src/domain/files/backup.rs +++ b/src/domain/files/backup.rs @@ -1,4 +1,4 @@ -use super::matcher::ExcludeMatcher; +use super::matcher::{ExcludeMatcher, one_file_system, source_root}; use super::platform::dev; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; @@ -20,20 +20,11 @@ pub async fn run( } fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Result { - // Canonical paths so the output location can be recognised while walking. - let root = std::fs::canonicalize(&cfg.path) - .with_context(|| format!("cannot read source directory {}", cfg.path))?; - if !root.is_dir() { - bail!("source path {} is not a directory", root.display()); - } + let root = source_root(cfg)?; let backup_real = std::fs::canonicalize(backup_dir) .with_context(|| format!("cannot read backup directory {}", backup_dir.display()))?; let matcher = ExcludeMatcher::from_config(cfg)?; - let one_file_system = cfg - .options - .get("one_file_system") - .and_then(|v| v.as_bool()) - .unwrap_or(false); + let one_file_system = one_file_system(cfg); let root_dev = dev(&std::fs::metadata(&root) .with_context(|| format!("cannot read source directory {}", root.display()))?); @@ -46,7 +37,7 @@ fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Resul logger.log("info", format!("Archiving {}", root.display())); let mut files = 0u64; - // Entries that could not be read: a mirror restore of this archive would delete them. + let mut unreadable = 0usize; let mut walker = WalkDir::new(&root) .follow_links(false) @@ -65,7 +56,6 @@ fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Resul let rel = entry.path().strip_prefix(&root)?.to_path_buf(); let ft = entry.file_type(); - // Never archive our own output (backup dir under the source, or the archive itself). let mut skip = matcher.is_excluded(&rel) || entry.path() == backup_real || entry.path() == out_real; if !skip && one_file_system && ft.is_dir() { @@ -86,8 +76,6 @@ fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Resul } if ft.is_file() { - // Only a failure to open is skippable: it happens before anything is - // written. Every later error leaves the tar stream misaligned and must abort. let mut f = match File::open(entry.path()) { Ok(f) => f, Err(e) => { @@ -100,20 +88,18 @@ fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Resul let size = meta.len(); let mut header = tar::Header::new_gnu(); header.set_metadata(&meta); - // The header size is fixed up front: cap the copy if the file grew and - // zero-pad if it shrank, so the entry always matches its header. + let reader = (&mut f).take(size).chain(std::io::repeat(0)).take(size); tar.append_data(&mut header, &rel, reader) .with_context(|| format!("failed to archive {}", rel.display()))?; files += 1; } else if ft.is_dir() || ft.is_symlink() { - // These fail on lstat/readlink before the header is written. + if let Err(e) = tar.append_path_with_name(entry.path(), &rel) { logger.log("warn", format!("Skipping {}: {e}", rel.display())); unreadable += 1; } } else { - // Sockets, fifos and devices hold no data to restore; restore keeps them too. logger.log("warn", format!("Skipping special file {}", rel.display())); } } diff --git a/src/domain/files/matcher.rs b/src/domain/files/matcher.rs index 3c3f5c0..88f96d7 100644 --- a/src/domain/files/matcher.rs +++ b/src/domain/files/matcher.rs @@ -1,10 +1,8 @@ use crate::services::config::DatabaseConfig; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use std::path::{Path, PathBuf}; -/// Exclude patterns, matched against paths relative to the source root. -/// No leading `/` matches at any depth; a leading `/` anchors at the root. pub struct ExcludeMatcher { set: GlobSet, literal: Vec, @@ -39,8 +37,6 @@ impl ExcludeMatcher { Self::new(&exclude_patterns(cfg)) } - /// Also exclude exactly this root-relative path (and its subtree), compared - /// literally so glob characters in the name mean nothing. pub fn exclude_path(&mut self, relative: PathBuf) { self.literal.push(relative); } @@ -50,7 +46,6 @@ impl ExcludeMatcher { } } -/// The `options.exclude` patterns of a files source, as configured. pub fn exclude_patterns(cfg: &DatabaseConfig) -> Vec { cfg.options .get("exclude") @@ -64,10 +59,40 @@ pub fn exclude_patterns(cfg: &DatabaseConfig) -> Vec { .unwrap_or_default() } -/// `options.one_file_system` of a files source. pub fn one_file_system(cfg: &DatabaseConfig) -> bool { cfg.options .get("one_file_system") .and_then(|v| v.as_bool()) .unwrap_or(false) } + +pub fn source_root(cfg: &DatabaseConfig) -> Result { + let root = std::fs::canonicalize(&cfg.path) + .with_context(|| format!("cannot read source directory {}", cfg.path))?; + if !root.is_dir() { + bail!("source path {} is not a directory", root.display()); + } + Ok(root) +} + +pub(crate) fn bracket_negation(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + let mut in_class = false; + while let Some(c) = chars.next() { + out.push(c); + match c { + '\\' => out.extend(chars.next()), + '[' if !in_class => { + in_class = true; + if chars.as_str().starts_with('!') { + chars.next(); + out.push('^'); + } + } + ']' => in_class = false, + _ => {} + } + } + out +} diff --git a/src/domain/files/mod.rs b/src/domain/files/mod.rs index d112947..c1fd77b 100644 --- a/src/domain/files/mod.rs +++ b/src/domain/files/mod.rs @@ -3,4 +3,6 @@ pub mod database; pub mod matcher; pub mod ping; pub mod platform; +pub mod restic; pub mod restore; +pub mod sync; diff --git a/src/domain/files/ping.rs b/src/domain/files/ping.rs index ee61cd3..2af3acf 100644 --- a/src/domain/files/ping.rs +++ b/src/domain/files/ping.rs @@ -4,7 +4,6 @@ use std::path::Path; use tokio::time::{Duration, timeout}; pub async fn run(cfg: DatabaseConfig) -> Result { - // A hung mount (NFS) blocks is_dir/read_dir: keep it off the runtime and bounded. let check = tokio::task::spawn_blocking(move || { let root = Path::new(&cfg.path); root.is_dir() && std::fs::read_dir(root).is_ok() diff --git a/src/domain/files/platform.rs b/src/domain/files/platform.rs index eddb031..60fca91 100644 --- a/src/domain/files/platform.rs +++ b/src/domain/files/platform.rs @@ -14,7 +14,6 @@ pub fn dev(_meta: &std::fs::Metadata) -> u64 { 0 } -/// Whether the agent can restore file ownership. #[cfg(unix)] pub fn is_root() -> bool { unsafe { libc::geteuid() == 0 } diff --git a/src/domain/files/restic/backup.rs b/src/domain/files/restic/backup.rs new file mode 100644 index 0000000..58b10e5 --- /dev/null +++ b/src/domain/files/restic/backup.rs @@ -0,0 +1,106 @@ +use super::excludes::backup_excludes; +use super::json::{BackupSummary, Snapshot, files_removed}; +use super::repo::{ResticRepo, short}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system, source_root}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; + +pub async fn snapshot( + repo: &ResticRepo, + cfg: &DatabaseConfig, + backup_storage_id: &str, + logger: &JobLogger, +) -> Result { + let root = source_root(cfg)?; + repo.ensure_initialized(logger).await?; + + let host = format!("portabase-{}", cfg.generated_id); + let mut args: Vec = vec![ + "backup".into(), + root.to_string_lossy().into_owned(), + "--json".into(), + "--host".into(), + host.clone(), + "--tag".into(), + "portabase".into(), + "--tag".into(), + format!("bs:{backup_storage_id}"), + ]; + for pattern in backup_excludes(&root, &exclude_patterns(cfg))? { + args.push("--exclude".into()); + args.push(pattern); + } + if one_file_system(cfg) { + args.push("--one-file-system".into()); + } + + logger.log("info", format!("Snapshotting {} into {}", root.display(), repo.repository())); + let run = repo.run(&args, logger).await?; + for error in &run.errors { + logger.log("warn", format!("Unreadable: {error}")); + } + let summary: Option = run + .summary + .clone() + .map(serde_json::from_value) + .transpose() + .context("unexpected restic backup summary")?; + + match (run.code, summary) { + (0, Some(mut s)) => { + s.files_removed = match removed(repo, &host, &s, logger).await { + Ok(removed) => removed, + Err(e) => { + logger.log("warn", format!("Could not count removed files: {e:#}")); + None + } + }; + let removed = s.files_removed.map(|n| format!(", {n} removed")).unwrap_or_default(); + logger.log( + "info", + format!( + "Snapshot {}: {} new, {} changed, {} unmodified{removed} file(s); {} byte(s) scanned, {} byte(s) of new data", + short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.total_bytes_processed, s.data_added_packed + ), + ); + Ok(s) + } + (0, None) => bail!("restic backup returned no summary"), + (3, summary) => { + let mut fate = "snapshot discarded"; + if let Some(s) = summary { + if repo.is_append_only() { + logger.log( + "warn", + format!("Incomplete snapshot {} left for the dashboard to forget", short(&s.snapshot_id)), + ); + fate = "snapshot not restorable (the dashboard forgets it)"; + } else { + let forget = repo.run(["forget", s.snapshot_id.as_str()], logger).await?; + if forget.code != 0 { + logger.log( + "error", + format!("Could not forget incomplete snapshot {}: {}", s.snapshot_id, forget.error("forget")), + ); + } + } + } + bail!( + "{} file(s) could not be read; {fate} so a mirror restore cannot delete them (fix permissions or add an exclude)", + run.errors.len() + ) + } + _ => Err(run.error("backup")), + } +} + +async fn removed(repo: &ResticRepo, host: &str, s: &BackupSummary, logger: &JobLogger) -> Result> { + let listed = repo.run(["snapshots", "--json", "--host", host], logger).await?; + if listed.code != 0 { + return Err(listed.error("snapshots")); + } + let snapshots: Vec = + serde_json::from_str(listed.stdout.trim()).context("unexpected `restic snapshots` output")?; + Ok(files_removed(&snapshots, &s.snapshot_id, s.files_changed, s.files_unmodified)) +} diff --git a/src/services/restic/excludes.rs b/src/domain/files/restic/excludes.rs similarity index 54% rename from src/services/restic/excludes.rs rename to src/domain/files/restic/excludes.rs index bc2dbdd..b894b84 100644 --- a/src/services/restic/excludes.rs +++ b/src/domain/files/restic/excludes.rs @@ -1,7 +1,7 @@ +use crate::domain::files::matcher::bracket_negation; use anyhow::{Result, bail}; use std::path::Path; -/// Escapes restic/filepath.Match metacharacters so a literal path matches only itself. pub fn glob_escape(literal: &str) -> String { let mut out = String::with_capacity(literal.len()); for c in literal.chars() { @@ -13,31 +13,6 @@ pub fn glob_escape(literal: &str) -> String { out } -/// globset (the P0 matcher) reads `[!x]` as bracket negation; restic (Go -/// `filepath.Match`) and rclone only know `[^x]` and take `!` literally. -pub(crate) fn bracket_negation(pattern: &str) -> String { - let mut out = String::with_capacity(pattern.len()); - let mut chars = pattern.chars(); - let mut in_class = false; - while let Some(c) = chars.next() { - out.push(c); - match c { - '\\' => out.extend(chars.next()), - '[' if !in_class => { - in_class = true; - if chars.as_str().starts_with('!') { - chars.next(); - out.push('^'); - } - } - ']' => in_class = false, - _ => {} - } - } - out -} - -/// P0 patterns, trimmed like the P0 matcher, as `(anchored, pattern)`; a leading `/` marks anchored ones. fn normalized(patterns: &[String]) -> Result> { let mut out = Vec::new(); for raw in patterns { @@ -57,9 +32,6 @@ fn normalized(patterns: &[String]) -> Result> { Ok(out) } -/// `restic backup` matches patterns against absolute paths, so every pattern is -/// rooted at the source; unanchored ones would otherwise also match directories -/// above it and exclude the whole source (spike S5). pub fn backup_excludes(root: &Path, patterns: &[String]) -> Result> { let root = glob_escape(root.to_string_lossy().trim_end_matches('/')); Ok(normalized(patterns)? @@ -70,7 +42,6 @@ pub fn backup_excludes(root: &Path, patterns: &[String]) -> Result> .collect()) } -/// `restic restore :` matches patterns relative to that subfolder (spike S1). pub fn restore_excludes(patterns: &[String]) -> Result> { Ok(normalized(patterns)? .into_iter() diff --git a/src/services/restic/json.rs b/src/domain/files/restic/json.rs similarity index 75% rename from src/services/restic/json.rs rename to src/domain/files/restic/json.rs index e933830..dcd53c0 100644 --- a/src/services/restic/json.rs +++ b/src/domain/files/restic/json.rs @@ -1,8 +1,6 @@ use serde::Deserialize; use serde_json::{Value, json}; -/// One line of `restic --json` output. stdout carries `status` / `summary`, -/// stderr carries `error` / `exit_error` and plain-text fatal messages. #[derive(Debug, PartialEq)] pub enum Line { Status { percent_done: f64 }, @@ -41,14 +39,11 @@ pub struct BackupSummary { pub files_new: u64, pub files_changed: u64, pub files_unmodified: u64, - /// Files of the parent snapshot this one no longer has: restic does not count them - /// (see `files_removed`). None when unknown. #[serde(skip)] pub files_removed: Option, } impl BackupSummary { - /// Counters the dashboard shows for this snapshot (`size` carries `data_added_packed`). pub fn report(&self) -> Value { json!({ "filesNew": self.files_new, @@ -59,14 +54,12 @@ impl BackupSummary { } } -/// One entry of `restic snapshots --json`. #[derive(Debug, Deserialize)] pub struct Snapshot { pub id: String, pub paths: Vec, #[serde(default)] pub parent: Option, - /// Written by restic >= 0.17. #[serde(default)] pub summary: Option, } @@ -76,10 +69,6 @@ pub struct SnapshotSummary { pub total_files_processed: u64, } -/// Files of the parent missing from `snapshot_id`, from a `restic snapshots --json` listing: -/// restic counts a file as changed or unmodified only when the parent has it, so the parent's -/// file count minus those is what was removed. Some(0) without a parent; None when the parent -/// is not listed or has no summary. pub fn files_removed(snapshots: &[Snapshot], snapshot_id: &str, changed: u64, unmodified: u64) -> Option { let find = |id: &str| snapshots.iter().find(|s| s.id == id); let Some(parent) = find(snapshot_id)?.parent.as_deref() else { diff --git a/src/domain/files/restic/mod.rs b/src/domain/files/restic/mod.rs new file mode 100644 index 0000000..12fbd55 --- /dev/null +++ b/src/domain/files/restic/mod.rs @@ -0,0 +1,6 @@ +pub mod backup; +pub mod excludes; +pub mod json; +pub mod password; +pub mod repo; +pub mod restore; diff --git a/src/domain/files/restic/password.rs b/src/domain/files/restic/password.rs new file mode 100644 index 0000000..a491fb5 --- /dev/null +++ b/src/domain/files/restic/password.rs @@ -0,0 +1,7 @@ +use crate::utils::edge_key::derive_key; +use anyhow::Result; + +pub fn derive_password(master_key_b64: &str, generated_id: &str) -> Result { + let info = format!("portabase/restic/v1/{generated_id}"); + Ok(hex::encode(derive_key(master_key_b64, info.as_bytes())?)) +} diff --git a/src/services/restic/command.rs b/src/domain/files/restic/repo.rs similarity index 80% rename from src/services/restic/command.rs rename to src/domain/files/restic/repo.rs index a99ab02..ff09e2f 100644 --- a/src/services/restic/command.rs +++ b/src/domain/files/restic/repo.rs @@ -1,11 +1,13 @@ use super::json::{self, Line}; -use super::password::{derive_password, local_storage_password, LOCAL_STORAGE_USER}; +use super::password::derive_password; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; +use crate::services::storage::providers::local::{LOCAL_STORAGE_USER, local_storage_password}; use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; use crate::services::storage::providers::rclone::target::{RcloneTarget, rclone_target}; use crate::settings::CONFIG; use crate::utils::edge_key::EdgeKey; +use crate::utils::file::folder_or_default; use anyhow::{Context, Result, anyhow}; use serde_json::Value; use std::ffi::{OsStr, OsString}; @@ -15,32 +17,20 @@ use tempfile::NamedTempFile; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Command; -/// One source's repository on one storage channel: -/// `rclone:://restic/`, or -/// `rest:/storage/restic//` on the dashboard's local storage. pub struct ResticRepo { repository: String, password: String, - /// `None` when the cache directory cannot be created: restic then runs `--no-cache`. cache_dir: Option, - /// rclone config of `rclone:` repositories; `None` on the dashboard's REST server. config: Option, - /// HTTP basic auth of `rest:` repositories (the dashboard's local storage). rest_auth: Option<(String, String)>, - /// Keeps temp files referenced by the config (sftp key) alive. _target: Option, } -/// What a finished restic command left behind. pub struct ResticRun { pub code: i32, - /// stdout lines that are neither `status` nor `summary` (e.g. `snapshots --json`). pub stdout: String, - /// The last `summary` line, if any. pub summary: Option, - /// `error` lines as "item: message" (unreadable files and the like). pub errors: Vec, - /// Last plain-text / `exit_error` stderr lines, for failure messages. pub stderr_tail: String, } @@ -55,8 +45,6 @@ impl ResticRun { } } -/// A full snapshot id. Ids from the dashboard reach restic as positional arguments: -/// anything else could be parsed as a flag (e.g. `--password-command=`). pub fn is_snapshot_id(id: &str) -> bool { id.len() == 64 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) } @@ -65,7 +53,6 @@ pub fn short(id: &str) -> &str { id.get(..8).unwrap_or(id) } -/// restic fails hard (not warns) on an unusable cache dir, so probe it up front. fn usable_cache_dir(cache_dir: PathBuf) -> Option { match std::fs::create_dir_all(&cache_dir) { Ok(()) => Some(cache_dir), @@ -90,17 +77,11 @@ impl ResticRepo { )); } let target = rclone_target(storage)?; - let folder = storage - .folder_name - .as_deref() - .map(|f| f.trim().trim_matches('/')) - .filter(|f| !f.is_empty()) - .unwrap_or("backups"); let mut repo = Self::new( &target.config_text, &target.remote_name, &target.base_path, - folder, + folder_or_default(storage.folder_name.as_deref()), generated_id, &password, cache_dir, @@ -129,8 +110,6 @@ impl ResticRepo { }) } - /// The dashboard's local storage (`rclone serve restic --append-only`): - /// `rest://`, basic auth only in the child env. pub fn rest(base_url: &str, generated_id: &str, password: &str, server_password: &str, cache_dir: PathBuf) -> Self { Self { repository: format!("rest:{}/{generated_id}/", base_url.trim_end_matches('/')), @@ -142,7 +121,6 @@ impl ResticRepo { } } - /// The dashboard's server refuses deletions: the agent cannot forget there. pub fn is_append_only(&self) -> bool { self.rest_auth.is_some() } @@ -151,8 +129,6 @@ impl ResticRepo { &self.repository } - /// `restic --retry-lock 5m` (+ `--no-cache` without a cache dir); secrets only in the child env. `status` - /// lines are logged every 10 %. pub async fn run(&self, args: I, logger: &JobLogger) -> Result where I: IntoIterator, @@ -177,7 +153,6 @@ impl ResticRepo { let mut child = cmd .env("RESTIC_REPOSITORY", &self.repository) .env("RESTIC_PASSWORD", &self.password) - // No TTY: without this restic prints no status lines at all. .env("RESTIC_PROGRESS_FPS", "0.2") .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -237,8 +212,6 @@ impl ResticRepo { Ok(run) } - /// `restic cat config`; exit 10 (no repository) → `restic init`. Callers hold - /// the source's FileLock, so two inits never race. pub async fn ensure_initialized(&self, logger: &JobLogger) -> Result<()> { let cat = self.run(["cat", "config"], logger).await?; match cat.code { diff --git a/src/services/restic/restore.rs b/src/domain/files/restic/restore.rs similarity index 68% rename from src/services/restic/restore.rs rename to src/domain/files/restic/restore.rs index 12c410f..4be279f 100644 --- a/src/services/restic/restore.rs +++ b/src/domain/files/restic/restore.rs @@ -1,20 +1,15 @@ -use super::command::{ResticRepo, is_snapshot_id, short}; use super::excludes::{glob_escape, restore_excludes}; use super::json::Snapshot; +use super::repo::{ResticRepo, is_snapshot_id, short}; use crate::domain::files::matcher::{exclude_patterns, one_file_system}; use crate::domain::files::platform::dev; use crate::domain::files::restore::ensure_restorable; -use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; -use crate::services::config::{DatabaseConfig, DbType}; -use crate::utils::edge_key::EdgeKey; -use crate::utils::locks::{DbOpLock, FileLock}; +use crate::services::config::DatabaseConfig; use anyhow::{Context, Result, bail}; use std::path::Path; use walkdir::WalkDir; -/// Mirror restore of `snapshot_id` into `cfg.path`: files created since are -/// deleted, paths matching the current exclude patterns are kept (spike S1). pub async fn restore( repo: &ResticRepo, cfg: &DatabaseConfig, @@ -32,7 +27,6 @@ pub async fn restore( } let snapshots: Vec = serde_json::from_str(listed.stdout.trim()) .context("unexpected `restic snapshots` output")?; - // The original path inside the snapshot; the mount point may have moved since. let Some(original) = snapshots.first().and_then(|s| s.paths.first()) else { bail!("snapshot {} not found in {}", short(snapshot_id), repo.repository()); }; @@ -78,8 +72,6 @@ pub async fn restore( Ok(()) } -/// Directories under `root` on another device. With `one_file_system` backup -/// never entered them, so the mirror restore must not delete their content. fn mount_points(root: &Path) -> Vec { let Ok(meta) = std::fs::metadata(root) else { return Vec::new() }; let root_dev = dev(&meta); @@ -99,26 +91,3 @@ fn mount_points(root: &Path) -> Vec { } out } - -/// Snapshot restore job: the source's FileLock is held for the whole restore. -pub async fn run( - edge_key: &EdgeKey, - cfg: &DatabaseConfig, - snapshot_id: &str, - storage: &DatabaseStorage, - logger: &JobLogger, -) -> Result<()> { - if !matches!(cfg.db_type, DbType::Files) { - bail!("snapshot restore needs a files source, got {}", cfg.db_type.as_str()); - } - FileLock::acquire(&cfg.generated_id, DbOpLock::Restore.as_str()) - .await - .context("another backup or restore of this source is running")?; - let result = async { - let repo = ResticRepo::open(storage, &cfg.generated_id, edge_key)?; - restore(&repo, cfg, snapshot_id, logger).await - } - .await; - FileLock::release(&cfg.generated_id).await?; - result -} diff --git a/src/domain/files/restore.rs b/src/domain/files/restore.rs index fb07abc..0a24c48 100644 --- a/src/domain/files/restore.rs +++ b/src/domain/files/restore.rs @@ -1,4 +1,4 @@ -use super::matcher::ExcludeMatcher; +use super::matcher::{ExcludeMatcher, one_file_system}; use super::platform::{dev, is_root}; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; @@ -14,25 +14,16 @@ pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc) } fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<()> { - // Everything below works on the canonical path, like backup.rs. let root = ensure_restorable(Path::new(&cfg.path))?; let mut matcher = ExcludeMatcher::from_config(cfg)?; - let one_file_system = cfg - .options - .get("one_file_system") - .and_then(|v| v.as_bool()) - .unwrap_or(false); + let one_file_system = one_file_system(cfg); - // Opened once: the wipe may delete the archive's path, never an open handle. let file = File::open(archive) .with_context(|| format!("cannot open archive {}", archive.display()))?; - // The wipe is irreversible: prove the whole archive reads back before touching data. logger.log("info", format!("Validating archive {}", archive.display())); validate_archive(&file).context("archive is corrupt or truncated, nothing was changed")?; - // Keep the archive's own directory (or the archive itself when it sits directly - // in the target), like backup.rs prunes its output. let archive_real = fs::canonicalize(archive)?; let keep = match archive_real.parent() { Some(dir) if dir != root => dir, @@ -46,7 +37,7 @@ fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<( "info", format!("Removing non-excluded content of {}", root.display()), ); - // Best effort: whatever could not be removed is kept, and extraction always runs. + let failures = wipe_except_excluded(&root, &matcher, one_file_system, logger)?; logger.log( @@ -65,9 +56,6 @@ fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<( Ok(()) } -/// Extract the whole archive into `root`. Filesystems that refuse chown/chmod/utime -/// (NFS root_squash, CIFS, NFSv4 idmap mismatch) fail the first pass, after the wipe, -/// so retry once without metadata: the content matters more than owner and mode. pub fn unpack( file: &File, root: &Path, @@ -79,9 +67,6 @@ pub fn unpack( "warn", format!("extraction with ownership/permissions failed: {e:#}; retrying without metadata preservation"), ); - // ponytail: tar-rs still chmods to `mode & 0o777` with preservation off, so a - // filesystem that rejects chmod on files the agent just created fails this pass - // too; per-entry extraction with deferred directories if that shows up. unpack_once(file, root, false, false)?; } Ok(()) @@ -103,7 +88,6 @@ fn unpack_once( Ok(()) } -/// Read every entry to the end, then drain the gzip stream so its CRC is checked. fn validate_archive(file: &File) -> Result<()> { let mut gz = GzDecoder::new(file); let mut ar = tar::Archive::new(&mut gz); @@ -111,14 +95,10 @@ fn validate_archive(file: &File) -> Result<()> { io::copy(&mut entry?, &mut io::sink())?; } drop(ar); - // tar stops at its end marker, before the gzip trailer. gz.read_to_end(&mut Vec::new())?; Ok(()) } -/// Refuse targets a mirror restore must never touch, and return the canonical path to -/// restore into. The canonical path is checked too, because `/data/..` style escapes -/// resolve to `/`. pub fn ensure_restorable(root: &Path) -> Result { if root.as_os_str().is_empty() || !root.is_absolute() || root == Path::new("/") { bail!( @@ -140,18 +120,6 @@ pub fn ensure_restorable(root: &Path) -> Result { Ok(canonical) } -/// Delete everything under `root` that does not match an exclude pattern. -/// Excluded entries (and their subtrees) are kept; `root` itself is kept. -/// -/// Mount points (a directory on a different device than its parent): with -/// `one_file_system` they are kept untouched like an excluded path, matching what -/// backup never archived. Without it their contents are wiped but the mount point -/// itself is never removed (`remove_dir` on it fails with EBUSY) and counts as kept. -/// Special files (sockets, fifos, devices) are kept: backup never archives them. -/// -/// Best effort: an entry that cannot be read or removed is logged, kept (so its -/// parents are kept too) and counted; the returned count is how many failed. -/// Only an unreadable `root` is an error, and then nothing was deleted. pub fn wipe_except_excluded( root: &Path, matcher: &ExcludeMatcher, @@ -183,8 +151,6 @@ impl Wipe<'_> { self.failures += 1; } - /// Returns true when something was kept inside `dir`. `rel` is `dir` relative to - /// the root and `dir_dev` its device. fn dir(&mut self, dir: &Path, rel: &Path, dir_dev: u64) -> bool { let entries = match fs::read_dir(dir) { Ok(entries) => entries, @@ -209,7 +175,7 @@ impl Wipe<'_> { kept = true; continue; } - // symlink_metadata: a symlinked directory is removed as a link, never followed. + let meta = match fs::symlink_metadata(&path) { Ok(meta) => meta, Err(e) => { @@ -233,7 +199,6 @@ impl Wipe<'_> { } else if ft.is_file() || ft.is_symlink() { fs::remove_file(&path) } else { - // Socket, fifo or device: backup never archives them, so keep them. kept = true; continue; }; diff --git a/src/domain/files/sync/backup.rs b/src/domain/files/sync/backup.rs new file mode 100644 index 0000000..fbca007 --- /dev/null +++ b/src/domain/files/sync/backup.rs @@ -0,0 +1,45 @@ +use super::rclone::{replica_target, sync_dir}; +use super::stats::SyncStats; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::services::storage::providers::local::{LOCAL_STORAGE_USER, local_storage_password}; +use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password}; +use crate::services::storage::providers::rclone::target::rclone_target; +use crate::utils::edge_key::EdgeKey; +use crate::utils::file::folder_or_default; +use anyhow::Result; + +pub fn replica_path(storage: &DatabaseStorage, generated_id: &str) -> String { + format!("{}/sync/{generated_id}/current", folder_or_default(storage.folder_name.as_deref())) +} + +const LOCAL_REMOTE: &str = "pblocal"; + +pub fn local_sync_config(edge_key: &EdgeKey) -> Result { + build_rclone_config( + LOCAL_REMOTE, + &[ + ("type", "webdav".into()), + ("url", format!("{}/storage/sync", edge_key.server_url.trim_end_matches('/'))), + ("vendor", "rclone".into()), + ("user", LOCAL_STORAGE_USER.into()), + ("pass", obscure_password(&local_storage_password(&edge_key.master_key_b64)?)?), + ], + ) +} + +pub async fn replicate(edge_key: &EdgeKey, cfg: &DatabaseConfig, storage: &DatabaseStorage, logger: &JobLogger) -> Result { + if storage.provider == "local" { + let dest = format!("{LOCAL_REMOTE}:{}/current", cfg.generated_id); + return sync_dir(&local_sync_config(edge_key)?, &dest, cfg, logger).await; + } + let target = rclone_target(storage)?; + let dest = replica_target( + &target.remote_name, + &target.base_path, + folder_or_default(storage.folder_name.as_deref()), + &cfg.generated_id, + ); + sync_dir(&target.config_text, &dest, cfg, logger).await +} diff --git a/src/services/sync/excludes.rs b/src/domain/files/sync/excludes.rs similarity index 68% rename from src/services/sync/excludes.rs rename to src/domain/files/sync/excludes.rs index fc24e90..28728e8 100644 --- a/src/services/sync/excludes.rs +++ b/src/domain/files/sync/excludes.rs @@ -1,7 +1,5 @@ -use crate::services::restic::excludes::bracket_negation; +use crate::domain::files::matcher::bracket_negation; -/// P0 exclude patterns → `rclone sync --exclude` values. rclone filters are -/// relative to the sync root (spike 2026-10-03: 19/19 patterns give the P0 file set). pub fn sync_excludes(patterns: &[String]) -> Vec { let mut out = Vec::new(); for raw in patterns { @@ -12,8 +10,6 @@ pub fn sync_excludes(patterns: &[String]) -> Vec { }; let mut rest = rest.trim_end_matches('/'); if !anchored { - // Unanchored rclone patterns already match at any depth; a leading - // `**/` would stop them from matching at the root. while let Some(stripped) = rest.strip_prefix("**/") { rest = stripped; } @@ -22,7 +18,6 @@ pub fn sync_excludes(patterns: &[String]) -> Vec { continue; } let rest = bracket_negation(rest); - // Collapse repeated /**/ into a single /**/ let rest = collapse_double_stars(&rest); let full = if anchored { format!("/{rest}") } else { rest }; for variant in zero_dir_variants(&full) { @@ -33,8 +28,6 @@ pub fn sync_excludes(patterns: &[String]) -> Vec { out } -/// Collapse repeated `/**/` sequences into a single `/**/`. -/// E.g., `a/**/**/b` becomes `a/**/b`. fn collapse_double_stars(pattern: &str) -> String { let mut result = pattern.to_string(); while result.contains("/**/**/") { @@ -43,12 +36,9 @@ fn collapse_double_stars(pattern: &str) -> String { result } -/// rclone's `**` between slashes needs at least one directory; globset's also -/// matches none. Emit every combination of each `/**/` kept or collapsed to `/`. fn zero_dir_variants(pattern: &str) -> Vec { let parts: Vec<&str> = pattern.split("/**/").collect(); let gaps = parts.len() - 1; - // ponytail: 2^gaps variants; past 4 gaps only "all kept" and "all collapsed". if gaps > 4 { return vec![pattern.to_string(), pattern.replace("/**/", "/")]; } diff --git a/src/domain/files/sync/mod.rs b/src/domain/files/sync/mod.rs new file mode 100644 index 0000000..5a26cb6 --- /dev/null +++ b/src/domain/files/sync/mod.rs @@ -0,0 +1,4 @@ +pub mod backup; +pub mod excludes; +pub mod rclone; +pub mod stats; diff --git a/src/services/sync/rclone.rs b/src/domain/files/sync/rclone.rs similarity index 80% rename from src/services/sync/rclone.rs rename to src/domain/files/sync/rclone.rs index 6d361a8..f940c7a 100644 --- a/src/services/sync/rclone.rs +++ b/src/domain/files/sync/rclone.rs @@ -1,6 +1,6 @@ use super::excludes::sync_excludes; use super::stats::{SyncStats, parse_log}; -use crate::domain::files::matcher::{exclude_patterns, one_file_system}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system, source_root}; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; @@ -9,22 +9,12 @@ use std::path::Path; use std::process::Stdio; use tokio::process::Command; -/// `://sync//current` pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generated_id: &str) -> String { remote_target(remote_name, base_path, &format!("{folder}/sync/{generated_id}/current")) } -/// Mirrors `cfg.path` onto `dest` (`:`); files matching an exclude are removed -/// from the replica too (`--delete-excluded`). rclone skips its deletions when an error -/// happens before the delete phase ("not deleting files as there were IO errors"); -/// an error during the delete phase can leave the replica partially updated. -/// An empty source is refused so an unmounted volume cannot wipe the replica. pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logger: &JobLogger) -> Result { - let root = std::fs::canonicalize(&cfg.path) - .with_context(|| format!("cannot read source directory {}", cfg.path))?; - if !root.is_dir() { - bail!("source path {} is not a directory", root.display()); - } + let root = source_root(cfg)?; if std::fs::read_dir(&root) .with_context(|| format!("cannot read source directory {}", root.display()))? .next() @@ -45,7 +35,6 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge .arg(dest) .args([ "--links", "--delete-excluded", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1", - // An unreachable endpoint fails in ~1–2 min instead of blocking the next runs. "--contimeout", "30s", "--low-level-retries", "3", ]); if one_file_system(cfg) { @@ -80,7 +69,6 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge bail!("rclone sync failed ({}): {reason}; the replica may be partially updated", out.status); } let mut stats = log.stats.unwrap_or_default(); - // Measured on the replica itself, so it is what is stored, not inferred from the source. stats.replica_bytes = match remote_size(config.path(), dest).await { Ok(bytes) => Some(bytes), Err(e) => { @@ -96,7 +84,6 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge Ok(stats) } -/// Bytes stored under `target` (`:`), per `rclone size`. async fn remote_size(config: &Path, target: &str) -> Result { let out = Command::new("rclone") .args(["--contimeout", "30s", "--low-level-retries", "3", "--config"]) diff --git a/src/services/sync/stats.rs b/src/domain/files/sync/stats.rs similarity index 80% rename from src/services/sync/stats.rs rename to src/domain/files/sync/stats.rs index 465c8c8..0772176 100644 --- a/src/services/sync/stats.rs +++ b/src/domain/files/sync/stats.rs @@ -1,7 +1,6 @@ use serde::Deserialize; use serde_json::{Value, json}; -/// Final `stats` object of `rclone --use-json-log --stats 1h --stats-log-level NOTICE`. #[derive(Debug, Default, Deserialize, PartialEq)] pub struct SyncStats { #[serde(default)] @@ -14,13 +13,11 @@ pub struct SyncStats { pub errors: u64, #[serde(default, rename = "lastError")] pub last_error: Option, - /// Replica size after a successful run, from `rclone size` on the destination (not an rclone stat). #[serde(skip)] pub replica_bytes: Option, } impl SyncStats { - /// Counters the dashboard shows for this run (`size` carries `bytes`). pub fn report(&self) -> Value { json!({ "filesTransferred": self.transfers, @@ -30,8 +27,6 @@ impl SyncStats { } } -/// What `rclone sync` printed on stderr: its last stats object and its error -/// messages (JSON `level: error|critical|alert|emergency` lines and any plain-text line). #[derive(Debug, Default)] pub struct SyncLog { pub stats: Option, @@ -61,7 +56,6 @@ pub fn parse_log(stderr: &str) -> SyncLog { } } } - // Keep only the last 20 messages if log.errors.len() > 20 { log.errors = log.errors.into_iter().rev().take(20).collect::>(); log.errors.reverse(); diff --git a/src/services/backup/executor.rs b/src/services/backup/executor.rs index c31884b..269bd3c 100644 --- a/src/services/backup/executor.rs +++ b/src/services/backup/executor.rs @@ -1,10 +1,9 @@ +use super::files_engine::{self, FilesEngine}; use super::logger::JobLogger; use super::models::BackupResult; use super::service::BackupService; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::config::{DatabaseConfig, DbType}; -use crate::services::restic; -use crate::services::sync; use crate::utils::common::BackupMethod; use crate::utils::locks::FileLock; @@ -34,13 +33,9 @@ impl BackupService { let backup = self.create_backup_record(&generated_id, &method).await?; let backup_id = backup.backup.id; - if engine == "restic" || engine == "sync" { + if let Some(files_engine) = FilesEngine::parse(&engine) { if matches!(db_cfg.db_type, DbType::Files) { - let uploads = if engine == "restic" { - restic::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await - } else { - sync::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await - }; + let uploads = files_engine::run(&self.ctx, &db_cfg, files_engine, &storages, &backup_id, &logger).await; logger.log("info", "Database backup job finished".to_string()); let result = BackupResult { generated_id: generated_id.clone(), @@ -51,8 +46,6 @@ impl BackupService { }; let duration_ms = start.elapsed().as_millis() as f64; let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries(); - // fileSize = average logical size across storages (send_result): restic reports - // the size each snapshot processed; sync reports none, so it stays null. self.send_result(result, uploads, &backup_id, logs, duration_ms).await?; return Ok(()); } diff --git a/src/services/backup/files_engine.rs b/src/services/backup/files_engine.rs new file mode 100644 index 0000000..e44389b --- /dev/null +++ b/src/services/backup/files_engine.rs @@ -0,0 +1,166 @@ +use super::logger::JobLogger; +use super::models::UploadResult; +use crate::core::context::Context as CoreContext; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::sync::backup::{replica_path, replicate}; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::Result; +use serde_json::Value; + +#[derive(Clone, Copy)] +pub enum FilesEngine { + Restic, + Sync, +} + +impl FilesEngine { + pub fn parse(engine: &str) -> Option { + match engine { + "restic" => Some(Self::Restic), + "sync" => Some(Self::Sync), + _ => None, + } + } + + fn as_str(self) -> &'static str { + match self { + Self::Restic => "restic", + Self::Sync => "sync", + } + } + + fn label(self) -> &'static str { + match self { + Self::Restic => "Snapshot", + Self::Sync => "Sync", + } + } +} + +struct Transfer { + remote_path: String, + size: u64, + report: Value, + total_size: Option, +} + +pub async fn run( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storages: &[DatabaseStorage], + backup_id: &str, + logger: &JobLogger, +) -> Vec { + if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { + logger.log("error", format!("{} aborted: {e}", engine.label())); + return Vec::new(); + } + let mut results = Vec::with_capacity(storages.len()); + for storage in storages { + results.push(one_storage(ctx, cfg, engine, storage, backup_id, logger).await); + } + if let Err(e) = FileLock::release(&cfg.generated_id).await { + logger.log("warn", format!("Failed to release the backup lock: {e}")); + } + results +} + +async fn transfer( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storage: &DatabaseStorage, + backup_storage_id: &str, + logger: &JobLogger, +) -> Result { + match engine { + FilesEngine::Restic => { + let repo = ResticRepo::open(storage, &cfg.generated_id, &ctx.edge_key)?; + let summary = snapshot(&repo, cfg, backup_storage_id, logger).await?; + Ok(Transfer { + size: summary.data_added_packed, + report: summary.report(), + total_size: Some(summary.total_bytes_processed), + remote_path: summary.snapshot_id, + }) + } + FilesEngine::Sync => { + let stats = replicate(&ctx.edge_key, cfg, storage, logger).await?; + Ok(Transfer { + remote_path: replica_path(storage, &cfg.generated_id), + size: stats.bytes, + report: stats.report(), + total_size: stats.replica_bytes, + }) + } + } +} + +pub(crate) async fn one_storage( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storage: &DatabaseStorage, + backup_id: &str, + logger: &JobLogger, +) -> UploadResult { + let agent_id = ctx.edge_key.agent_id.clone(); + let failed = |error: String| UploadResult { + storage_id: storage.id.clone(), + success: false, + error: Some(error), + remote_file_path: None, + total_size: None, + }; + + let backup_storage_id = match ctx + .api + .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some(engine.as_str())) + .await + { + Ok(Some(response)) => response.backup_storage.id, + Ok(None) => { + logger.log("error", "Upload init returned empty response"); + return failed("backup_upload_init returned empty response".into()); + } + Err(e) => { + logger.log("error", format!("Upload init failed: {e}")); + return failed("backup_upload_init failed".into()); + } + }; + + match transfer(ctx, cfg, engine, storage, &backup_storage_id, logger).await { + Ok(t) => match ctx + .api + .backup_upload_success(agent_id, cfg.generated_id.clone(), backup_storage_id, t.remote_path.clone(), t.size, t.report, backup_id) + .await + { + Ok(_) => UploadResult { + storage_id: storage.id.clone(), + success: true, + error: None, + remote_file_path: Some(t.remote_path), + total_size: t.total_size, + }, + Err(e) => { + logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); + failed(e.to_string()) + } + }, + Err(e) => { + logger.log("error", format!("{} to storage {} failed: {e:#}", engine.label(), storage.id)); + if let Err(err) = ctx + .api + .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) + .await + { + logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); + } + failed(format!("{e:#}")) + } + } +} diff --git a/src/services/backup/mod.rs b/src/services/backup/mod.rs index b761556..aa65590 100644 --- a/src/services/backup/mod.rs +++ b/src/services/backup/mod.rs @@ -2,6 +2,7 @@ pub mod compressor; pub mod logger; pub mod dispatcher; pub mod executor; +pub mod files_engine; pub mod helpers; pub mod models; pub mod result; diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs index e5d6b91..babc834 100644 --- a/src/services/dashboard_config.rs +++ b/src/services/dashboard_config.rs @@ -20,7 +20,6 @@ pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> Database DatabasesConfig { databases } } -/// `generated_id`s of local (databases.json) sources that no dashboard config replaces. pub fn local_only_ids(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> HashSet { local .iter() diff --git a/src/services/mod.rs b/src/services/mod.rs index f6516ed..e6c391a 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -3,8 +3,6 @@ pub mod backup; pub mod config; pub mod cron; pub mod dashboard_config; -pub mod restic; pub mod restore; pub mod status; pub mod storage; -pub mod sync; diff --git a/src/services/restic/backup.rs b/src/services/restic/backup.rs deleted file mode 100644 index 401fb83..0000000 --- a/src/services/restic/backup.rs +++ /dev/null @@ -1,224 +0,0 @@ -use super::command::{ResticRepo, short}; -use super::excludes::backup_excludes; -use super::json::{BackupSummary, Snapshot, files_removed}; -use crate::core::context::Context as CoreContext; -use crate::domain::files::matcher::{exclude_patterns, one_file_system}; -use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::backup::logger::JobLogger; -use crate::services::backup::models::UploadResult; -use crate::services::config::DatabaseConfig; -use crate::utils::locks::{DbOpLock, FileLock}; -use anyhow::{Context, Result, bail}; - -/// One snapshot of `cfg.path`, tagged with its `backup_storage` id. -/// Exit 3 (unreadable files) fails the run and forgets the snapshot: a mirror -/// restore of it would delete the files it is missing (P0 rule R21). On an -/// append-only repository the agent cannot forget it, so the dashboard does. -pub async fn snapshot( - repo: &ResticRepo, - cfg: &DatabaseConfig, - backup_storage_id: &str, - logger: &JobLogger, -) -> Result { - let root = std::fs::canonicalize(&cfg.path) - .with_context(|| format!("cannot read source directory {}", cfg.path))?; - if !root.is_dir() { - bail!("source path {} is not a directory", root.display()); - } - repo.ensure_initialized(logger).await?; - - // Stable host: container hostnames change on recreate and would break - // parent-snapshot detection (full rescan every time). - let host = format!("portabase-{}", cfg.generated_id); - let mut args: Vec = vec![ - "backup".into(), - root.to_string_lossy().into_owned(), - "--json".into(), - "--host".into(), - host.clone(), - "--tag".into(), - "portabase".into(), - "--tag".into(), - format!("bs:{backup_storage_id}"), - ]; - for pattern in backup_excludes(&root, &exclude_patterns(cfg))? { - args.push("--exclude".into()); - args.push(pattern); - } - if one_file_system(cfg) { - args.push("--one-file-system".into()); - } - - logger.log("info", format!("Snapshotting {} into {}", root.display(), repo.repository())); - let run = repo.run(&args, logger).await?; - for error in &run.errors { - logger.log("warn", format!("Unreadable: {error}")); - } - let summary: Option = run - .summary - .clone() - .map(serde_json::from_value) - .transpose() - .context("unexpected restic backup summary")?; - - match (run.code, summary) { - (0, Some(mut s)) => { - s.files_removed = match removed(repo, &host, &s, logger).await { - Ok(removed) => removed, - Err(e) => { - logger.log("warn", format!("Could not count removed files: {e:#}")); - None - } - }; - let removed = s.files_removed.map(|n| format!(", {n} removed")).unwrap_or_default(); - logger.log( - "info", - // data_added_packed: new file and directory blobs, compressed and encrypted. The - // repository grows a little more (pack headers, index and snapshot files). - format!( - "Snapshot {}: {} new, {} changed, {} unmodified{removed} file(s); {} byte(s) scanned, {} byte(s) of new data", - short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.total_bytes_processed, s.data_added_packed - ), - ); - Ok(s) - } - (0, None) => bail!("restic backup returned no summary"), - (3, summary) => { - let mut fate = "snapshot discarded"; - if let Some(s) = summary { - if repo.is_append_only() { - logger.log( - "warn", - format!("Incomplete snapshot {} left for the dashboard to forget", short(&s.snapshot_id)), - ); - fate = "snapshot not restorable (the dashboard forgets it)"; - } else { - let forget = repo.run(["forget", s.snapshot_id.as_str()], logger).await?; - if forget.code != 0 { - logger.log( - "error", - format!("Could not forget incomplete snapshot {}: {}", s.snapshot_id, forget.error("forget")), - ); - } - } - } - bail!( - "{} file(s) could not be read; {fate} so a mirror restore cannot delete them (fix permissions or add an exclude)", - run.errors.len() - ) - } - _ => Err(run.error("backup")), - } -} - -/// Removed files of snapshot `s`, against the parent restic actually used (one listing of -/// the source's snapshots, no index load). -async fn removed(repo: &ResticRepo, host: &str, s: &BackupSummary, logger: &JobLogger) -> Result> { - let listed = repo.run(["snapshots", "--json", "--host", host], logger).await?; - if listed.code != 0 { - return Err(listed.error("snapshots")); - } - let snapshots: Vec = - serde_json::from_str(listed.stdout.trim()).context("unexpected `restic snapshots` output")?; - Ok(files_removed(&snapshots, &s.snapshot_id, s.files_changed, s.files_unmodified)) -} - -/// Snapshots every storage in turn (one scan each, v1). Never errors: each -/// storage reports its own `backup_storage` row, like the archive uploader. -pub async fn run( - ctx: &CoreContext, - cfg: &DatabaseConfig, - storages: &[DatabaseStorage], - backup_id: &str, - logger: &JobLogger, -) -> Vec { - if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { - logger.log("error", format!("Snapshot aborted: {e}")); - return Vec::new(); - } - let mut results = Vec::with_capacity(storages.len()); - for storage in storages { - results.push(one_storage(ctx, cfg, storage, backup_id, logger).await); - } - if let Err(e) = FileLock::release(&cfg.generated_id).await { - logger.log("warn", format!("Failed to release the backup lock: {e}")); - } - results -} - -pub(crate) async fn one_storage( - ctx: &CoreContext, - cfg: &DatabaseConfig, - storage: &DatabaseStorage, - backup_id: &str, - logger: &JobLogger, -) -> UploadResult { - let agent_id = ctx.edge_key.agent_id.clone(); - let failed = |error: String| UploadResult { - storage_id: storage.id.clone(), - success: false, - error: Some(error), - remote_file_path: None, - total_size: None, - }; - - let backup_storage_id = match ctx - .api - .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("restic")) - .await - { - Ok(Some(response)) => response.backup_storage.id, - Ok(None) => { - logger.log("error", "Upload init returned empty response"); - return failed("backup_upload_init returned empty response".into()); - } - Err(e) => { - logger.log("error", format!("Upload init failed: {e}")); - return failed("backup_upload_init failed".into()); - } - }; - - let outcome = match ResticRepo::open(storage, &cfg.generated_id, &ctx.edge_key) { - Ok(repo) => snapshot(&repo, cfg, &backup_storage_id, logger).await, - Err(e) => Err(e), - }; - - match outcome { - Ok(summary) => match ctx - .api - .backup_upload_success( - agent_id, - cfg.generated_id.clone(), - backup_storage_id, - summary.snapshot_id.clone(), - summary.data_added_packed, - summary.report(), - backup_id, - ) - .await - { - Ok(_) => UploadResult { - storage_id: storage.id.clone(), - success: true, - error: None, - remote_file_path: Some(summary.snapshot_id), - total_size: Some(summary.total_bytes_processed), - }, - Err(e) => { - logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); - failed(e.to_string()) - } - }, - Err(e) => { - logger.log("error", format!("Snapshot to storage {} failed: {e:#}", storage.id)); - if let Err(err) = ctx - .api - .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) - .await - { - logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); - } - failed(format!("{e:#}")) - } - } -} diff --git a/src/services/restic/mod.rs b/src/services/restic/mod.rs deleted file mode 100644 index 8d112b5..0000000 --- a/src/services/restic/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Snapshots mode: restic repositories reached through the P1 rclone mapper. -pub mod backup; -pub mod command; -pub mod excludes; -pub mod json; -pub mod password; -pub mod restore; diff --git a/src/services/restic/password.rs b/src/services/restic/password.rs deleted file mode 100644 index ca08919..0000000 --- a/src/services/restic/password.rs +++ /dev/null @@ -1,42 +0,0 @@ -use anyhow::{Context, Result}; -use base64::{Engine as _, engine::general_purpose}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -type HmacSha256 = Hmac; - -/// Repository password: hex(HKDF-SHA256(master key, salt = "", -/// info = "portabase/restic/v1/", 32 bytes)). The dashboard derives -/// the same value (`src/lib/restic/repo.ts`), so it is never stored nor sent. -pub fn derive_password(master_key_b64: &str, generated_id: &str) -> Result { - let master_key = general_purpose::STANDARD - .decode(master_key_b64.trim()) - .context("master key is not valid base64")?; - let info = format!("portabase/restic/v1/{generated_id}"); - Ok(hex::encode(hkdf_sha256_32(&master_key, info.as_bytes()))) -} - -/// User of the dashboard's local-storage servers (`/storage/restic`, `/storage/sync`). -pub const LOCAL_STORAGE_USER: &str = "portabase"; - -/// Their password: hex(HKDF-SHA256(master key, salt = "", info = "portabase/local-storage/v1", -/// 32 bytes)). The dashboard derives the same value (`src/lib/local-storage/credential.ts`). -pub fn local_storage_password(master_key_b64: &str) -> Result { - let master_key = general_purpose::STANDARD - .decode(master_key_b64.trim()) - .context("master key is not valid base64")?; - Ok(hex::encode(hkdf_sha256_32(&master_key, b"portabase/local-storage/v1"))) -} - -/// RFC 5869 with an empty salt and a single expand block (L = HashLen = 32). -pub fn hkdf_sha256_32(ikm: &[u8], info: &[u8]) -> [u8; 32] { - // An empty HMAC key is zero-padded: the RFC's default salt of HashLen zeros. - let mut extract = HmacSha256::new_from_slice(&[]).expect("HMAC takes any key length"); - extract.update(ikm); - let prk = extract.finalize().into_bytes(); - - let mut expand = HmacSha256::new_from_slice(&prk).expect("HMAC takes any key length"); - expand.update(info); - expand.update(&[1]); - expand.finalize().into_bytes().into() -} diff --git a/src/services/restore/executor.rs b/src/services/restore/executor.rs index a38597b..d7575cc 100644 --- a/src/services/restore/executor.rs +++ b/src/services/restore/executor.rs @@ -2,9 +2,11 @@ use super::models::RestoreResult; use super::service::RestoreService; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; -use crate::services::config::DatabaseConfig; -use crate::services::restic; -use anyhow::Result; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::{Context, Result, bail}; use std::sync::Arc; use std::time::Instant; use tempfile::TempDir; @@ -58,9 +60,7 @@ impl RestoreService { logger.log("info", "Snapshot restore job started".to_string()); let outcome = match (snapshot_id, storage) { - (Some(id), Some(storage)) => { - restic::restore::run(&self.ctx.edge_key, &cfg, &id, &storage, &logger).await - } + (Some(id), Some(storage)) => self.restic_restore(&cfg, &id, &storage, &logger).await, _ => Err(anyhow::anyhow!( "incomplete snapshot restore payload (snapshotId or storage missing)" )), @@ -82,4 +82,26 @@ impl RestoreService { self.send_result(result, logs, duration_ms).await?; Ok(()) } + + async fn restic_restore( + &self, + cfg: &DatabaseConfig, + snapshot_id: &str, + storage: &DatabaseStorage, + logger: &JobLogger, + ) -> Result<()> { + if !matches!(cfg.db_type, DbType::Files) { + bail!("snapshot restore needs a files source, got {}", cfg.db_type.as_str()); + } + FileLock::acquire(&cfg.generated_id, DbOpLock::Restore.as_str()) + .await + .context("another backup or restore of this source is running")?; + let result = async { + let repo = ResticRepo::open(storage, &cfg.generated_id, &self.ctx.edge_key)?; + restore(&repo, cfg, snapshot_id, logger).await + } + .await; + FileLock::release(&cfg.generated_id).await?; + result + } } diff --git a/src/services/status.rs b/src/services/status.rs index 5e9ab9f..ed85daa 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -36,7 +36,6 @@ pub fn resolve_dashboard_config( Ok(()) } -/// A storage channel sent encrypted as a one-element JSON array. fn decrypt_storage(ciphertext: &str, master_key_b64: &str, what: &str) -> Result, String> { let plaintext = decrypt_json_gcm(ciphertext, master_key_b64) .map_err(|e| format!("Failed to decrypt {what} storage: {e}"))?; @@ -45,7 +44,6 @@ fn decrypt_storage(ciphertext: &str, master_key_b64: &str, what: &str) -> Result Ok(storages.into_iter().next()) } -/// Decrypts the storage channel of a snapshot restore (`restore.storageCiphertext`). pub fn resolve_restore_storage(status: &mut DatabaseStatus, master_key_b64: &str) -> Result<(), String> { if let Some(ciphertext) = status.data.restore.storage_ciphertext.clone() { status.data.restore.storage = decrypt_storage(&ciphertext, master_key_b64, "restore")?; @@ -53,8 +51,6 @@ pub fn resolve_restore_storage(status: &mut DatabaseStatus, master_key_b64: &str Ok(()) } -/// `method` reported in the ping for a files source declared in databases.json; -/// `None` for dashboard-managed sources (the dashboard owns their method) and other dbms. pub fn payload_method(db: &DatabaseConfig, local_ids: &HashSet) -> Option<&'static str> { (matches!(db.db_type, DbType::Files) && local_ids.contains(&db.generated_id)).then(|| files_method(db)) } diff --git a/src/services/storage/providers/local.rs b/src/services/storage/providers/local.rs index b93920a..2bd923a 100644 --- a/src/services/storage/providers/local.rs +++ b/src/services/storage/providers/local.rs @@ -3,15 +3,23 @@ use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::utils::common::BackupMethod; +use crate::utils::edge_key::derive_key; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; use crate::utils::tus::upload_to_tus_stream_with_headers; +use anyhow::Result; use async_trait::async_trait; use reqwest::header::{HeaderMap, HeaderValue}; use std::sync::Arc; use tokio::fs; use tracing::error; +pub const LOCAL_STORAGE_USER: &str = "portabase"; + +pub fn local_storage_password(master_key_b64: &str) -> Result { + Ok(hex::encode(derive_key(master_key_b64, b"portabase/local-storage/v1")?)) +} + pub struct LocalProvider; #[async_trait] diff --git a/src/services/sync/backup.rs b/src/services/sync/backup.rs deleted file mode 100644 index 33fbd3b..0000000 --- a/src/services/sync/backup.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Sync as a backup method: one `rclone sync` per storage channel, each reported -//! as its own `backup_storage` row (`engine = sync`), like restic snapshots. -use super::rclone::{replica_target, sync_dir}; -use super::stats::SyncStats; -use crate::core::context::Context as CoreContext; -use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::backup::logger::JobLogger; -use crate::services::backup::models::UploadResult; -use crate::services::config::DatabaseConfig; -use crate::services::restic::password::{LOCAL_STORAGE_USER, local_storage_password}; -use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password}; -use crate::services::storage::providers::rclone::target::rclone_target; -use crate::utils::edge_key::EdgeKey; -use crate::utils::locks::{DbOpLock, FileLock}; -use anyhow::Result; - -/// Syncs every storage in turn, under the source's backup lock. Never errors: -/// each storage reports its own `backup_storage` row. -pub async fn run( - ctx: &CoreContext, - cfg: &DatabaseConfig, - storages: &[DatabaseStorage], - backup_id: &str, - logger: &JobLogger, -) -> Vec { - if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { - logger.log("error", format!("Sync aborted: {e}")); - return Vec::new(); - } - let mut results = Vec::with_capacity(storages.len()); - for storage in storages { - results.push(one_storage(ctx, cfg, storage, backup_id, logger).await); - } - if let Err(e) = FileLock::release(&cfg.generated_id).await { - logger.log("warn", format!("Failed to release the backup lock: {e}")); - } - results -} - -fn folder(storage: &DatabaseStorage) -> &str { - storage - .folder_name - .as_deref() - .map(|f| f.trim().trim_matches('/')) - .filter(|f| !f.is_empty()) - .unwrap_or("backups") -} - -/// `/sync//current`, relative to the channel's base path. -pub fn replica_path(storage: &DatabaseStorage, generated_id: &str) -> String { - format!("{}/sync/{generated_id}/current", folder(storage)) -} - -const LOCAL_REMOTE: &str = "pblocal"; - -/// rclone config of the dashboard's local storage (`rclone serve webdav` at `/storage/sync`). -pub fn local_sync_config(edge_key: &EdgeKey) -> Result { - build_rclone_config( - LOCAL_REMOTE, - &[ - ("type", "webdav".into()), - ("url", format!("{}/storage/sync", edge_key.server_url.trim_end_matches('/'))), - // rclone's own WebDAV extensions keep modification times: unchanged files are skipped. - ("vendor", "rclone".into()), - ("user", LOCAL_STORAGE_USER.into()), - ("pass", obscure_password(&local_storage_password(&edge_key.master_key_b64)?)?), - ], - ) -} - -async fn replicate(ctx: &CoreContext, cfg: &DatabaseConfig, storage: &DatabaseStorage, logger: &JobLogger) -> Result { - if storage.provider == "local" { - let dest = format!("{LOCAL_REMOTE}:{}/current", cfg.generated_id); - return sync_dir(&local_sync_config(&ctx.edge_key)?, &dest, cfg, logger).await; - } - let target = rclone_target(storage)?; - let dest = replica_target(&target.remote_name, &target.base_path, folder(storage), &cfg.generated_id); - sync_dir(&target.config_text, &dest, cfg, logger).await -} - -pub(crate) async fn one_storage( - ctx: &CoreContext, - cfg: &DatabaseConfig, - storage: &DatabaseStorage, - backup_id: &str, - logger: &JobLogger, -) -> UploadResult { - let agent_id = ctx.edge_key.agent_id.clone(); - let failed = |error: String| UploadResult { - storage_id: storage.id.clone(), - success: false, - error: Some(error), - remote_file_path: None, - total_size: None, - }; - - let backup_storage_id = match ctx - .api - .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("sync")) - .await - { - Ok(Some(response)) => response.backup_storage.id, - Ok(None) => { - logger.log("error", "Upload init returned empty response"); - return failed("backup_upload_init returned empty response".into()); - } - Err(e) => { - logger.log("error", format!("Upload init failed: {e}")); - return failed("backup_upload_init failed".into()); - } - }; - - match replicate(ctx, cfg, storage, logger).await { - Ok(stats) => { - let path = replica_path(storage, &cfg.generated_id); - match ctx - .api - .backup_upload_success(agent_id, cfg.generated_id.clone(), backup_storage_id, path.clone(), stats.bytes, stats.report(), backup_id) - .await - { - Ok(_) => UploadResult { - storage_id: storage.id.clone(), - success: true, - error: None, - remote_file_path: Some(path), - // backups.file_size: the replica's size, like an archive's file size. - total_size: stats.replica_bytes, - }, - Err(e) => { - logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); - failed(e.to_string()) - } - } - } - Err(e) => { - logger.log("error", format!("Sync to storage {} failed: {e:#}", storage.id)); - if let Err(err) = ctx - .api - .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) - .await - { - logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); - } - failed(format!("{e:#}")) - } - } -} diff --git a/src/services/sync/mod.rs b/src/services/sync/mod.rs deleted file mode 100644 index b8501ae..0000000 --- a/src/services/sync/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Sync mode: a destination kept identical to a files source (`rclone sync`). -//! Replication, not backup: deletions propagate. -pub mod backup; -pub mod excludes; -pub mod stats; -pub mod rclone; diff --git a/src/tests/domain/files.rs b/src/tests/domain/files/mod.rs similarity index 99% rename from src/tests/domain/files.rs rename to src/tests/domain/files/mod.rs index d4695ea..4e73066 100644 --- a/src/tests/domain/files.rs +++ b/src/tests/domain/files/mod.rs @@ -1,3 +1,6 @@ +mod restic; +mod sync; + use crate::domain::files::matcher::ExcludeMatcher; use std::path::Path; diff --git a/src/tests/services/restic_tests.rs b/src/tests/domain/files/restic.rs similarity index 87% rename from src/tests/services/restic_tests.rs rename to src/tests/domain/files/restic.rs index d9046a3..accd825 100644 --- a/src/tests/services/restic_tests.rs +++ b/src/tests/domain/files/restic.rs @@ -1,13 +1,11 @@ -//! Snapshots mode (restic). Pure helpers first; the integration tests further -//! down need `restic` and `rclone` on PATH. - use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; -use crate::services::restic::command::is_snapshot_id; -use crate::services::restic::excludes::{backup_excludes, glob_escape, restore_excludes}; -use crate::services::restic::json::{self, BackupSummary, Line}; -use crate::services::restic::password::{derive_password, hkdf_sha256_32}; +use crate::domain::files::restic::repo::is_snapshot_id; +use crate::domain::files::restic::excludes::{backup_excludes, glob_escape, restore_excludes}; +use crate::domain::files::restic::json::{self, BackupSummary, Line}; +use crate::domain::files::restic::password::derive_password; use crate::tests::domain::files::files_config; +use crate::utils::edge_key::hkdf_sha256_32; use serde_json::json; use std::path::Path; @@ -34,7 +32,6 @@ fn hkdf_matches_rfc5869_case_3_first_block() { #[test] fn password_matches_the_dashboard_vector() { - // Same vector as src/lib/restic/repo.ts in the dashboard (spike S4). let password = derive_password( "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "test-generated-id", @@ -83,7 +80,6 @@ fn restore_excludes_are_relative_to_the_snapshot_subfolder() { #[test] fn bracket_negation_is_rewritten_for_restic() { - // globset reads `[!a]`; restic (Go filepath.Match) only knows `[^a]`. assert_eq!( backup_excludes(Path::new("/data"), &strings(&["[!a]*.log", "/x/[!0-9]", r"\[!a]", "[a[!]"])).unwrap(), strings(&["/data/**/[^a]*.log", "/data/x/[^0-9]", r"/data/**/\[!a]", "/data/**/[a[!]"]) @@ -106,7 +102,6 @@ fn glob_escape_escapes_every_metacharacter() { #[test] fn json_lines_are_classified() { - // Recorded from restic 0.19.1 (paths shortened). let status = r#"{"message_type":"status","percent_done":0.9999997777778271,"total_files":4,"files_done":2,"total_bytes":9000002,"bytes_done":9000000,"error_count":1}"#; let error = r#"{"message_type":"error","error":{"message":"open /src/u: permission denied"},"during":"archival","item":"/src/u"}"#; let exit = r#"{"message_type":"exit_error","code":3,"message":"Warning: at least one source file could not be read"}"#; @@ -137,7 +132,7 @@ fn backup_summary_deserializes_from_a_recorded_line() { #[test] fn removed_files_come_from_the_parent_summary() { - use crate::services::restic::json::{Snapshot, files_removed}; + use crate::domain::files::restic::json::{Snapshot, files_removed}; let listing: Vec = serde_json::from_value(serde_json::json!([ {"id": "p", "paths": ["/src"], "summary": {"total_files_processed": 3, "files_new": 3}}, {"id": "c", "paths": ["/src"], "parent": "p"}, @@ -168,27 +163,13 @@ fn a_local_channel_opens_the_dashboard_rest_repository() { assert!(repo.is_append_only()); } -#[test] -fn local_storage_password_matches_the_dashboard_vector() { - use crate::services::restic::password::local_storage_password; - // Same vector as src/lib/local-storage/credential.ts in the dashboard. - assert_eq!( - local_storage_password("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=").unwrap(), - "8abfb788701b8c502658154746af6af768ea08f579381ad73b0f1e7dd43caea2" - ); -} - -// ---------- integration: restic + rclone on PATH ---------- - use crate::domain::files::platform::is_root; -use crate::services::restic::backup::snapshot; -use crate::services::restic::command::ResticRepo; -use crate::services::restic::restore::restore; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; use std::fs; use tempfile::TempDir; -/// A repository on an rclone `alias` remote pointing at a temp dir (test-only: -/// production channels always go through `rclone_target`). fn local_repo(dir: &Path, generated_id: &str) -> ResticRepo { fs::create_dir_all(dir).unwrap(); let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); @@ -204,7 +185,6 @@ fn tree(root: &Path) { fs::write(root.join("app.log"), "log").unwrap(); } -/// `relative/path=content` for every regular file, sorted. fn files(root: &Path) -> Vec { let mut out: Vec = walkdir::WalkDir::new(root) .min_depth(1) @@ -242,8 +222,6 @@ async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_par assert_eq!(second.files_removed, Some(0)); let tagged = repo.run(["snapshots", "--json", "--tag", "bs:bs-2"], &logger).await.unwrap(); - // The tag selects only the second snapshot, and its `parent` is the first - // (a plain `contains(first)` check would match that `parent` field). let listed: serde_json::Value = serde_json::from_str(tagged.stdout.trim()).unwrap(); let listed = listed.as_array().unwrap(); assert_eq!(listed.len(), 1, "{}", tagged.stdout); @@ -291,7 +269,7 @@ async fn restore_is_a_mirror_that_keeps_excluded_paths() { #[tokio::test] async fn unreadable_file_fails_the_snapshot_and_forgets_it() { if is_root() { - return; // root reads a 000 file anyway + return; } use std::os::unix::fs::PermissionsExt; let base = TempDir::new().unwrap(); @@ -313,7 +291,6 @@ async fn unreadable_file_fails_the_snapshot_and_forgets_it() { #[tokio::test] async fn unanchored_exclude_never_matches_directories_above_the_source() { - // Regression for spike S5: the source lives under a directory named like the pattern. let base = TempDir::new().unwrap(); let src = base.path().join("tmp/src"); fs::create_dir_all(src.join("tmp")).unwrap(); @@ -367,7 +344,7 @@ async fn an_unusable_cache_dir_runs_restic_without_cache() { let cfg = files_config(src.path(), &[]); let dir = base.path().join("repo"); fs::create_dir_all(&dir).unwrap(); - let blocker = tempfile::NamedTempFile::new().unwrap(); // a regular file: nothing can be created below it + let blocker = tempfile::NamedTempFile::new().unwrap(); let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); let repo = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "test-password", blocker.path().join("cache")).unwrap(); snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); diff --git a/src/tests/services/sync_tests.rs b/src/tests/domain/files/sync.rs similarity index 93% rename from src/tests/services/sync_tests.rs rename to src/tests/domain/files/sync.rs index 6c6a2e6..7197a26 100644 --- a/src/tests/services/sync_tests.rs +++ b/src/tests/domain/files/sync.rs @@ -1,7 +1,5 @@ -//! Sync mode (rclone replica). Pure helpers first; integration tests need `rclone` on PATH. - -use crate::services::sync::excludes::sync_excludes; -use crate::services::sync::stats::{SyncStats, parse_log}; +use crate::domain::files::sync::excludes::sync_excludes; +use crate::domain::files::sync::stats::{SyncStats, parse_log}; fn strings(items: &[&str]) -> Vec { items.iter().map(|s| s.to_string()).collect() @@ -109,11 +107,9 @@ fn sync_excludes_collapses_repeated_double_stars() { ); } -// ---------- integration: rclone on PATH, `alias` remote under a temp dir ---------- - use crate::domain::files::platform::is_root; use crate::services::backup::logger::JobLogger; -use crate::services::sync::rclone::{replica_target, sync_dir}; +use crate::domain::files::sync::rclone::{replica_target, sync_dir}; use crate::tests::domain::files::files_config; use std::fs; use std::path::{Path, PathBuf}; @@ -128,7 +124,6 @@ fn replica(store: &Path, generated_id: &str) -> PathBuf { store.join("backups/sync").join(generated_id).join("current") } -/// `relative/path=content` for every regular file, sorted. fn files(root: &Path) -> Vec { let mut out: Vec = walkdir::WalkDir::new(root) .min_depth(1) @@ -202,7 +197,7 @@ async fn a_later_exclude_removes_files_from_the_replica() { #[tokio::test] async fn failed_sync_keeps_files_the_failed_run_would_delete() { if is_root() { - return; // root reads a 000 file anyway + return; } use std::os::unix::fs::PermissionsExt; let store = TempDir::new().unwrap(); @@ -214,7 +209,6 @@ async fn failed_sync_keeps_files_the_failed_run_would_delete() { fs::remove_file(src.path().join("docs/b.txt")).unwrap(); let locked = src.path().join("docs/a.txt"); - // rclone only opens a file whose size/mtime changed, so the unreadable file must differ from the replica's copy. fs::write(&locked, "changed and now unreadable").unwrap(); fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); let err = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err(); @@ -234,10 +228,8 @@ async fn empty_source_is_refused_and_the_replica_is_untouched() { assert!(!store.path().join("backups").exists()); } -// ---------- sync as a backup method ---------- - use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::sync::backup::replica_path; +use crate::domain::files::sync::backup::replica_path; use serde_json::json; #[test] @@ -252,7 +244,7 @@ fn replica_path_uses_the_channel_folder() { #[test] fn local_sync_config_points_at_the_dashboard_webdav_server() { - use crate::services::sync::backup::local_sync_config; + use crate::domain::files::sync::backup::local_sync_config; use crate::utils::edge_key::EdgeKey; let edge_key = EdgeKey { server_url: "http://dash:8887/".into(), diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index f09c24e..5c62f5d 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -329,7 +329,7 @@ fn ping_payload_sends_method_only_when_set() { #[test] fn upload_status_sends_stats_only_when_set() { use crate::services::api::endpoints::agent::backup::upload::status::StatusUploadRequest; - use crate::services::sync::stats::SyncStats; + use crate::domain::files::sync::stats::SyncStats; let request = |stats: Option| StatusUploadRequest { generated_id: "g".into(), backup_storage_id: "bs".into(), diff --git a/src/tests/services/local_storage_tests.rs b/src/tests/services/local_storage_tests.rs index 1c46d71..e44a4f9 100644 --- a/src/tests/services/local_storage_tests.rs +++ b/src/tests/services/local_storage_tests.rs @@ -4,9 +4,10 @@ use crate::domain::files::platform::is_root; use crate::services::backup::logger::JobLogger; -use crate::services::restic::backup::snapshot; -use crate::services::restic::command::ResticRepo; -use crate::services::restic::restore::restore; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; +use crate::services::storage::providers::local::local_storage_password; use crate::tests::domain::files::files_config; use std::fs; use std::path::Path; @@ -134,8 +135,7 @@ async fn local_sync_mirrors_through_the_webdav_server() { use crate::core::context::Context; use crate::services::api::ApiClient; use crate::services::api::models::agent::status::DatabaseStorage; - use crate::services::restic::password::local_storage_password; - use crate::services::sync::backup::one_storage; + use crate::services::backup::files_engine::{FilesEngine, one_storage}; use crate::utils::edge_key::EdgeKey; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -177,14 +177,22 @@ async fn local_sync_mirrors_through_the_webdav_server() { let cfg = files_config(&src, &[]); let replica = store.path().join(&cfg.generated_id).join("current"); - let first = one_storage(&ctx, &cfg, &storage, "backup-1", &JobLogger::new()).await; + let first = one_storage(&ctx, &cfg, FilesEngine::Sync, &storage, "backup-1", &JobLogger::new()).await; assert!(first.success, "{:?}", first.error); assert_eq!(first.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); assert_eq!(fs::read_to_string(replica.join("docs/b.txt")).unwrap(), "b"); fs::remove_file(src.join("docs/b.txt")).unwrap(); - let second = one_storage(&ctx, &cfg, &storage, "backup-2", &JobLogger::new()).await; + let second = one_storage(&ctx, &cfg, FilesEngine::Sync, &storage, "backup-2", &JobLogger::new()).await; assert!(second.success, "{:?}", second.error); assert!(!replica.join("docs/b.txt").exists(), "deletion propagated"); assert!(replica.join("a.txt").exists()); } + +#[test] +fn local_storage_password_matches_the_dashboard_vector() { + assert_eq!( + local_storage_password(MASTER_KEY_B64).unwrap(), + "8abfb788701b8c502658154746af6af768ea08f579381ad73b0f1e7dd43caea2" + ); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index e0e7408..158c1a2 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -5,6 +5,4 @@ mod config_tests; mod cron_tests; mod dashboard_config_tests; pub(crate) mod local_storage_tests; -mod restic_tests; mod restore_downloader_tests; -mod sync_tests; diff --git a/src/tests/storage/rclone_target.rs b/src/tests/storage/rclone_target.rs index 4812bf1..190c726 100644 --- a/src/tests/storage/rclone_target.rs +++ b/src/tests/storage/rclone_target.rs @@ -194,8 +194,8 @@ async fn gcs_emulator_target_reaches_fake_gcs_through_rclone() { #[tokio::test] async fn s3_target_hosts_a_restic_repository() { use crate::services::backup::logger::JobLogger; - use crate::services::restic::backup::snapshot; - use crate::services::restic::command::ResticRepo; + use crate::domain::files::restic::backup::snapshot; + use crate::domain::files::restic::repo::ResticRepo; use base64::{Engine as _, engine::general_purpose}; let (_container, endpoint) = start_minio().await; // "http://host:port" @@ -236,7 +236,7 @@ async fn s3_target_hosts_a_restic_repository() { #[tokio::test] async fn s3_targets_host_one_sync_replica_per_channel() { use crate::services::backup::logger::JobLogger; - use crate::services::sync::backup::one_storage; + use crate::services::backup::files_engine::{FilesEngine, one_storage}; use crate::tests::services::backup_uploader_tests::ctx_pointing_at; use wiremock::matchers::{body_partial_json, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -284,7 +284,7 @@ async fn s3_targets_host_one_sync_replica_per_channel() { let ctx = ctx_pointing_at(server.uri()); for c in &channels { - let result = one_storage(&ctx, &cfg, c, "backup-1", &JobLogger::new()).await; + let result = one_storage(&ctx, &cfg, FilesEngine::Sync, c, "backup-1", &JobLogger::new()).await; assert!(result.success, "{:?}", result.error); assert_eq!(result.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); assert_eq!(result.total_size, Some(2), "backups.file_size is the replica size"); diff --git a/src/utils/edge_key.rs b/src/utils/edge_key.rs index d18b87d..7a8875d 100644 --- a/src/utils/edge_key.rs +++ b/src/utils/edge_key.rs @@ -1,6 +1,9 @@ +use anyhow::Context; use base64::{Engine as _, engine::general_purpose}; +use hmac::{Hmac, Mac}; use serde::Deserialize; use serde_json::Value; +use sha2::Sha256; use thiserror::Error; #[derive(Debug, Deserialize)] @@ -45,3 +48,21 @@ pub fn decode_edge_key(edge_key: &str) -> Result { Err(EdgeKeyError::InvalidKey) } } + +pub fn derive_key(master_key_b64: &str, info: &[u8]) -> anyhow::Result<[u8; 32]> { + let master_key = general_purpose::STANDARD + .decode(master_key_b64.trim()) + .context("master key is not valid base64")?; + Ok(hkdf_sha256_32(&master_key, info)) +} + +pub fn hkdf_sha256_32(ikm: &[u8], info: &[u8]) -> [u8; 32] { + let mut extract = Hmac::::new_from_slice(&[]).expect("HMAC takes any key length"); + extract.update(ikm); + let prk = extract.finalize().into_bytes(); + + let mut expand = Hmac::::new_from_slice(&prk).expect("HMAC takes any key length"); + expand.update(info); + expand.update(&[1]); + expand.finalize().into_bytes().into() +} diff --git a/src/utils/file.rs b/src/utils/file.rs index 10bac58..1168f45 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -46,14 +46,15 @@ pub fn full_file_name(encrypt: bool) -> String { } } -pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String { - let folder_name = folder_name - .map(str::trim) - .map(|folder_name| folder_name.trim_matches(char::from(47))) +pub fn folder_or_default(folder_name: Option<&str>) -> &str { + folder_name + .map(|folder_name| folder_name.trim().trim_matches('/')) .filter(|folder_name| !folder_name.is_empty()) - .unwrap_or("backups"); + .unwrap_or("backups") +} - format!("{}/{}/{}", folder_name, chrono::Local::now().format("%Y-%m-%d"), file_name) +pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String { + format!("{}/{}/{}", folder_or_default(folder_name), chrono::Local::now().format("%Y-%m-%d"), file_name) } const CHUNK_SIZE: usize = 16 * 1024 * 1024; From 84ffff39011d1acf25559d51e4bc20bb1a97f5e6 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Sun, 18 Oct 2026 19:02:30 +0200 Subject: [PATCH 04/11] fix: files seed for large tests --- scripts/files/seed-1gb.sh | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 scripts/files/seed-1gb.sh diff --git a/scripts/files/seed-1gb.sh b/scripts/files/seed-1gb.sh new file mode 100755 index 0000000..9a4b317 --- /dev/null +++ b/scripts/files/seed-1gb.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +SIZE_MB=${SIZE_MB:-1024} +ROOT=${ROOT:-./test-files} +TARGETS=${TARGETS:-archive snapshot sync} + +bytes() { echo $(( SIZE_MB * $1 / 100 * 1024 * 1024 )); } + +random_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" /dev/urandom > "$1" +} + +random_tree() { + mkdir -p "$1" + head -c "$2" /dev/urandom | split -b "$3" -a 4 -d - "$1/part-" +} + +text_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" < <(base64 -w 76 /dev/urandom) > "$1" +} + +log_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" < <(yes "2026-10-18T12:00:00Z INFO GET /api/backups 200 12ms") > "$1" +} + +seed() { + local dir=$1 + rm -rf "$dir"/{media,projects,docs,logs,cache,build,scratch} "$dir/latest.bin" + mkdir -p "$dir" + + for i in 1 2 3 4 5 6; do + random_file "$dir/media/video-$i.bin" "$(bytes 10)" + done + for i in $(seq -w 1 20); do + random_tree "$dir/projects/p$i/src" "$(bytes 1)" 128K + done + for i in 1 2 3 4; do + text_file "$dir/docs/report-$i.txt" "$(bytes 2)" + done + log_file "$dir/logs/app.log" "$(bytes 4)" + random_tree "$dir/projects/p01/node_modules/pkg" "$(bytes 4)" 32K + random_tree "$dir/cache" "$(bytes 2)" 1M + random_tree "$dir/build" "$(bytes 1)" 1M + random_file "$dir/scratch/session.tmp" "$(bytes 1)" + ln -s media/video-1.bin "$dir/latest.bin" + + echo "$dir: $(du -sh "$dir" | cut -f1), $(find "$dir" -type f | wc -l) files" +} + +for target in $TARGETS; do + seed "${ROOT:?}/$target" +done From 2139dd494efddda6d5b15ed39e39991d20da23c5 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Sun, 18 Oct 2026 19:32:20 +0200 Subject: [PATCH 05/11] fix: stats and logs --- databases.json | 11 +++------- src/domain/files/sync/rclone.rs | 33 ++++++++++++++++++++++-------- src/domain/files/sync/stats.rs | 21 +++++++++++++++++++ src/services/restore/downloader.rs | 11 +--------- src/tests/domain/files/sync.rs | 15 +++++++++++++- src/utils/text.rs | 10 +++++++++ 6 files changed, 74 insertions(+), 27 deletions(-) diff --git a/databases.json b/databases.json index f480f42..5e65499 100644 --- a/databases.json +++ b/databases.json @@ -138,9 +138,7 @@ "path": "/data/files-archive", "generated_id": "64aeaf80-e03b-4503-a5c1-00a1b829f0e5", "options": { - "method": "archive", - "exclude": ["node_modules", "*.tmp", "/cache"], - "one_file_system": false + "method": "archive" } }, { @@ -149,9 +147,7 @@ "path": "/data/files-snapshot", "generated_id": "9093ad85-3f67-489f-9759-aa369572b30d", "options": { - "method": "snapshot", - "exclude": ["node_modules", "*.log", "/build"], - "one_file_system": true + "method": "snapshot" } }, { @@ -160,8 +156,7 @@ "path": "/data/files-sync", "generated_id": "353ecc44-7c75-40e7-803e-ac07683c2c05", "options": { - "method": "sync", - "exclude": ["*.tmp", "/cache"] + "method": "sync" } } ] diff --git a/src/domain/files/sync/rclone.rs b/src/domain/files/sync/rclone.rs index f940c7a..6dad00c 100644 --- a/src/domain/files/sync/rclone.rs +++ b/src/domain/files/sync/rclone.rs @@ -1,5 +1,5 @@ use super::excludes::sync_excludes; -use super::stats::{SyncStats, parse_log}; +use super::stats::{SyncStats, parse_log, progress}; use crate::domain::files::matcher::{exclude_patterns, one_file_system, source_root}; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; @@ -7,6 +7,7 @@ use crate::services::storage::providers::rclone::helpers::{remote_target, write_ use anyhow::{Context, Result, anyhow, bail}; use std::path::Path; use std::process::Stdio; +use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Command; pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generated_id: &str) -> String { @@ -34,7 +35,7 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge .arg(&root) .arg(dest) .args([ - "--links", "--delete-excluded", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1", + "--links", "--delete-excluded", "--use-json-log", "--stats-log-level", "NOTICE", "--retries", "1", "--contimeout", "30s", "--low-level-retries", "3", ]); if one_file_system(cfg) { @@ -44,29 +45,45 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge cmd.arg("--exclude").arg(pattern); } + let debug = tracing::enabled!(tracing::Level::DEBUG); + cmd.args(["--stats", if debug { "5s" } else { "1h" }]); + logger.log("info", format!("Syncing {} to {dest}", root.display())); - let out = cmd + let mut child = cmd .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) .kill_on_drop(true) - .output() - .await + .spawn() .map_err(|e| match e.kind() { std::io::ErrorKind::NotFound => anyhow!("rclone binary not found"), _ => anyhow!(e).context("failed to start rclone"), })?; + let mut lines = BufReader::new(child.stderr.take().context("rclone stderr unavailable")?).lines(); + let mut stderr = String::new(); + while let Some(line) = lines.next_line().await.context("failed to read rclone output")? { + if debug { + if let Some(progress) = progress(&line) { + tracing::debug!("{progress}"); + } + } + stderr.push_str(&line); + stderr.push('\n'); + } + let status = child.wait().await.context("failed to wait for rclone")?; - let log = parse_log(&String::from_utf8_lossy(&out.stderr)); + let log = parse_log(&stderr); for error in &log.errors { logger.log("warn", format!("rclone: {error}")); } - if !out.status.success() { + if !status.success() { let reason = log .stats .as_ref() .and_then(|s| s.last_error.clone()) .or_else(|| log.errors.last().cloned()) .unwrap_or_else(|| "no error message".to_string()); - bail!("rclone sync failed ({}): {reason}; the replica may be partially updated", out.status); + bail!("rclone sync failed ({status}): {reason}; the replica may be partially updated"); } let mut stats = log.stats.unwrap_or_default(); stats.replica_bytes = match remote_size(config.path(), dest).await { diff --git a/src/domain/files/sync/stats.rs b/src/domain/files/sync/stats.rs index 0772176..bfa925e 100644 --- a/src/domain/files/sync/stats.rs +++ b/src/domain/files/sync/stats.rs @@ -1,3 +1,4 @@ +use crate::utils::text::human_size; use serde::Deserialize; use serde_json::{Value, json}; @@ -62,3 +63,23 @@ pub fn parse_log(stderr: &str) -> SyncLog { } log } + +pub fn progress(line: &str) -> Option { + let value: Value = serde_json::from_str(line).ok()?; + let stats = value.get("stats")?; + let n = |key: &str| stats[key].as_u64().unwrap_or(0); + let (bytes, total) = (n("bytes"), n("totalBytes")); + let percent = if total > 0 { bytes * 100 / total } else { 100 }; + let eta = stats["eta"].as_u64().map(|s| format!(", ETA {s}s")).unwrap_or_default(); + Some(format!( + "rclone sync: {} / {} ({percent}%), {}/{} transferred, {}/{} checked, {} deleted, {}/s{eta}", + human_size(bytes), + human_size(total), + n("transfers"), + n("totalTransfers"), + n("checks"), + n("totalChecks"), + n("deletes"), + human_size(stats["speed"].as_f64().unwrap_or(0.0) as u64), + )) +} diff --git a/src/services/restore/downloader.rs b/src/services/restore/downloader.rs index bd70a98..c4ae1e0 100644 --- a/src/services/restore/downloader.rs +++ b/src/services/restore/downloader.rs @@ -2,6 +2,7 @@ use super::service::RestoreService; use crate::services::backup::logger::JobLogger; use crate::utils::retry::{RetryPolicy, retry}; +use crate::utils::text::human_size; use anyhow::Result; use futures::StreamExt; use reqwest::{Client, Url}; @@ -10,16 +11,6 @@ use std::sync::Arc; use std::time::Instant; use tokio::io::AsyncWriteExt; -fn human_size(bytes: u64) -> String { - if bytes >= 1024 * 1024 { - format!("{} MB", bytes / 1024 / 1024) - } else if bytes >= 1024 { - format!("{} KB", bytes / 1024) - } else { - format!("{bytes} B") - } -} - impl RestoreService { pub async fn download_backup( &self, diff --git a/src/tests/domain/files/sync.rs b/src/tests/domain/files/sync.rs index 7197a26..f1baea8 100644 --- a/src/tests/domain/files/sync.rs +++ b/src/tests/domain/files/sync.rs @@ -1,5 +1,5 @@ use crate::domain::files::sync::excludes::sync_excludes; -use crate::domain::files::sync::stats::{SyncStats, parse_log}; +use crate::domain::files::sync::stats::{SyncStats, parse_log, progress}; fn strings(items: &[&str]) -> Vec { items.iter().map(|s| s.to_string()).collect() @@ -71,6 +71,19 @@ fn parse_log_reads_a_successful_run() { assert!(log.errors.is_empty()); } +#[test] +fn progress_formats_periodic_stats_and_ignores_other_lines() { + let line = r#"{"level":"notice","msg":"stats","stats":{"bytes":536870912,"totalBytes":1073741824,"transfers":1200,"totalTransfers":2924,"checks":3,"totalChecks":10,"deletes":0,"speed":89128960.5,"eta":6}}"#; + assert_eq!( + progress(line).unwrap(), + "rclone sync: 512 MB / 1024 MB (50%), 1200/2924 transferred, 3/10 checked, 0 deleted, 85 MB/s, ETA 6s" + ); + let idle = r#"{"level":"notice","stats":{"bytes":0,"totalBytes":0,"speed":0,"eta":null}}"#; + assert_eq!(progress(idle).unwrap(), "rclone sync: 0 B / 0 B (100%), 0/0 transferred, 0/0 checked, 0 deleted, 0 B/s"); + assert!(progress(r#"{"level":"error","msg":"boom"}"#).is_none()); + assert!(progress("plain text").is_none()); +} + #[test] fn parse_log_keeps_plain_text_lines_as_errors() { let log = parse_log("Error: unknown flag: --bogus\n"); diff --git a/src/utils/text.rs b/src/utils/text.rs index a8814ea..e3ef593 100644 --- a/src/utils/text.rs +++ b/src/utils/text.rs @@ -53,3 +53,13 @@ fn convert_dow_part(part: &str) -> String { None => converted, } } + +pub fn human_size(bytes: u64) -> String { + if bytes >= 1024 * 1024 { + format!("{} MB", bytes / 1024 / 1024) + } else if bytes >= 1024 { + format!("{} KB", bytes / 1024) + } else { + format!("{bytes} B") + } +} From 912bd05547618251e3b8209d7a6abb842531a05e Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Sun, 18 Oct 2026 20:03:07 +0200 Subject: [PATCH 06/11] fix: databases.json --- databases.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/databases.json b/databases.json index 5e65499..f480f42 100644 --- a/databases.json +++ b/databases.json @@ -138,7 +138,9 @@ "path": "/data/files-archive", "generated_id": "64aeaf80-e03b-4503-a5c1-00a1b829f0e5", "options": { - "method": "archive" + "method": "archive", + "exclude": ["node_modules", "*.tmp", "/cache"], + "one_file_system": false } }, { @@ -147,7 +149,9 @@ "path": "/data/files-snapshot", "generated_id": "9093ad85-3f67-489f-9759-aa369572b30d", "options": { - "method": "snapshot" + "method": "snapshot", + "exclude": ["node_modules", "*.log", "/build"], + "one_file_system": true } }, { @@ -156,7 +160,8 @@ "path": "/data/files-sync", "generated_id": "353ecc44-7c75-40e7-803e-ac07683c2c05", "options": { - "method": "sync" + "method": "sync", + "exclude": ["*.tmp", "/cache"] } } ] From bb2d79dc65ed02fcbfab42216ae5133f034f8f94 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 20 Oct 2026 16:22:58 +0200 Subject: [PATCH 07/11] feat(api): sign dashboard callbacks with the agent key --- src/core/context.rs | 3 +- src/services/api/client.rs | 65 ++++++++++++++++++-- src/tests/services/api_client_tests.rs | 84 ++++++++++++++++++++++++++ src/tests/services/mod.rs | 1 + 4 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 src/tests/services/api_client_tests.rs diff --git a/src/core/context.rs b/src/core/context.rs index c37823d..d1e7197 100644 --- a/src/core/context.rs +++ b/src/core/context.rs @@ -37,7 +37,8 @@ impl Context { }; let server_url = format!("{}/api", edge_key.server_url); - let api_client = ApiClient::new(server_url); + let api_client = ApiClient::new(server_url) + .with_agent_auth(&edge_key.agent_id, &edge_key.master_key_b64); Context { edge_key: edge_key, diff --git a/src/services/api/client.rs b/src/services/api/client.rs index f1e396c..d447334 100644 --- a/src/services/api/client.rs +++ b/src/services/api/client.rs @@ -1,14 +1,37 @@ #![allow(dead_code)] use crate::services::api::ApiError; -use reqwest::{Client, Method}; +use base64::{Engine as _, engine::general_purpose}; +use hmac::{Hmac, Mac}; +use reqwest::header::CONTENT_TYPE; +use reqwest::{Client, Method, RequestBuilder}; use serde::de::DeserializeOwned; -use std::time::Duration; +use sha2::{Digest, Sha256}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +type HmacSha256 = Hmac; #[derive(Clone, Debug)] pub struct ApiClient { base_url: String, http: Client, + agent_key: Option>, +} + +/// Per-agent signing key: HMAC-SHA256(master key, "portabase-agent-auth:v1:" + agent id). +pub fn derive_agent_key(master_key: &[u8], agent_id: &str) -> Vec { + let mut mac = HmacSha256::new_from_slice(master_key).expect("HMAC accepts any key length"); + mac.update(format!("portabase-agent-auth:v1:{agent_id}").as_bytes()); + mac.finalize().into_bytes().to_vec() +} + +/// Lowercase hex HMAC-SHA256(agent key, METHOD \n PATH \n TIMESTAMP \n hex(sha256(body))). +/// Must match the dashboard's `signAgentRequest` (src/utils/status-crypto.ts). +pub fn sign_request(agent_key: &[u8], method: &str, path: &str, timestamp: &str, body: &[u8]) -> String { + let body_hash = hex::encode(Sha256::digest(body)); + let mut mac = HmacSha256::new_from_slice(agent_key).expect("HMAC accepts any key length"); + mac.update(format!("{method}\n{path}\n{timestamp}\n{body_hash}").as_bytes()); + hex::encode(mac.finalize().into_bytes()) } impl ApiClient { @@ -21,9 +44,36 @@ impl ApiClient { Self { base_url: base_url.into(), http, + agent_key: None, } } + /// Sign every dashboard call with the key derived from the EDGE_KEY master key. + pub fn with_agent_auth(mut self, agent_id: &str, master_key_b64: &str) -> Self { + let master_key = general_purpose::STANDARD + .decode(master_key_b64) + .expect("EDGE_KEY master key is not valid base64"); + self.agent_key = Some(derive_agent_key(&master_key, agent_id)); + self + } + + /// Adds X-Agent-Timestamp / X-Agent-Signature, computed per call. The signed path is the + /// dashboard route ("/api" + path), independent of any prefix in the server URL. + fn sign(&self, request: RequestBuilder, method: &Method, path: &str, body: &[u8]) -> RequestBuilder { + let Some(agent_key) = &self.agent_key else { + return request; + }; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before 1970") + .as_secs() + .to_string(); + let signature = sign_request(agent_key, method.as_str(), &format!("/api{path}"), ×tamp, body); + request + .header("X-Agent-Timestamp", timestamp) + .header("X-Agent-Signature", signature) + } + pub async fn request( &self, method: Method, @@ -31,7 +81,8 @@ impl ApiClient { ) -> Result, ApiError> { let url = format!("{}{}", self.base_url, path); - let res = self.http.request(method, &url).send().await?; + let request = self.sign(self.http.request(method.clone(), &url), &method, path, b""); + let res = request.send().await?; let status = res.status(); let body = res.text().await.unwrap_or_default(); @@ -57,7 +108,13 @@ impl ApiClient { { let url = format!("{}{}", self.base_url, path); - let res = self.http.request(method, &url).json(body).send().await?; + // Serialize once: the signature covers exactly the bytes that are sent. + let bytes = serde_json::to_vec(body)?; + let request = self + .http + .request(method.clone(), &url) + .header(CONTENT_TYPE, "application/json"); + let res = self.sign(request, &method, path, &bytes).body(bytes).send().await?; let status = res.status(); let body_text = res.text().await.unwrap_or_default(); diff --git a/src/tests/services/api_client_tests.rs b/src/tests/services/api_client_tests.rs new file mode 100644 index 0000000..91b65a1 --- /dev/null +++ b/src/tests/services/api_client_tests.rs @@ -0,0 +1,84 @@ +use crate::services::api::ApiClient; +use crate::services::api::client::{derive_agent_key, sign_request}; + +use base64::{Engine as _, engine::general_purpose}; +use reqwest::Method; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +// Shared test vector, asserted identically by the dashboard (src/utils/status-crypto.ts). +const AGENT_ID: &str = "11111111-1111-4111-8111-111111111111"; + +fn vector_master_key() -> Vec { + (0u8..32).collect() +} + +#[test] +fn derived_agent_key_matches_the_dashboard_vector() { + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + + assert_eq!( + hex::encode(key), + "16bee8dcf9d5f497cdd882ca208ff7f630ca02a5ecfe157f34239838fdcd9006" + ); +} + +#[test] +fn request_signature_matches_the_dashboard_vector() { + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + let path = format!("/api/agent/{AGENT_ID}/status"); + + assert_eq!( + sign_request(&key, "POST", &path, "1760968800", br#"{"version":"1.24.0","databases":[]}"#), + "ad1e56a50ac0c4a3f539f3180d2957ebeb9a61bd1df44e57b50f8e59dd269069" + ); + assert_eq!( + sign_request(&key, "POST", &path, "1760968800", "{\"name\":\"Base é\"}".as_bytes()), + "d3e6911c9273e0224b8dbae72cabdd3720973f1866c651fb4c99e1ac03e73231" + ); +} + +#[tokio::test] +async fn every_request_is_signed_with_a_fresh_timestamp() { + let server = MockServer::start().await; + let route = format!("/agent/{AGENT_ID}/status"); + Mock::given(method("POST")) + .and(path(route.as_str())) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + + let client = ApiClient::new(server.uri()) + .with_agent_auth(AGENT_ID, &general_purpose::STANDARD.encode(vector_master_key())); + let body = serde_json::json!({"version": "1.24.0", "databases": [{"name": "Base é"}]}); + + for i in 0..2 { + if i > 0 { + tokio::time::sleep(Duration::from_millis(1100)).await; + } + let _: Option = client + .request_with_body(Method::POST, &route, &body) + .await + .expect("signed request accepted by the mock"); + } + + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + let requests = server.received_requests().await.unwrap(); + let mut timestamps = Vec::new(); + for req in &requests { + let ts = req.headers.get("x-agent-timestamp").unwrap().to_str().unwrap(); + assert!(ts.chars().all(|c| c.is_ascii_digit()), "timestamp {ts}"); + assert!(now.abs_diff(ts.parse::().unwrap()) <= 5); + assert_eq!(req.headers.get("content-type").unwrap(), "application/json"); + assert_eq!( + req.headers.get("x-agent-signature").unwrap().to_str().unwrap(), + sign_request(&key, "POST", &format!("/api{route}"), ts, &req.body) + ); + timestamps.push(ts.to_string()); + } + assert_eq!(requests.len(), 2); + assert_ne!(timestamps[0], timestamps[1], "each call signs with its own timestamp"); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index d825608..b784374 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -1,3 +1,4 @@ +mod api_client_tests; mod api_models_tests; mod backup_runner_tests; mod backup_uploader_tests; From c6794fd919c42daab44cbc248cc5d9899e011459 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 20 Oct 2026 16:45:10 +0200 Subject: [PATCH 08/11] fix(api): log an invalid EDGE_KEY master key before stopping --- src/services/api/client.rs | 11 +++++++--- src/tests/services/api_client_tests.rs | 30 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/services/api/client.rs b/src/services/api/client.rs index d447334..67eef62 100644 --- a/src/services/api/client.rs +++ b/src/services/api/client.rs @@ -8,6 +8,7 @@ use reqwest::{Client, Method, RequestBuilder}; use serde::de::DeserializeOwned; use sha2::{Digest, Sha256}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tracing::error; type HmacSha256 = Hmac; @@ -50,9 +51,13 @@ impl ApiClient { /// Sign every dashboard call with the key derived from the EDGE_KEY master key. pub fn with_agent_auth(mut self, agent_id: &str, master_key_b64: &str) -> Self { - let master_key = general_purpose::STANDARD - .decode(master_key_b64) - .expect("EDGE_KEY master key is not valid base64"); + let master_key = match general_purpose::STANDARD.decode(master_key_b64) { + Ok(key) => key, + Err(_) => { + error!("Invalid EDGE_KEY: master key is not valid base64"); + panic!("Cannot initialize AgentContext due to invalid EDGE_KEY"); + } + }; self.agent_key = Some(derive_agent_key(&master_key, agent_id)); self } diff --git a/src/tests/services/api_client_tests.rs b/src/tests/services/api_client_tests.rs index 91b65a1..bd68019 100644 --- a/src/tests/services/api_client_tests.rs +++ b/src/tests/services/api_client_tests.rs @@ -3,6 +3,7 @@ use crate::services::api::client::{derive_agent_key, sign_request}; use base64::{Engine as _, engine::general_purpose}; use reqwest::Method; +use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -39,6 +40,35 @@ fn request_signature_matches_the_dashboard_vector() { ); } +#[test] +fn invalid_master_key_logs_an_error_before_panicking() { + #[derive(Clone, Default)] + struct Logs(Arc>>); + impl std::io::Write for Logs { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let logs = Logs::default(); + let writer = logs.clone(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + + let result = tracing::subscriber::with_default(subscriber, || { + std::panic::catch_unwind(|| ApiClient::new("http://dashboard").with_agent_auth(AGENT_ID, "not base64!")) + }); + + assert!(result.is_err(), "an invalid master key must stop the agent"); + let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!(output.contains("ERROR") && output.contains("Invalid EDGE_KEY"), "logs: {output}"); +} + #[tokio::test] async fn every_request_is_signed_with_a_fresh_timestamp() { let server = MockServer::start().await; From 391eaf03366e2da3ac104540cf647625e44e1c07 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 20 Oct 2026 16:50:22 +0200 Subject: [PATCH 09/11] fix: refactoring --- docker-compose.yml | 2 +- src/services/api/client.rs | 6 ------ src/tests/services/api_client_tests.rs | 1 - 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4b18ba7..3557832 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiN2ZjMGViODMtYTY0YS00NzI3LTg5ZjktNzFhZWJmZDQ3NmEyIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMjA1Zjk2YzAtMjhmMi00MzdiLTkzOGYtNzExOTY1MGE4Y2YwIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 diff --git a/src/services/api/client.rs b/src/services/api/client.rs index 67eef62..3d734de 100644 --- a/src/services/api/client.rs +++ b/src/services/api/client.rs @@ -19,15 +19,12 @@ pub struct ApiClient { agent_key: Option>, } -/// Per-agent signing key: HMAC-SHA256(master key, "portabase-agent-auth:v1:" + agent id). pub fn derive_agent_key(master_key: &[u8], agent_id: &str) -> Vec { let mut mac = HmacSha256::new_from_slice(master_key).expect("HMAC accepts any key length"); mac.update(format!("portabase-agent-auth:v1:{agent_id}").as_bytes()); mac.finalize().into_bytes().to_vec() } -/// Lowercase hex HMAC-SHA256(agent key, METHOD \n PATH \n TIMESTAMP \n hex(sha256(body))). -/// Must match the dashboard's `signAgentRequest` (src/utils/status-crypto.ts). pub fn sign_request(agent_key: &[u8], method: &str, path: &str, timestamp: &str, body: &[u8]) -> String { let body_hash = hex::encode(Sha256::digest(body)); let mut mac = HmacSha256::new_from_slice(agent_key).expect("HMAC accepts any key length"); @@ -49,7 +46,6 @@ impl ApiClient { } } - /// Sign every dashboard call with the key derived from the EDGE_KEY master key. pub fn with_agent_auth(mut self, agent_id: &str, master_key_b64: &str) -> Self { let master_key = match general_purpose::STANDARD.decode(master_key_b64) { Ok(key) => key, @@ -62,8 +58,6 @@ impl ApiClient { self } - /// Adds X-Agent-Timestamp / X-Agent-Signature, computed per call. The signed path is the - /// dashboard route ("/api" + path), independent of any prefix in the server URL. fn sign(&self, request: RequestBuilder, method: &Method, path: &str, body: &[u8]) -> RequestBuilder { let Some(agent_key) = &self.agent_key else { return request; diff --git a/src/tests/services/api_client_tests.rs b/src/tests/services/api_client_tests.rs index bd68019..15275f5 100644 --- a/src/tests/services/api_client_tests.rs +++ b/src/tests/services/api_client_tests.rs @@ -8,7 +8,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; -// Shared test vector, asserted identically by the dashboard (src/utils/status-crypto.ts). const AGENT_ID: &str = "11111111-1111-4111-8111-111111111111"; fn vector_master_key() -> Vec { From 7f8ecb735d5a712898ef4b28f67dab140e57120e Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 20 Oct 2026 18:27:36 +0200 Subject: [PATCH 10/11] fix: refactoring --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c8a0fc2..b8631c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGUyMzgxYzgtNzI4Mi00MzIxLTk1ZjQtYTU3NTRiZTc2YmFlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMjA1Zjk2YzAtMjhmMi00MzdiLTkzOGYtNzExOTY1MGE4Y2YwIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 From d81b0ff5408650c7524aaba4e333194475ea1054 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 20 Oct 2026 19:53:32 +0200 Subject: [PATCH 11/11] fix: rclone logs --- Cargo.lock | 777 +++++++++++++++----------------- docker-compose.yml | 2 +- src/domain/files/sync/rclone.rs | 2 +- 3 files changed, 369 insertions(+), 412 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2ceb93..127d12f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,12 +65,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "android_system_properties" version = "0.1.6" @@ -94,9 +88,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" dependencies = [ "rustversion", ] @@ -135,9 +129,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" dependencies = [ "compression-codecs", "compression-core", @@ -186,7 +180,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -230,7 +224,7 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -259,9 +253,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -270,9 +264,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -283,9 +277,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -299,9 +293,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", "http-body 1.1.0", "percent-encoding", "pin-project-lite", @@ -311,9 +303,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.146.1" +version = "1.152.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971" +checksum = "004d5acbd6850ab58789a43822e0a2ef23927a46b012c17f8fa51253103db9f8" dependencies = [ "arc-swap", "aws-credential-types", @@ -327,7 +319,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -335,22 +327,21 @@ dependencies = [ "fastrand", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" dependencies = [ "arc-swap", "aws-credential-types", @@ -361,12 +352,11 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -374,9 +364,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" dependencies = [ "arc-swap", "aws-credential-types", @@ -387,12 +377,11 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -400,9 +389,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" dependencies = [ "arc-swap", "aws-credential-types", @@ -414,12 +403,11 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -427,9 +415,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -441,11 +429,10 @@ dependencies = [ "form_urlencoded", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "p256", "percent-encoding", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "time", "tracing", @@ -454,9 +441,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" dependencies = [ "futures-util", "pin-project-lite", @@ -480,15 +467,15 @@ dependencies = [ "md-5", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.61.2" +version = "0.61.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f" +checksum = "f4aecd9c411046779355be31692bdd451ae35b3983ff40dd65258edffcc1a1ce" dependencies = [ "aws-smithy-types", "bytes", @@ -497,9 +484,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.64.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -513,48 +500,47 @@ dependencies = [ "http-body-util", "percent-encoding", "pin-project-lite", - "pin-utils", "tracing", ] [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.19", + "h2 0.4.20", "http 0.2.12", "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.63.0" +version = "0.63.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", ] @@ -569,12 +555,12 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "urlencoding", @@ -582,35 +568,32 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.0.0", "aws-smithy-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", "http-body 1.1.0", "http-body-util", "pin-project-lite", - "pin-utils", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -637,9 +620,20 @@ dependencies = [ [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "1d7a3b01a2a1b453176457b260b0ed55d33de61d35d21b74022c51ec62a42895" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -648,9 +642,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" dependencies = [ "base64-simd", "bytes", @@ -664,7 +658,6 @@ dependencies = [ "itoa", "num-integer", "pin-project-lite", - "pin-utils", "ryu", "serde", "time", @@ -674,12 +667,12 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "xmlparser", ] @@ -693,7 +686,7 @@ dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "rustc_version", "tracing", @@ -906,23 +899,23 @@ dependencies = [ "home", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-named-pipe", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "hyperlocal", "log", "num", "pin-project-lite", "rand 0.9.5", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-pki-types", "serde", "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tokio", "tokio-stream", @@ -1037,9 +1030,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" dependencies = [ "find-msvc-tools", "jobserver", @@ -1049,9 +1042,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -1112,9 +1105,9 @@ checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "combine" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" dependencies = [ "bytes", "futures-core", @@ -1126,9 +1119,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" dependencies = [ "compression-core", "flate2", @@ -1137,9 +1130,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" [[package]] name = "connection-string" @@ -1265,9 +1258,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -1363,9 +1356,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", ] @@ -1390,7 +1383,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1401,7 +1394,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1430,9 +1423,9 @@ checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1440,23 +1433,23 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1472,9 +1465,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" dependencies = [ "serde_core", ] @@ -1492,13 +1485,13 @@ dependencies = [ [[package]] name = "derive-where" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1556,7 +1549,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1610,9 +1603,9 @@ dependencies = [ [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "elliptic-curve" @@ -1636,13 +1629,12 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", "core_detect", - "multiversion", "multiversion_no_op", "rustversion", "scopeguard", @@ -1671,9 +1663,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -1734,7 +1726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" dependencies = [ "portable-atomic", - "rand 0.10.2", + "rand 0.10.3", "web-time", ] @@ -1760,9 +1752,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -1873,7 +1865,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1981,9 +1973,9 @@ dependencies = [ [[package]] name = "google-cloud-auth" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff461519b1a948200f163574be072753bcfb462a323f0eb426629d89872dd685" +checksum = "ac9870b8ea484519aa8601e14886f2efc148ea490034e43d04bef1d246170cc9" dependencies = [ "async-trait", "aws-lc-rs", @@ -1997,12 +1989,12 @@ dependencies = [ "jsonwebtoken", "reqwest 0.13.5", "rustc_version", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", - "thiserror 2.0.20", + "sha2 0.11.1", + "thiserror 2.0.21", "time", "tokio", "url", @@ -2010,9 +2002,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5615cff28ee59cfe52fbb4c11b8b1e77f650296e2ea4f4c2b7757ac6b19e752" +checksum = "fc64ee071c0d22cb68662b454f1bea5424506cafeaddc51260fbc585bac1d29d" dependencies = [ "bytes", "futures", @@ -2020,19 +2012,19 @@ dependencies = [ "google-cloud-wkt", "http 1.5.0", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", ] [[package]] name = "google-cloud-gax-internal" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2766757d877a7a8ac23da9884cb0e3f10ed9b75a0ce59801ce6b19bf9d5819e" +checksum = "e873f7e6e39030c07b6d9697bf3b999a269175c8b433e058f56ae10d129f12f7" dependencies = [ "bytes", "futures", @@ -2040,11 +2032,11 @@ dependencies = [ "google-cloud-gax", "google-cloud-rpc", "google-cloud-wkt", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "opentelemetry", "opentelemetry-semantic-conventions", @@ -2057,7 +2049,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tonic", @@ -2069,9 +2061,9 @@ dependencies = [ [[package]] name = "google-cloud-iam-v1" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f962b40234b1531e6ef73f7558871c96e117231e962086c98804323fb8d2c82" +checksum = "48f0895fa7b7bd70629ae58daaa962677e93a72c251b2f297bc058a035940821" dependencies = [ "async-trait", "bytes", @@ -2087,9 +2079,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c0363c5389ffda2b55cd8a86eef4b19a3481a48467c91dc5d77f626a9572766" +checksum = "e8ea252eee22a4e2187025ea6789c682a0d791a9748ea99de2abec3026a32cb4" dependencies = [ "async-trait", "bytes", @@ -2105,9 +2097,9 @@ dependencies = [ [[package]] name = "google-cloud-lro" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47af3deef75c14a2983c430898d960c765bddbcc9f9188ca0563108e9227cfe7" +checksum = "dfb938b40ede9ef70c24f49c2687d84f289c9e9104ddfece739985e4b4844b8f" dependencies = [ "google-cloud-gax", "google-cloud-gax-internal", @@ -2134,9 +2126,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.18.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973399251b245c63f1d02d0768772833dcf1372ee358f593fb9159de8fe9c7d4" +checksum = "570eb8d0784118a5f4ae1219fd76ff3b14e90b7b0f877478c02b1374e9bbcb8c" dependencies = [ "async-trait", "base64 0.23.1", @@ -2163,8 +2155,8 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2 0.11.0", - "thiserror 2.0.20", + "sha2 0.11.1", + "thiserror 2.0.21", "tokio", "tokio-stream", "tracing", @@ -2187,16 +2179,16 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7" +checksum = "7f3037907e542cf8b8580aa4cf8ad68568d7ac8e7274519a7d0a121867d434da" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "url", ] @@ -2233,9 +2225,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -2262,8 +2254,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] @@ -2295,8 +2285,8 @@ dependencies = [ "idna", "ipnet", "jni", - "rand 0.10.2", - "thiserror 2.0.20", + "rand 0.10.3", + "thiserror 2.0.21", "tinyvec", "tokio", "tracing", @@ -2315,9 +2305,9 @@ dependencies = [ "jni", "once_cell", "prefix-trie", - "rand 0.10.2", + "rand 0.10.3", "ring", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "url", @@ -2340,11 +2330,11 @@ dependencies = [ "ndk-context", "once_cell", "parking_lot", - "rand 0.10.2", + "rand 0.10.3", "resolv-conf", "smallvec", "system-configuration", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", ] @@ -2478,15 +2468,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -2505,7 +2495,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2529,17 +2519,17 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower-service", "webpki-roots", ] @@ -2550,7 +2540,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2559,17 +2549,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.1", + "httparse", + "hyper 1.12.0", "ipnet", "libc", "percent-encoding", @@ -2590,7 +2581,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2756,9 +2747,9 @@ dependencies = [ [[package]] name = "inout" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" dependencies = [ "hybrid-array", ] @@ -2802,9 +2793,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -2829,9 +2820,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -2841,9 +2832,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -2866,7 +2857,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -2915,9 +2906,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -2926,9 +2917,9 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881733cbc631fc9e472e24447ce32a64bedf2da498d6d8570b08edc87de71f65" +checksum = "e75fe14a82d81e5f5af639997db37d8b96045938a7ac6ab18cdbe1c7467e05e1" dependencies = [ "aws-lc-rs", "base64 0.22.1", @@ -2942,21 +2933,21 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libredox" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -2990,18 +2981,18 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" dependencies = [ "hashbrown 0.17.1", ] [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "macro_magic" @@ -3116,9 +3107,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3213,19 +3204,19 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "rustc_version_runtime", - "rustls 0.23.44", + "rustls 0.23.46", "serde", "serde_bytes", "serde_with", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "socket2 0.6.5", "stringprep", "strsim", "take_mut", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-util", "typed-builder", "uuid", @@ -3244,27 +3235,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.5", -] - [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -3386,9 +3356,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "cabcc654086861f9e1bc5115258de62ef92c79b20f4615a1b0cc57c4e82027d4" dependencies = [ "libc", ] @@ -3455,29 +3425,29 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.1" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" +checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", @@ -3486,7 +3456,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -3630,12 +3600,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkcs8" version = "0.10.2" @@ -3692,7 +3656,7 @@ dependencies = [ "google-cloud-storage", "hex", "hmac 0.12.1", - "hyper 1.11.1", + "hyper 1.12.0", "libc", "log", "mockall", @@ -3704,7 +3668,7 @@ dependencies = [ "rand 0.9.5", "redis", "reqwest 0.13.5", - "rustls 0.23.44", + "rustls 0.23.46", "serde", "serde_json", "sha2 0.10.9", @@ -3712,7 +3676,7 @@ dependencies = [ "tempfile", "testcontainers", "testcontainers-modules", - "thiserror 2.0.20", + "thiserror 2.0.21", "tiberius", "time", "tokio", @@ -3773,8 +3737,8 @@ dependencies = [ "hmac 0.13.0", "md-5", "memchr", - "rand 0.10.2", - "sha2 0.11.0", + "rand 0.10.3", + "sha2 0.11.1", "stringprep", ] @@ -3800,9 +3764,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -3918,9 +3882,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -3928,9 +3892,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.44", + "rustls 0.23.46", "socket2 0.6.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -3938,22 +3902,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -3961,9 +3925,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -4023,9 +3987,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -4087,9 +4051,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9" dependencies = [ "arcstr", "async-lock", @@ -4145,7 +4109,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -4195,22 +4159,22 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper 1.12.0", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower", "tower-http", "tower-service", @@ -4233,12 +4197,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper 1.12.0", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", @@ -4247,7 +4211,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -4255,7 +4219,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-util", "tower", "tower-http", @@ -4324,9 +4288,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.13.2", "errno", @@ -4349,16 +4313,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.44" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.15", + "rustls-webpki 0.103.16", "subtle", "zeroize", ] @@ -4377,9 +4341,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" dependencies = [ "openssl-probe 0.2.1", "rustls-pki-types", @@ -4408,19 +4372,19 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", "once_cell", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-platform-verifier-android", - "rustls-webpki 0.103.15", + "rustls-webpki 0.103.16", "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", @@ -4429,9 +4393,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" @@ -4445,9 +4409,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4478,9 +4442,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -4499,9 +4463,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -4618,15 +4582,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", "indexmap 2.14.2", "itoa", "memchr", @@ -4654,14 +4619,14 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "serde_spanned" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" dependencies = [ "serde_core", ] @@ -4680,9 +4645,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" dependencies = [ "base64 0.23.1", "bs58", @@ -4692,7 +4657,7 @@ dependencies = [ "indexmap 2.14.2", "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -4701,14 +4666,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ "darling", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -4752,9 +4717,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -4820,9 +4785,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "slab" @@ -4832,9 +4797,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.1" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" [[package]] name = "socket2" @@ -4943,9 +4908,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -4963,13 +4928,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -5065,7 +5030,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -5092,11 +5057,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -5112,13 +5077,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -5211,24 +5176,15 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" dependencies = [ "bytes", "libc", @@ -5249,7 +5205,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -5271,7 +5227,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.10.2", + "rand 0.10.3", "socket2 0.6.5", "tokio", "tokio-util", @@ -5290,11 +5246,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ - "rustls 0.23.44", + "rustls 0.23.46", "tokio", ] @@ -5327,9 +5283,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -5367,18 +5323,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow 1.0.4", ] [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" [[package]] name = "tonic" @@ -5390,20 +5346,20 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "rustls-native-certs 0.8.4", + "rustls-native-certs 0.8.5", "socket2 0.6.5", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-stream", "tower", "tower-layer", @@ -5496,7 +5452,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -5535,9 +5491,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" dependencies = [ "js-sys", "opentelemetry", @@ -5626,7 +5582,7 @@ dependencies = [ "dyn-clone", "futures", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "serde", "serde_json", @@ -5653,9 +5609,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-bidi" @@ -5665,15 +5621,15 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" dependencies = [ "tinyvec", ] @@ -5720,14 +5676,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5546be8f5378d5414f83733f5c9a2526f4645829edbc1c41790aeef1b38e8b" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" dependencies = [ "base64 0.23.1", "log", "percent-encoding", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "ureq-proto", "utf8-zero", @@ -5735,9 +5691,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabc3e92916c89c95b20eef7b06b00b066bc217ef9ea3a4ac9bf1a7e35261e10" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" dependencies = [ "base64 0.23.1", "http 1.5.0", @@ -5778,9 +5734,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.26.1" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -5818,9 +5774,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -5860,9 +5816,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -5873,19 +5829,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5893,22 +5850,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -5928,9 +5885,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -5948,18 +5905,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -6199,7 +6156,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "log", "once_cell", @@ -6249,9 +6206,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yoke" @@ -6266,30 +6223,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -6307,21 +6264,21 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] @@ -6367,14 +6324,14 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" diff --git a/docker-compose.yml b/docker-compose.yml index b8631c2..660a417 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMjA1Zjk2YzAtMjhmMi00MzdiLTkzOGYtNzExOTY1MGE4Y2YwIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMzM1YjcwZjYtMjQxNC00OGZiLTkxZDQtZjQzZjJjNTJlYzUzIiwibWFzdGVyS2V5QjY0IjoiOEdScEFNMzhDeE1seTAzeTFIbHpBSTdZbnRDYVhuYTV1bDBIYkxKSG81ND0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 diff --git a/src/domain/files/sync/rclone.rs b/src/domain/files/sync/rclone.rs index 6dad00c..d85525d 100644 --- a/src/domain/files/sync/rclone.rs +++ b/src/domain/files/sync/rclone.rs @@ -64,7 +64,7 @@ pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logge while let Some(line) = lines.next_line().await.context("failed to read rclone output")? { if debug { if let Some(progress) = progress(&line) { - tracing::debug!("{progress}"); + logger.log("debug", format!("progress: {progress}")); } } stderr.push_str(&line);