diff --git a/.gitignore b/.gitignore index 675c67b..1e2220f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ /docs .superpowers + +/test-files diff --git a/Cargo.lock b/Cargo.lock index 84d3d9d..127d12f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,12 +65,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "android_system_properties" version = "0.1.6" @@ -94,9 +88,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" dependencies = [ "rustversion", ] @@ -135,9 +129,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" dependencies = [ "compression-codecs", "compression-core", @@ -186,7 +180,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -230,7 +224,7 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -259,9 +253,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -270,9 +264,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -283,9 +277,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -299,9 +293,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", "http-body 1.1.0", "percent-encoding", "pin-project-lite", @@ -311,9 +303,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.146.1" +version = "1.152.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971" +checksum = "004d5acbd6850ab58789a43822e0a2ef23927a46b012c17f8fa51253103db9f8" dependencies = [ "arc-swap", "aws-credential-types", @@ -327,7 +319,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -335,22 +327,21 @@ dependencies = [ "fastrand", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "http-body 1.1.0", "lru", "percent-encoding", "regex-lite", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" dependencies = [ "arc-swap", "aws-credential-types", @@ -361,12 +352,11 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -374,9 +364,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" dependencies = [ "arc-swap", "aws-credential-types", @@ -387,12 +377,11 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -400,9 +389,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" dependencies = [ "arc-swap", "aws-credential-types", @@ -414,12 +403,11 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -427,9 +415,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -441,11 +429,10 @@ dependencies = [ "form_urlencoded", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "p256", "percent-encoding", - "sha2 0.11.0", + "sha2 0.11.1", "subtle", "time", "tracing", @@ -454,9 +441,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" dependencies = [ "futures-util", "pin-project-lite", @@ -480,15 +467,15 @@ dependencies = [ "md-5", "pin-project-lite", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.61.2" +version = "0.61.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f" +checksum = "f4aecd9c411046779355be31692bdd451ae35b3983ff40dd65258edffcc1a1ce" dependencies = [ "aws-smithy-types", "bytes", @@ -497,9 +484,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.64.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -513,48 +500,47 @@ dependencies = [ "http-body-util", "percent-encoding", "pin-project-lite", - "pin-utils", "tracing", ] [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.19", + "h2 0.4.20", "http 0.2.12", "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.63.0" +version = "0.63.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", ] @@ -569,12 +555,12 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "urlencoding", @@ -582,35 +568,32 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.0.0", "aws-smithy-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", "http-body 1.1.0", "http-body-util", "pin-project-lite", - "pin-utils", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -637,9 +620,20 @@ dependencies = [ [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "1d7a3b01a2a1b453176457b260b0ed55d33de61d35d21b74022c51ec62a42895" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -648,9 +642,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" dependencies = [ "base64-simd", "bytes", @@ -664,7 +658,6 @@ dependencies = [ "itoa", "num-integer", "pin-project-lite", - "pin-utils", "ryu", "serde", "time", @@ -674,12 +667,12 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "xmlparser", ] @@ -693,7 +686,7 @@ dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "rustc_version", "tracing", @@ -906,23 +899,23 @@ dependencies = [ "home", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-named-pipe", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "hyperlocal", "log", "num", "pin-project-lite", "rand 0.9.5", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-pki-types", "serde", "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tokio", "tokio-stream", @@ -994,6 +987,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -1027,9 +1030,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" dependencies = [ "find-msvc-tools", "jobserver", @@ -1039,9 +1042,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -1102,9 +1105,9 @@ checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "combine" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" dependencies = [ "bytes", "futures-core", @@ -1116,9 +1119,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" dependencies = [ "compression-core", "flate2", @@ -1127,9 +1130,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" [[package]] name = "connection-string" @@ -1255,9 +1258,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -1353,9 +1356,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", ] @@ -1380,7 +1383,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1391,7 +1394,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1420,9 +1423,9 @@ checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -1430,23 +1433,23 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -1462,9 +1465,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" dependencies = [ "serde_core", ] @@ -1482,13 +1485,13 @@ dependencies = [ [[package]] name = "derive-where" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -1546,7 +1549,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1600,9 +1603,9 @@ dependencies = [ [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "elliptic-curve" @@ -1626,13 +1629,12 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", "core_detect", - "multiversion", "multiversion_no_op", "rustversion", "scopeguard", @@ -1661,9 +1663,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -1724,7 +1726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" dependencies = [ "portable-atomic", - "rand 0.10.2", + "rand 0.10.3", "web-time", ] @@ -1750,9 +1752,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -1863,7 +1865,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -1956,11 +1958,24 @@ dependencies = [ "polyval", ] +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "google-cloud-auth" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff461519b1a948200f163574be072753bcfb462a323f0eb426629d89872dd685" +checksum = "ac9870b8ea484519aa8601e14886f2efc148ea490034e43d04bef1d246170cc9" dependencies = [ "async-trait", "aws-lc-rs", @@ -1974,12 +1989,12 @@ dependencies = [ "jsonwebtoken", "reqwest 0.13.5", "rustc_version", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "serde", "serde_json", - "sha2 0.11.0", - "thiserror 2.0.20", + "sha2 0.11.1", + "thiserror 2.0.21", "time", "tokio", "url", @@ -1987,9 +2002,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5615cff28ee59cfe52fbb4c11b8b1e77f650296e2ea4f4c2b7757ac6b19e752" +checksum = "fc64ee071c0d22cb68662b454f1bea5424506cafeaddc51260fbc585bac1d29d" dependencies = [ "bytes", "futures", @@ -1997,19 +2012,19 @@ dependencies = [ "google-cloud-wkt", "http 1.5.0", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", ] [[package]] name = "google-cloud-gax-internal" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2766757d877a7a8ac23da9884cb0e3f10ed9b75a0ce59801ce6b19bf9d5819e" +checksum = "e873f7e6e39030c07b6d9697bf3b999a269175c8b433e058f56ae10d129f12f7" dependencies = [ "bytes", "futures", @@ -2017,11 +2032,11 @@ dependencies = [ "google-cloud-gax", "google-cloud-rpc", "google-cloud-wkt", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "opentelemetry", "opentelemetry-semantic-conventions", @@ -2034,7 +2049,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tonic", @@ -2046,9 +2061,9 @@ dependencies = [ [[package]] name = "google-cloud-iam-v1" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f962b40234b1531e6ef73f7558871c96e117231e962086c98804323fb8d2c82" +checksum = "48f0895fa7b7bd70629ae58daaa962677e93a72c251b2f297bc058a035940821" dependencies = [ "async-trait", "bytes", @@ -2064,9 +2079,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c0363c5389ffda2b55cd8a86eef4b19a3481a48467c91dc5d77f626a9572766" +checksum = "e8ea252eee22a4e2187025ea6789c682a0d791a9748ea99de2abec3026a32cb4" dependencies = [ "async-trait", "bytes", @@ -2082,9 +2097,9 @@ dependencies = [ [[package]] name = "google-cloud-lro" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47af3deef75c14a2983c430898d960c765bddbcc9f9188ca0563108e9227cfe7" +checksum = "dfb938b40ede9ef70c24f49c2687d84f289c9e9104ddfece739985e4b4844b8f" dependencies = [ "google-cloud-gax", "google-cloud-gax-internal", @@ -2111,9 +2126,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.18.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "973399251b245c63f1d02d0768772833dcf1372ee358f593fb9159de8fe9c7d4" +checksum = "570eb8d0784118a5f4ae1219fd76ff3b14e90b7b0f877478c02b1374e9bbcb8c" dependencies = [ "async-trait", "base64 0.23.1", @@ -2140,8 +2155,8 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2 0.11.0", - "thiserror 2.0.20", + "sha2 0.11.1", + "thiserror 2.0.21", "tokio", "tokio-stream", "tracing", @@ -2164,16 +2179,16 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7" +checksum = "7f3037907e542cf8b8580aa4cf8ad68568d7ac8e7274519a7d0a121867d434da" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "url", ] @@ -2210,9 +2225,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -2239,8 +2254,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] @@ -2272,8 +2285,8 @@ dependencies = [ "idna", "ipnet", "jni", - "rand 0.10.2", - "thiserror 2.0.20", + "rand 0.10.3", + "thiserror 2.0.21", "tinyvec", "tokio", "tracing", @@ -2292,9 +2305,9 @@ dependencies = [ "jni", "once_cell", "prefix-trie", - "rand 0.10.2", + "rand 0.10.3", "ring", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "url", @@ -2317,11 +2330,11 @@ dependencies = [ "ndk-context", "once_cell", "parking_lot", - "rand 0.10.2", + "rand 0.10.3", "resolv-conf", "smallvec", "system-configuration", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", ] @@ -2455,15 +2468,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -2482,7 +2495,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2506,17 +2519,17 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower-service", "webpki-roots", ] @@ -2527,7 +2540,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2536,17 +2549,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", "http 1.5.0", "http-body 1.1.0", - "hyper 1.11.1", + "httparse", + "hyper 1.12.0", "ipnet", "libc", "percent-encoding", @@ -2567,7 +2581,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2733,9 +2747,9 @@ dependencies = [ [[package]] name = "inout" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" dependencies = [ "hybrid-array", ] @@ -2779,9 +2793,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -2806,9 +2820,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -2818,9 +2832,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -2843,7 +2857,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -2892,9 +2906,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -2903,9 +2917,9 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "11.0.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881733cbc631fc9e472e24447ce32a64bedf2da498d6d8570b08edc87de71f65" +checksum = "e75fe14a82d81e5f5af639997db37d8b96045938a7ac6ab18cdbe1c7467e05e1" dependencies = [ "aws-lc-rs", "base64 0.22.1", @@ -2919,21 +2933,21 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libredox" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -2967,18 +2981,18 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" dependencies = [ "hashbrown 0.17.1", ] [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "macro_magic" @@ -3093,9 +3107,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3190,19 +3204,19 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "rustc_version_runtime", - "rustls 0.23.44", + "rustls 0.23.46", "serde", "serde_bytes", "serde_with", "sha1 0.11.0", - "sha2 0.11.0", + "sha2 0.11.1", "socket2 0.6.5", "stringprep", "strsim", "take_mut", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-util", "typed-builder", "uuid", @@ -3221,27 +3235,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.5", -] - [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -3363,9 +3356,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "cabcc654086861f9e1bc5115258de62ef92c79b20f4615a1b0cc57c4e82027d4" dependencies = [ "libc", ] @@ -3432,29 +3425,29 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.32.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +checksum = "8c1f1a6ac9c875a4941e4c7c085b61e8f4daa326de03b4701768c3491a6ad9ee" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.32.1" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" +checksum = "7a3cf639fefd3e852869114f9925b578da6dd8e99281a2089bb0e773ead46962" [[package]] name = "opentelemetry_sdk" -version = "0.32.1" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +checksum = "372a912db2974d1990207006fd0e3aa0ef3fd4ad3be43c65cfaadf2206866061" dependencies = [ "futures-channel", "futures-executor", @@ -3463,7 +3456,7 @@ dependencies = [ "percent-encoding", "portable-atomic", "rand 0.9.5", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -3607,12 +3600,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkcs8" version = "0.10.2" @@ -3664,11 +3651,13 @@ dependencies = [ "futures", "futures-util", "generic-array", + "globset", "google-cloud-auth", "google-cloud-storage", "hex", "hmac 0.12.1", - "hyper 1.11.1", + "hyper 1.12.0", + "libc", "log", "mockall", "mongodb", @@ -3679,7 +3668,7 @@ dependencies = [ "rand 0.9.5", "redis", "reqwest 0.13.5", - "rustls 0.23.44", + "rustls 0.23.46", "serde", "serde_json", "sha2 0.10.9", @@ -3687,7 +3676,7 @@ dependencies = [ "tempfile", "testcontainers", "testcontainers-modules", - "thiserror 2.0.20", + "thiserror 2.0.21", "tiberius", "time", "tokio", @@ -3702,6 +3691,7 @@ dependencies = [ "typenum", "url", "uuid", + "walkdir", "wiremock", ] @@ -3747,8 +3737,8 @@ dependencies = [ "hmac 0.13.0", "md-5", "memchr", - "rand 0.10.2", - "sha2 0.11.0", + "rand 0.10.3", + "sha2 0.11.1", "stringprep", ] @@ -3774,9 +3764,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -3892,9 +3882,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -3902,9 +3892,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.44", + "rustls 0.23.46", "socket2 0.6.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -3912,22 +3902,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -3935,9 +3925,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -3997,9 +3987,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -4061,9 +4051,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +checksum = "ed5b98ae99461e02895c3a96a25dbe4d852a3d5e5b815d31d5d9e4b5344d99e9" dependencies = [ "arcstr", "async-lock", @@ -4119,7 +4109,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -4169,22 +4159,22 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper 1.12.0", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tower", "tower-http", "tower-service", @@ -4207,12 +4197,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper 1.12.0", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", @@ -4221,7 +4211,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -4229,7 +4219,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-util", "tower", "tower-http", @@ -4298,9 +4288,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.13.2", "errno", @@ -4323,16 +4313,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.44" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.15", + "rustls-webpki 0.103.16", "subtle", "zeroize", ] @@ -4351,9 +4341,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" dependencies = [ "openssl-probe 0.2.1", "rustls-pki-types", @@ -4382,19 +4372,19 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", "once_cell", - "rustls 0.23.44", - "rustls-native-certs 0.8.4", + "rustls 0.23.46", + "rustls-native-certs 0.8.5", "rustls-platform-verifier-android", - "rustls-webpki 0.103.15", + "rustls-webpki 0.103.16", "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", @@ -4403,9 +4393,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" @@ -4419,9 +4409,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -4452,9 +4442,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -4473,9 +4463,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -4592,15 +4582,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", "indexmap 2.14.2", "itoa", "memchr", @@ -4628,14 +4619,14 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "serde_spanned" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" dependencies = [ "serde_core", ] @@ -4654,9 +4645,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" dependencies = [ "base64 0.23.1", "bs58", @@ -4666,7 +4657,7 @@ dependencies = [ "indexmap 2.14.2", "jiff", "schemars 0.9.0", - "schemars 1.2.2", + "schemars 1.2.3", "serde_core", "serde_json", "serde_with_macros", @@ -4675,14 +4666,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" dependencies = [ "darling", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -4726,9 +4717,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -4794,9 +4785,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "siphasher" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" [[package]] name = "slab" @@ -4806,9 +4797,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.1" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" [[package]] name = "socket2" @@ -4917,9 +4908,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -4937,13 +4928,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] @@ -5039,7 +5030,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -5066,11 +5057,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -5086,13 +5077,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -5185,24 +5176,15 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" dependencies = [ "bytes", "libc", @@ -5223,7 +5205,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] @@ -5245,7 +5227,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.10.2", + "rand 0.10.3", "socket2 0.6.5", "tokio", "tokio-util", @@ -5264,11 +5246,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ - "rustls 0.23.44", + "rustls 0.23.46", "tokio", ] @@ -5301,9 +5283,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -5341,18 +5323,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow 1.0.4", ] [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" [[package]] name = "tonic" @@ -5364,20 +5346,20 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "rustls-native-certs 0.8.4", + "rustls-native-certs 0.8.5", "socket2 0.6.5", "sync_wrapper", "tokio", - "tokio-rustls 0.26.5", + "tokio-rustls 0.26.6", "tokio-stream", "tower", "tower-layer", @@ -5470,7 +5452,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -5509,9 +5491,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe" dependencies = [ "js-sys", "opentelemetry", @@ -5600,7 +5582,7 @@ dependencies = [ "dyn-clone", "futures", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "serde", "serde_json", @@ -5627,9 +5609,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-bidi" @@ -5639,15 +5621,15 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" dependencies = [ "tinyvec", ] @@ -5694,14 +5676,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5546be8f5378d5414f83733f5c9a2526f4645829edbc1c41790aeef1b38e8b" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" dependencies = [ "base64 0.23.1", "log", "percent-encoding", - "rustls 0.23.44", + "rustls 0.23.46", "rustls-pki-types", "ureq-proto", "utf8-zero", @@ -5709,9 +5691,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabc3e92916c89c95b20eef7b06b00b066bc217ef9ea3a4ac9bf1a7e35261e10" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" dependencies = [ "base64 0.23.1", "http 1.5.0", @@ -5752,9 +5734,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.26.1" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -5792,9 +5774,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -5834,9 +5816,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -5847,19 +5829,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5867,22 +5850,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -5902,9 +5885,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -5922,18 +5905,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -6173,7 +6156,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "log", "once_cell", @@ -6223,9 +6206,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yoke" @@ -6240,30 +6223,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -6281,21 +6264,21 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] @@ -6341,14 +6324,14 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.9", ] [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index dd57430..3ac5691 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.29" toml = "0.9.10" reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] } anyhow = "1.0.100" -tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] } +tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util", "time"] } async-trait = "0.1.89" tempfile = "3.24.0" hmac = "0.12" @@ -62,6 +62,11 @@ url = "2.5.8" percent-encoding = "2.3.2" bollard = "0.20.0" rustls = "0.23" +globset = "0.4" +walkdir = "2.5" + +[target.'cfg(unix)'.dependencies] +libc = "0.2" [dev-dependencies] tokio = { version = "1", features = ["full"] } diff --git a/databases.json b/databases.json index 3b66894..f480f42 100644 --- a/databases.json +++ b/databases.json @@ -131,6 +131,38 @@ "volume_name": "databases_sqlite-data", "generated_id": "16706126-ff7e-4c97-8c83-0adeff214690", "container_name": "db-sqlite" + }, + { + "name": "Files - Archive", + "type": "files", + "path": "/data/files-archive", + "generated_id": "64aeaf80-e03b-4503-a5c1-00a1b829f0e5", + "options": { + "method": "archive", + "exclude": ["node_modules", "*.tmp", "/cache"], + "one_file_system": false + } + }, + { + "name": "Files - Snapshots", + "type": "files", + "path": "/data/files-snapshot", + "generated_id": "9093ad85-3f67-489f-9759-aa369572b30d", + "options": { + "method": "snapshot", + "exclude": ["node_modules", "*.log", "/build"], + "one_file_system": true + } + }, + { + "name": "Files - Sync", + "type": "files", + "path": "/data/files-sync", + "generated_id": "353ecc44-7c75-40e7-803e-ac07683c2c05", + "options": { + "method": "sync", + "exclude": ["*.tmp", "/cache"] + } } ] } diff --git a/docker-compose.yml b/docker-compose.yml index 4b18ba7..660a417 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,12 +16,15 @@ services: - databases_sqlite-data:/sqlite-data/workspace/data - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data # - /bigdisk:/scratch + - ./test-files/archive:/data/files-archive + - ./test-files/snapshot:/data/files-snapshot + - ./test-files/sync:/data/files-sync environment: APP_ENV: development LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiN2ZjMGViODMtYTY0YS00NzI3LTg5ZjktNzFhZWJmZDQ3NmEyIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMzM1YjcwZjYtMjQxNC00OGZiLTkxZDQtZjQzZjJjNTJlYzUzIiwibWFzdGVyS2V5QjY0IjoiOEdScEFNMzhDeE1seTAzeTFIbHpBSTdZbnRDYVhuYTV1bDBIYkxKSG81ND0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #RETRY_ATTEMPTS: 3 diff --git a/docker/Dockerfile b/docker/Dockerfile index 7634388..86c5bed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,6 +58,17 @@ RUN ARCH=$(dpkg --print-architecture) \ && dpkg -i /tmp/rclone.deb \ && rm /tmp/rclone.deb +# ========================= +# restic (Snapshots mode) +# ========================= +ARG RESTIC_VERSION=0.19.1 +RUN ARCH=$(dpkg --print-architecture) \ + && curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \ + && bzip2 -d /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm /tmp/restic \ + && restic version + ARG TARGETARCH # ========================= @@ -146,6 +157,7 @@ RUN apt-get update && apt-get install -y \ libncurses6 \ zlib1g \ curl \ + bzip2 \ mariadb-client \ sqlite3 \ redis-tools \ @@ -159,6 +171,17 @@ RUN ARCH=$(dpkg --print-architecture) \ && dpkg -i /tmp/rclone.deb \ && rm /tmp/rclone.deb +# ========================= +# restic (Snapshots mode) +# ========================= +ARG RESTIC_VERSION=0.19.1 +RUN ARCH=$(dpkg --print-architecture) \ + && curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \ + && bzip2 -d /tmp/restic.bz2 \ + && install -m 0755 /tmp/restic /usr/local/bin/restic \ + && rm /tmp/restic \ + && restic version + ENV DOTNET_ROOT=/usr/local/dotnet RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \ && chmod +x /tmp/dotnet-install.sh \ diff --git a/helm/README.md b/helm/README.md index c41f381..b69522a 100644 --- a/helm/README.md +++ b/helm/README.md @@ -57,3 +57,23 @@ kubectl logs portabase-agent-6f7d4f5c6b-abc12 ``` bash helm uninstall portabase-agent ``` + +## Mount a directory for a `files` source + +A `files` source backs up a directory the agent can see. Mount it into the pod +with `extraVolumes` and `extraVolumeMounts`: + +```yaml +extraVolumes: + - name: shared-files + hostPath: + path: /srv/files + type: Directory + +extraVolumeMounts: + - name: shared-files + mountPath: /data/files +``` + +Mount it read-write: restores write into it. The `mountPath` (here `/data/files`) +is the `path` to enter for the source in Portabase. diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 97b1c41..d38a231 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -44,6 +44,9 @@ spec: # uses ConfigMap content {{- end }} {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} volumes: {{- if .Values.volume.configFile.enabled }} {{- if .Values.volume.configFile.hostPath }} @@ -59,4 +62,7 @@ spec: - key: config.json path: config.json {{- end }} + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} {{- end }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index dad96c4..46b3130 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -42,6 +42,17 @@ volume: ] } +# Extra volumes for the agent pod, e.g. a host directory used by a `files` +# source. Mount it read-write: restores write into it. +extraVolumes: [] +# - name: shared-files +# hostPath: +# path: /srv/files +# type: Directory + +extraVolumeMounts: [] +# - name: shared-files +# mountPath: /data/files # use this path as the source `path` network: hostAliases: diff --git a/justfile b/justfile index 3063c15..390e720 100644 --- a/justfile +++ b/justfile @@ -77,3 +77,10 @@ test: docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync" echo "Down volumes tests databases" docker compose -f docker-compose.test.yml down --volumes + +seed-files: + for d in archive snapshot sync; do mkdir -p test-files/$d && echo one > test-files/$d/a.txt && echo two > test-files/$d/b.txt; done + +seed-files-1gb: + echo "Seeding files sources (1GB each)..." + bash ./scripts/files/seed-1gb.sh diff --git a/scripts/README.md b/scripts/README.md index f94a41e..4b4b58a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -11,6 +11,7 @@ make seed-postgres-1gb make seed-all make seed-sqlite make seed-sqlite SEED=big +just seed-files-1gb ``` ## Verify commands diff --git a/scripts/files/seed-1gb.sh b/scripts/files/seed-1gb.sh new file mode 100755 index 0000000..9a4b317 --- /dev/null +++ b/scripts/files/seed-1gb.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +SIZE_MB=${SIZE_MB:-1024} +ROOT=${ROOT:-./test-files} +TARGETS=${TARGETS:-archive snapshot sync} + +bytes() { echo $(( SIZE_MB * $1 / 100 * 1024 * 1024 )); } + +random_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" /dev/urandom > "$1" +} + +random_tree() { + mkdir -p "$1" + head -c "$2" /dev/urandom | split -b "$3" -a 4 -d - "$1/part-" +} + +text_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" < <(base64 -w 76 /dev/urandom) > "$1" +} + +log_file() { + mkdir -p "$(dirname "$1")" + head -c "$2" < <(yes "2026-10-18T12:00:00Z INFO GET /api/backups 200 12ms") > "$1" +} + +seed() { + local dir=$1 + rm -rf "$dir"/{media,projects,docs,logs,cache,build,scratch} "$dir/latest.bin" + mkdir -p "$dir" + + for i in 1 2 3 4 5 6; do + random_file "$dir/media/video-$i.bin" "$(bytes 10)" + done + for i in $(seq -w 1 20); do + random_tree "$dir/projects/p$i/src" "$(bytes 1)" 128K + done + for i in 1 2 3 4; do + text_file "$dir/docs/report-$i.txt" "$(bytes 2)" + done + log_file "$dir/logs/app.log" "$(bytes 4)" + random_tree "$dir/projects/p01/node_modules/pkg" "$(bytes 4)" 32K + random_tree "$dir/cache" "$(bytes 2)" 1M + random_tree "$dir/build" "$(bytes 1)" 1M + random_file "$dir/scratch/session.tmp" "$(bytes 1)" + ln -s media/video-1.bin "$dir/latest.bin" + + echo "$dir: $(du -sh "$dir" | cut -f1), $(find "$dir" -type f | wc -l) files" +} + +for target in $TARGETS; do + seed "${ROOT:?}/$target" +done diff --git a/src/core/agent.rs b/src/core/agent.rs index e6b9a48..fb1bb17 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -4,7 +4,7 @@ use crate::core::context::Context; use crate::services::backup::BackupService; use crate::services::config::{ConfigService, DatabaseConfig}; use crate::services::cron::CronService; -use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache}; +use crate::services::dashboard_config::{collect_configs, load_cache, local_only_ids, merge, persist_cache}; use crate::services::restore::RestoreService; use crate::services::status::StatusService; use crate::settings::CONFIG; @@ -51,9 +51,10 @@ impl Agent { pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box> { let local = self.config_service.load_optional(None); + let local_ids = local_only_ids(&local.databases, &self.dashboard_cache); let merged_in = merge(&local.databases, &self.dashboard_cache); - let ping_result = self.status_service.ping(&merged_in.databases).await?; + let ping_result = self.status_service.ping(&merged_in.databases, &local_ids).await?; self.dashboard_cache = collect_configs(&ping_result); if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) { @@ -93,6 +94,7 @@ impl Agent { method.clone(), &db.storages, db.encrypt, + db.data.backup.engine.as_deref().unwrap_or("archive"), ) .await; } else if db.data.restore.action { diff --git a/src/core/context.rs b/src/core/context.rs index c37823d..d1e7197 100644 --- a/src/core/context.rs +++ b/src/core/context.rs @@ -37,7 +37,8 @@ impl Context { }; let server_url = format!("{}/api", edge_key.server_url); - let api_client = ApiClient::new(server_url); + let api_client = ApiClient::new(server_url) + .with_agent_auth(&edge_key.agent_id, &edge_key.master_key_b64); Context { edge_key: edge_key, diff --git a/src/domain/factory.rs b/src/domain/factory.rs index d4d1975..c755522 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -1,4 +1,5 @@ use crate::domain::docker_volume::database::DockerVolumeDatabase; +use crate::domain::files::database::FilesDatabase; use crate::domain::mongodb::database::MongoDatabase; use crate::domain::mysql::database::MySQLDatabase; use crate::domain::postgres::cluster::database::PostgresClusterDatabase; @@ -43,6 +44,7 @@ impl DatabaseFactory { DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)), DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)), DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)), + DbType::Files => Arc::new(FilesDatabase::new(cfg)), } } @@ -62,6 +64,7 @@ impl DatabaseFactory { DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)), DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)), DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)), + DbType::Files => Arc::new(FilesDatabase::new(cfg)), } } } diff --git a/src/domain/files/backup.rs b/src/domain/files/backup.rs new file mode 100644 index 0000000..153afbd --- /dev/null +++ b/src/domain/files/backup.rs @@ -0,0 +1,119 @@ +use super::matcher::{ExcludeMatcher, one_file_system, source_root}; +use super::platform::dev; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use flate2::Compression; +use flate2::write::GzEncoder; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use walkdir::WalkDir; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + logger: Arc, +) -> Result { + tokio::task::spawn_blocking(move || archive(&cfg, &backup_dir, &logger)).await? +} + +fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Result { + let root = source_root(cfg)?; + let backup_real = std::fs::canonicalize(backup_dir) + .with_context(|| format!("cannot read backup directory {}", backup_dir.display()))?; + let matcher = ExcludeMatcher::from_config(cfg)?; + let one_file_system = one_file_system(cfg); + let root_dev = dev(&std::fs::metadata(&root) + .with_context(|| format!("cannot read source directory {}", root.display()))?); + + let file_name = format!("{}.tar.gz", cfg.generated_id); + let out = backup_dir.join(&file_name); + let out_real = backup_real.join(&file_name); + let gz = GzEncoder::new(File::create(&out)?, Compression::default()); + let mut tar = tar::Builder::new(gz); + tar.follow_symlinks(false); + + logger.log("info", format!("Archiving {}", root.display())); + let mut files = 0u64; + + let mut unreadable = 0usize; + let mut walker = WalkDir::new(&root) + .follow_links(false) + .min_depth(1) + .into_iter(); + + while let Some(entry) = walker.next() { + let entry = match entry { + Ok(e) => e, + Err(e) => { + logger.log("warn", format!("Skipping unreadable entry: {e}")); + unreadable += 1; + continue; + } + }; + let rel = entry.path().strip_prefix(&root)?.to_path_buf(); + let ft = entry.file_type(); + + let mut skip = + matcher.is_excluded(&rel) || entry.path() == backup_real || entry.path() == out_real; + if !skip && one_file_system && ft.is_dir() { + match entry.metadata() { + Ok(m) => skip = dev(&m) != root_dev, + Err(e) => { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + skip = true; + } + } + } + if skip { + if ft.is_dir() { + walker.skip_current_dir(); + } + continue; + } + + if ft.is_file() { + let mut f = match File::open(entry.path()) { + Ok(f) => f, + Err(e) => { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + continue; + } + }; + let meta = f.metadata()?; + let size = meta.len(); + let mut header = tar::Header::new_gnu(); + header.set_metadata(&meta); + + let reader = (&mut f).take(size).chain(std::io::repeat(0)).take(size); + tar.append_data(&mut header, &rel, reader) + .with_context(|| format!("failed to archive {}", rel.display()))?; + files += 1; + } else if ft.is_dir() || ft.is_symlink() { + + if let Err(e) = tar.append_path_with_name(entry.path(), &rel) { + logger.log("warn", format!("Skipping {}: {e}", rel.display())); + unreadable += 1; + } + } else { + logger.log("warn", format!("Skipping special file {}", rel.display())); + } + } + + if unreadable > 0 { + let noun = if unreadable == 1 { "entry" } else { "entries" }; + bail!( + "{unreadable} {noun} could not be read; backup aborted so a mirror restore cannot delete unarchived data (fix permissions or add an exclude)" + ); + } + tar.into_inner()?.finish()?; + logger.log( + "info", + format!("Archived {files} file(s) into {}", out.display()), + ); + Ok(out) +} diff --git a/src/domain/files/database.rs b/src/domain/files/database.rs new file mode 100644 index 0000000..a205886 --- /dev/null +++ b/src/domain/files/database.rs @@ -0,0 +1,45 @@ +use anyhow::Result; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use super::{backup, ping, restore}; +use crate::domain::factory::Database; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct FilesDatabase { + cfg: DatabaseConfig, +} + +impl FilesDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for FilesDatabase { + fn file_extension(&self) -> &'static str { + ".tar.gz" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path, logger: Arc) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } + + async fn restore(&self, file: &Path, logger: Arc) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } +} diff --git a/src/domain/files/matcher.rs b/src/domain/files/matcher.rs new file mode 100644 index 0000000..88f96d7 --- /dev/null +++ b/src/domain/files/matcher.rs @@ -0,0 +1,98 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use std::path::{Path, PathBuf}; + +pub struct ExcludeMatcher { + set: GlobSet, + literal: Vec, +} + +impl ExcludeMatcher { + pub fn new(patterns: &[String]) -> Result { + let mut builder = GlobSetBuilder::new(); + for raw in patterns { + let pattern = raw.trim(); + if pattern.is_empty() { + continue; + } + let glob = match pattern.strip_prefix('/') { + Some(anchored) => anchored.trim_end_matches('/').to_string(), + None => format!("**/{}", pattern.trim_end_matches('/')), + }; + builder.add( + GlobBuilder::new(&glob) + .literal_separator(true) + .build() + .with_context(|| format!("invalid exclude pattern '{pattern}'"))?, + ); + } + Ok(Self { + set: builder.build()?, + literal: Vec::new(), + }) + } + + pub fn from_config(cfg: &DatabaseConfig) -> Result { + Self::new(&exclude_patterns(cfg)) + } + + pub fn exclude_path(&mut self, relative: PathBuf) { + self.literal.push(relative); + } + + pub fn is_excluded(&self, relative: &Path) -> bool { + self.set.is_match(relative) || self.literal.iter().any(|p| p == relative) + } +} + +pub fn exclude_patterns(cfg: &DatabaseConfig) -> Vec { + cfg.options + .get("exclude") + .and_then(|v| v.as_array()) + .map(|items| { + items + .iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect() + }) + .unwrap_or_default() +} + +pub fn one_file_system(cfg: &DatabaseConfig) -> bool { + cfg.options + .get("one_file_system") + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + +pub fn source_root(cfg: &DatabaseConfig) -> Result { + let root = std::fs::canonicalize(&cfg.path) + .with_context(|| format!("cannot read source directory {}", cfg.path))?; + if !root.is_dir() { + bail!("source path {} is not a directory", root.display()); + } + Ok(root) +} + +pub(crate) fn bracket_negation(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + let mut in_class = false; + while let Some(c) = chars.next() { + out.push(c); + match c { + '\\' => out.extend(chars.next()), + '[' if !in_class => { + in_class = true; + if chars.as_str().starts_with('!') { + chars.next(); + out.push('^'); + } + } + ']' => in_class = false, + _ => {} + } + } + out +} diff --git a/src/domain/files/mod.rs b/src/domain/files/mod.rs new file mode 100644 index 0000000..c1fd77b --- /dev/null +++ b/src/domain/files/mod.rs @@ -0,0 +1,8 @@ +pub mod backup; +pub mod database; +pub mod matcher; +pub mod ping; +pub mod platform; +pub mod restic; +pub mod restore; +pub mod sync; diff --git a/src/domain/files/ping.rs b/src/domain/files/ping.rs new file mode 100644 index 0000000..2af3acf --- /dev/null +++ b/src/domain/files/ping.rs @@ -0,0 +1,15 @@ +use crate::services::config::DatabaseConfig; +use anyhow::Result; +use std::path::Path; +use tokio::time::{Duration, timeout}; + +pub async fn run(cfg: DatabaseConfig) -> Result { + let check = tokio::task::spawn_blocking(move || { + let root = Path::new(&cfg.path); + root.is_dir() && std::fs::read_dir(root).is_ok() + }); + Ok(matches!( + timeout(Duration::from_secs(10), check).await, + Ok(Ok(true)) + )) +} diff --git a/src/domain/files/platform.rs b/src/domain/files/platform.rs new file mode 100644 index 0000000..60fca91 --- /dev/null +++ b/src/domain/files/platform.rs @@ -0,0 +1,25 @@ +//! Unix-only metadata behind `cfg`, with inert fallbacks so the Windows release +//! build compiles. + +/// Device id, used to spot mount points. Off unix every entry reports 0, so no +/// directory is ever treated as a mount point. +#[cfg(unix)] +pub fn dev(meta: &std::fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + meta.dev() +} + +#[cfg(not(unix))] +pub fn dev(_meta: &std::fs::Metadata) -> u64 { + 0 +} + +#[cfg(unix)] +pub fn is_root() -> bool { + unsafe { libc::geteuid() == 0 } +} + +#[cfg(not(unix))] +pub fn is_root() -> bool { + false +} diff --git a/src/domain/files/restic/backup.rs b/src/domain/files/restic/backup.rs new file mode 100644 index 0000000..58b10e5 --- /dev/null +++ b/src/domain/files/restic/backup.rs @@ -0,0 +1,106 @@ +use super::excludes::backup_excludes; +use super::json::{BackupSummary, Snapshot, files_removed}; +use super::repo::{ResticRepo, short}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system, source_root}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; + +pub async fn snapshot( + repo: &ResticRepo, + cfg: &DatabaseConfig, + backup_storage_id: &str, + logger: &JobLogger, +) -> Result { + let root = source_root(cfg)?; + repo.ensure_initialized(logger).await?; + + let host = format!("portabase-{}", cfg.generated_id); + let mut args: Vec = vec![ + "backup".into(), + root.to_string_lossy().into_owned(), + "--json".into(), + "--host".into(), + host.clone(), + "--tag".into(), + "portabase".into(), + "--tag".into(), + format!("bs:{backup_storage_id}"), + ]; + for pattern in backup_excludes(&root, &exclude_patterns(cfg))? { + args.push("--exclude".into()); + args.push(pattern); + } + if one_file_system(cfg) { + args.push("--one-file-system".into()); + } + + logger.log("info", format!("Snapshotting {} into {}", root.display(), repo.repository())); + let run = repo.run(&args, logger).await?; + for error in &run.errors { + logger.log("warn", format!("Unreadable: {error}")); + } + let summary: Option = run + .summary + .clone() + .map(serde_json::from_value) + .transpose() + .context("unexpected restic backup summary")?; + + match (run.code, summary) { + (0, Some(mut s)) => { + s.files_removed = match removed(repo, &host, &s, logger).await { + Ok(removed) => removed, + Err(e) => { + logger.log("warn", format!("Could not count removed files: {e:#}")); + None + } + }; + let removed = s.files_removed.map(|n| format!(", {n} removed")).unwrap_or_default(); + logger.log( + "info", + format!( + "Snapshot {}: {} new, {} changed, {} unmodified{removed} file(s); {} byte(s) scanned, {} byte(s) of new data", + short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.total_bytes_processed, s.data_added_packed + ), + ); + Ok(s) + } + (0, None) => bail!("restic backup returned no summary"), + (3, summary) => { + let mut fate = "snapshot discarded"; + if let Some(s) = summary { + if repo.is_append_only() { + logger.log( + "warn", + format!("Incomplete snapshot {} left for the dashboard to forget", short(&s.snapshot_id)), + ); + fate = "snapshot not restorable (the dashboard forgets it)"; + } else { + let forget = repo.run(["forget", s.snapshot_id.as_str()], logger).await?; + if forget.code != 0 { + logger.log( + "error", + format!("Could not forget incomplete snapshot {}: {}", s.snapshot_id, forget.error("forget")), + ); + } + } + } + bail!( + "{} file(s) could not be read; {fate} so a mirror restore cannot delete them (fix permissions or add an exclude)", + run.errors.len() + ) + } + _ => Err(run.error("backup")), + } +} + +async fn removed(repo: &ResticRepo, host: &str, s: &BackupSummary, logger: &JobLogger) -> Result> { + let listed = repo.run(["snapshots", "--json", "--host", host], logger).await?; + if listed.code != 0 { + return Err(listed.error("snapshots")); + } + let snapshots: Vec = + serde_json::from_str(listed.stdout.trim()).context("unexpected `restic snapshots` output")?; + Ok(files_removed(&snapshots, &s.snapshot_id, s.files_changed, s.files_unmodified)) +} diff --git a/src/domain/files/restic/excludes.rs b/src/domain/files/restic/excludes.rs new file mode 100644 index 0000000..b894b84 --- /dev/null +++ b/src/domain/files/restic/excludes.rs @@ -0,0 +1,50 @@ +use crate::domain::files::matcher::bracket_negation; +use anyhow::{Result, bail}; +use std::path::Path; + +pub fn glob_escape(literal: &str) -> String { + let mut out = String::with_capacity(literal.len()); + for c in literal.chars() { + if matches!(c, '\\' | '*' | '?' | '[' | ']') { + out.push('\\'); + } + out.push(c); + } + out +} + +fn normalized(patterns: &[String]) -> Result> { + let mut out = Vec::new(); + for raw in patterns { + let pattern = raw.trim(); + if pattern.contains(['{', '}']) { + bail!("exclude pattern '{pattern}': brace patterns are not supported in Snapshots mode"); + } + let (anchored, rest) = match pattern.strip_prefix('/') { + Some(rest) => (true, rest), + None => (false, pattern), + }; + let rest = rest.trim_end_matches('/'); + if !rest.is_empty() { + out.push((anchored, bracket_negation(rest))); + } + } + Ok(out) +} + +pub fn backup_excludes(root: &Path, patterns: &[String]) -> Result> { + let root = glob_escape(root.to_string_lossy().trim_end_matches('/')); + Ok(normalized(patterns)? + .into_iter() + .map(|(anchored, p)| { + if anchored { format!("{root}/{p}") } else { format!("{root}/**/{p}") } + }) + .collect()) +} + +pub fn restore_excludes(patterns: &[String]) -> Result> { + Ok(normalized(patterns)? + .into_iter() + .map(|(anchored, p)| if anchored { format!("/{p}") } else { format!("/**/{p}") }) + .collect()) +} diff --git a/src/domain/files/restic/json.rs b/src/domain/files/restic/json.rs new file mode 100644 index 0000000..dcd53c0 --- /dev/null +++ b/src/domain/files/restic/json.rs @@ -0,0 +1,79 @@ +use serde::Deserialize; +use serde_json::{Value, json}; + +#[derive(Debug, PartialEq)] +pub enum Line { + Status { percent_done: f64 }, + Summary(Value), + Error { message: String, item: Option }, + ExitError { code: i32, message: String }, + Text(String), +} + +pub fn parse(line: &str) -> Line { + let Ok(value) = serde_json::from_str::(line) else { + return Line::Text(line.to_string()); + }; + match value.get("message_type").and_then(Value::as_str) { + Some("status") => Line::Status { + percent_done: value["percent_done"].as_f64().unwrap_or(0.0), + }, + Some("summary") => Line::Summary(value), + Some("error") => Line::Error { + message: value["error"]["message"].as_str().unwrap_or("unknown error").to_string(), + item: value["item"].as_str().map(str::to_string), + }, + Some("exit_error") => Line::ExitError { + code: value["code"].as_i64().unwrap_or(1) as i32, + message: value["message"].as_str().unwrap_or_default().to_string(), + }, + _ => Line::Text(line.to_string()), + } +} + +#[derive(Debug, Deserialize)] +pub struct BackupSummary { + pub snapshot_id: String, + pub data_added_packed: u64, + pub total_bytes_processed: u64, + pub files_new: u64, + pub files_changed: u64, + pub files_unmodified: u64, + #[serde(skip)] + pub files_removed: Option, +} + +impl BackupSummary { + pub fn report(&self) -> Value { + json!({ + "filesNew": self.files_new, + "filesChanged": self.files_changed, + "filesUnmodified": self.files_unmodified, + "filesRemoved": self.files_removed, + }) + } +} + +#[derive(Debug, Deserialize)] +pub struct Snapshot { + pub id: String, + pub paths: Vec, + #[serde(default)] + pub parent: Option, + #[serde(default)] + pub summary: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SnapshotSummary { + pub total_files_processed: u64, +} + +pub fn files_removed(snapshots: &[Snapshot], snapshot_id: &str, changed: u64, unmodified: u64) -> Option { + let find = |id: &str| snapshots.iter().find(|s| s.id == id); + let Some(parent) = find(snapshot_id)?.parent.as_deref() else { + return Some(0); + }; + let total = find(parent)?.summary.as_ref()?.total_files_processed; + Some(total.saturating_sub(changed + unmodified)) +} diff --git a/src/domain/files/restic/mod.rs b/src/domain/files/restic/mod.rs new file mode 100644 index 0000000..12fbd55 --- /dev/null +++ b/src/domain/files/restic/mod.rs @@ -0,0 +1,6 @@ +pub mod backup; +pub mod excludes; +pub mod json; +pub mod password; +pub mod repo; +pub mod restore; diff --git a/src/domain/files/restic/password.rs b/src/domain/files/restic/password.rs new file mode 100644 index 0000000..a491fb5 --- /dev/null +++ b/src/domain/files/restic/password.rs @@ -0,0 +1,7 @@ +use crate::utils::edge_key::derive_key; +use anyhow::Result; + +pub fn derive_password(master_key_b64: &str, generated_id: &str) -> Result { + let info = format!("portabase/restic/v1/{generated_id}"); + Ok(hex::encode(derive_key(master_key_b64, info.as_bytes())?)) +} diff --git a/src/domain/files/restic/repo.rs b/src/domain/files/restic/repo.rs new file mode 100644 index 0000000..ff09e2f --- /dev/null +++ b/src/domain/files/restic/repo.rs @@ -0,0 +1,227 @@ +use super::json::{self, Line}; +use super::password::derive_password; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::storage::providers::local::{LOCAL_STORAGE_USER, local_storage_password}; +use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; +use crate::services::storage::providers::rclone::target::{RcloneTarget, rclone_target}; +use crate::settings::CONFIG; +use crate::utils::edge_key::EdgeKey; +use crate::utils::file::folder_or_default; +use anyhow::{Context, Result, anyhow}; +use serde_json::Value; +use std::ffi::{OsStr, OsString}; +use std::path::PathBuf; +use std::process::Stdio; +use tempfile::NamedTempFile; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +pub struct ResticRepo { + repository: String, + password: String, + cache_dir: Option, + config: Option, + rest_auth: Option<(String, String)>, + _target: Option, +} + +pub struct ResticRun { + pub code: i32, + pub stdout: String, + pub summary: Option, + pub errors: Vec, + pub stderr_tail: String, +} + +impl ResticRun { + pub fn error(&self, op: &str) -> anyhow::Error { + match self.code { + 10 => anyhow!("restic {op}: repository does not exist"), + 11 => anyhow!("restic {op}: repository is locked by another operation (waited 5 minutes)"), + 12 => anyhow!("restic {op}: wrong repository password (did the master key change?)"), + code => anyhow!("restic {op} failed (exit {code}): {}", self.stderr_tail.trim()), + } + } +} + +pub fn is_snapshot_id(id: &str) -> bool { + id.len() == 64 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +pub fn short(id: &str) -> &str { + id.get(..8).unwrap_or(id) +} + +fn usable_cache_dir(cache_dir: PathBuf) -> Option { + match std::fs::create_dir_all(&cache_dir) { + Ok(()) => Some(cache_dir), + Err(e) => { + tracing::warn!("restic cache dir {} unusable ({e}); running without cache", cache_dir.display()); + None + } + } +} + +impl ResticRepo { + pub fn open(storage: &DatabaseStorage, generated_id: &str, edge_key: &EdgeKey) -> Result { + let password = derive_password(&edge_key.master_key_b64, generated_id)?; + let cache_dir = PathBuf::from(&CONFIG.data_path).join("cache/restic"); + if storage.provider == "local" { + return Ok(Self::rest( + &format!("{}/storage/restic", edge_key.server_url.trim_end_matches('/')), + generated_id, + &password, + &local_storage_password(&edge_key.master_key_b64)?, + cache_dir, + )); + } + let target = rclone_target(storage)?; + let mut repo = Self::new( + &target.config_text, + &target.remote_name, + &target.base_path, + folder_or_default(storage.folder_name.as_deref()), + generated_id, + &password, + cache_dir, + )?; + repo._target = Some(target); + Ok(repo) + } + + pub fn new( + config_text: &str, + remote_name: &str, + base_path: &str, + folder: &str, + generated_id: &str, + password: &str, + cache_dir: PathBuf, + ) -> Result { + let path = remote_target(remote_name, base_path, &format!("{folder}/restic/{generated_id}")); + Ok(Self { + repository: format!("rclone:{path}"), + password: password.to_string(), + cache_dir: usable_cache_dir(cache_dir), + config: Some(write_config(config_text)?), + rest_auth: None, + _target: None, + }) + } + + pub fn rest(base_url: &str, generated_id: &str, password: &str, server_password: &str, cache_dir: PathBuf) -> Self { + Self { + repository: format!("rest:{}/{generated_id}/", base_url.trim_end_matches('/')), + password: password.to_string(), + cache_dir: usable_cache_dir(cache_dir), + config: None, + rest_auth: Some((LOCAL_STORAGE_USER.to_string(), server_password.to_string())), + _target: None, + } + } + + pub fn is_append_only(&self) -> bool { + self.rest_auth.is_some() + } + + pub fn repository(&self) -> &str { + &self.repository + } + + pub async fn run(&self, args: I, logger: &JobLogger) -> Result + where + I: IntoIterator, + S: AsRef, + { + let args: Vec = args.into_iter().map(|a| a.as_ref().to_owned()).collect(); + let shown: Vec = args.iter().map(|a| a.to_string_lossy().into_owned()).collect(); + logger.log("debug", format!("restic {}", shown.join(" "))); + + let mut cmd = Command::new("restic"); + cmd.args(&args).args(["--retry-lock", "5m"]); + match &self.cache_dir { + Some(dir) => cmd.env("RESTIC_CACHE_DIR", dir), + None => cmd.arg("--no-cache"), + }; + if let Some(config) = &self.config { + cmd.env("RCLONE_CONFIG", config.path()); + } + if let Some((user, pass)) = &self.rest_auth { + cmd.env("RESTIC_REST_USERNAME", user).env("RESTIC_REST_PASSWORD", pass); + } + let mut child = cmd + .env("RESTIC_REPOSITORY", &self.repository) + .env("RESTIC_PASSWORD", &self.password) + .env("RESTIC_PROGRESS_FPS", "0.2") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => anyhow!("restic binary not found"), + _ => anyhow!(e).context("failed to start restic"), + })?; + + let stderr = child.stderr.take().context("restic stderr unavailable")?; + let stderr_task = tokio::spawn(async move { + let mut errors = Vec::new(); + let mut tail = Vec::new(); + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + match json::parse(&line) { + Line::Error { message, item } => errors.push(match item { + Some(item) => format!("{item}: {message}"), + None => message, + }), + Line::ExitError { message, .. } => tail.push(message), + Line::Text(text) if !text.trim().is_empty() => tail.push(text), + _ => {} + } + } + (errors, tail) + }); + + let mut run = ResticRun { code: -1, stdout: String::new(), summary: None, errors: Vec::new(), stderr_tail: String::new() }; + let stdout = child.stdout.take().context("restic stdout unavailable")?; + let mut lines = BufReader::new(stdout).lines(); + let mut next_pct = 10u32; + while let Some(line) = lines.next_line().await.context("failed to read restic output")? { + match json::parse(&line) { + Line::Status { percent_done } => { + let pct = (percent_done * 100.0) as u32; + if pct >= next_pct { + logger.log("info", format!("restic: {pct}% done")); + next_pct = pct / 10 * 10 + 10; + } + } + Line::Summary(value) => run.summary = Some(value), + _ => { + run.stdout.push_str(&line); + run.stdout.push('\n'); + } + } + } + + let status = child.wait().await.context("failed to wait for restic")?; + let (errors, tail) = stderr_task.await.unwrap_or_default(); + run.code = status.code().unwrap_or(-1); + run.errors = errors; + run.stderr_tail = tail[tail.len().saturating_sub(10)..].join("\n"); + Ok(run) + } + + pub async fn ensure_initialized(&self, logger: &JobLogger) -> Result<()> { + let cat = self.run(["cat", "config"], logger).await?; + match cat.code { + 0 => Ok(()), + 10 => { + logger.log("info", format!("Initializing restic repository {}", self.repository)); + let init = self.run(["init"], logger).await?; + if init.code == 0 { Ok(()) } else { Err(init.error("init")) } + } + _ => Err(cat.error("cat config")), + } + } +} diff --git a/src/domain/files/restic/restore.rs b/src/domain/files/restic/restore.rs new file mode 100644 index 0000000..4be279f --- /dev/null +++ b/src/domain/files/restic/restore.rs @@ -0,0 +1,93 @@ +use super::excludes::{glob_escape, restore_excludes}; +use super::json::Snapshot; +use super::repo::{ResticRepo, is_snapshot_id, short}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system}; +use crate::domain::files::platform::dev; +use crate::domain::files::restore::ensure_restorable; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use std::path::Path; +use walkdir::WalkDir; + +pub async fn restore( + repo: &ResticRepo, + cfg: &DatabaseConfig, + snapshot_id: &str, + logger: &JobLogger, +) -> Result<()> { + if !is_snapshot_id(snapshot_id) { + bail!("invalid snapshot id in the restore payload"); + } + let target = ensure_restorable(Path::new(&cfg.path))?; + + let listed = repo.run(["snapshots", snapshot_id, "--json"], logger).await?; + if listed.code != 0 { + return Err(listed.error("snapshots")); + } + let snapshots: Vec = serde_json::from_str(listed.stdout.trim()) + .context("unexpected `restic snapshots` output")?; + let Some(original) = snapshots.first().and_then(|s| s.paths.first()) else { + bail!("snapshot {} not found in {}", short(snapshot_id), repo.repository()); + }; + + let mut excludes = restore_excludes(&exclude_patterns(cfg))?; + if one_file_system(cfg) { + excludes.extend(mount_points(&target).into_iter().map(|rel| format!("/{}", glob_escape(&rel)))); + } + let mut args: Vec = vec![ + "restore".into(), + format!("{snapshot_id}:{original}"), + "--target".into(), + target.to_string_lossy().into_owned(), + "--delete".into(), + "--json".into(), + ]; + for pattern in excludes { + args.push("--exclude".into()); + args.push(pattern); + } + + logger.log( + "info", + format!("Restoring snapshot {} ({original}) into {}", short(snapshot_id), target.display()), + ); + let run = repo.run(&args, logger).await?; + for error in &run.errors { + logger.log("warn", format!("Restore error: {error}")); + } + if run.code != 0 { + return Err(run.error("restore")); + } + if let Some(s) = &run.summary { + logger.log( + "info", + format!( + "Restored {} file(s), deleted {} file(s)", + s["files_restored"].as_u64().unwrap_or(0), + s["files_deleted"].as_u64().unwrap_or(0) + ), + ); + } + Ok(()) +} + +fn mount_points(root: &Path) -> Vec { + let Ok(meta) = std::fs::metadata(root) else { return Vec::new() }; + let root_dev = dev(&meta); + let mut out = Vec::new(); + let mut walker = WalkDir::new(root).min_depth(1).follow_links(false).into_iter(); + while let Some(entry) = walker.next() { + let Ok(entry) = entry else { continue }; + if !entry.file_type().is_dir() { + continue; + } + if matches!(entry.metadata(), Ok(m) if dev(&m) != root_dev) { + if let Ok(rel) = entry.path().strip_prefix(root) { + out.push(rel.to_string_lossy().into_owned()); + } + walker.skip_current_dir(); + } + } + out +} diff --git a/src/domain/files/restore.rs b/src/domain/files/restore.rs new file mode 100644 index 0000000..0a24c48 --- /dev/null +++ b/src/domain/files/restore.rs @@ -0,0 +1,212 @@ +use super::matcher::{ExcludeMatcher, one_file_system}; +use super::platform::{dev, is_root}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result, bail}; +use flate2::read::GzDecoder; +use std::fs::{self, File}; +use std::io::{self, Read, Seek}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc) -> Result<()> { + tokio::task::spawn_blocking(move || restore(&cfg, &archive, &logger)).await? +} + +fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<()> { + let root = ensure_restorable(Path::new(&cfg.path))?; + let mut matcher = ExcludeMatcher::from_config(cfg)?; + let one_file_system = one_file_system(cfg); + + let file = File::open(archive) + .with_context(|| format!("cannot open archive {}", archive.display()))?; + + logger.log("info", format!("Validating archive {}", archive.display())); + validate_archive(&file).context("archive is corrupt or truncated, nothing was changed")?; + + let archive_real = fs::canonicalize(archive)?; + let keep = match archive_real.parent() { + Some(dir) if dir != root => dir, + _ => archive_real.as_path(), + }; + if let Ok(rel) = keep.strip_prefix(&root) { + matcher.exclude_path(rel.to_path_buf()); + } + + logger.log( + "info", + format!("Removing non-excluded content of {}", root.display()), + ); + + let failures = wipe_except_excluded(&root, &matcher, one_file_system, logger)?; + + logger.log( + "info", + format!("Extracting archive into {}", root.display()), + ); + unpack(&file, &root, is_root(), logger)?; + + if failures > 0 { + let noun = if failures == 1 { "entry" } else { "entries" }; + bail!( + "restore extracted the archive but {failures} existing {noun} could not be removed; see warnings" + ); + } + logger.log("info", "Files restore finished".to_string()); + Ok(()) +} + +pub fn unpack( + file: &File, + root: &Path, + preserve_ownerships: bool, + logger: &JobLogger, +) -> Result<()> { + if let Err(e) = unpack_once(file, root, true, preserve_ownerships) { + logger.log( + "warn", + format!("extraction with ownership/permissions failed: {e:#}; retrying without metadata preservation"), + ); + unpack_once(file, root, false, false)?; + } + Ok(()) +} + +fn unpack_once( + mut file: &File, + root: &Path, + preserve: bool, + preserve_ownerships: bool, +) -> Result<()> { + file.rewind()?; + let mut ar = tar::Archive::new(GzDecoder::new(file)); + ar.set_preserve_permissions(preserve); + ar.set_preserve_mtime(preserve); + ar.set_preserve_ownerships(preserve_ownerships); + ar.set_overwrite(true); + ar.unpack(root)?; + Ok(()) +} + +fn validate_archive(file: &File) -> Result<()> { + let mut gz = GzDecoder::new(file); + let mut ar = tar::Archive::new(&mut gz); + for entry in ar.entries()? { + io::copy(&mut entry?, &mut io::sink())?; + } + drop(ar); + gz.read_to_end(&mut Vec::new())?; + Ok(()) +} + +pub fn ensure_restorable(root: &Path) -> Result { + if root.as_os_str().is_empty() || !root.is_absolute() || root == Path::new("/") { + bail!( + "refusing to restore into {:?}: path must be absolute and not /", + root + ); + } + if !root.is_dir() { + bail!( + "restore target {} is not an existing directory", + root.display() + ); + } + let canonical = fs::canonicalize(root) + .with_context(|| format!("cannot resolve restore target {}", root.display()))?; + if canonical == Path::new("/") { + bail!("refusing to restore into {:?}: it resolves to /", root); + } + Ok(canonical) +} + +pub fn wipe_except_excluded( + root: &Path, + matcher: &ExcludeMatcher, + one_file_system: bool, + logger: &JobLogger, +) -> Result { + let meta = fs::symlink_metadata(root).with_context(|| format!("reading {}", root.display()))?; + let mut wipe = Wipe { + matcher, + one_file_system, + logger, + failures: 0, + }; + wipe.dir(root, Path::new(""), dev(&meta)); + Ok(wipe.failures) +} + +struct Wipe<'a> { + matcher: &'a ExcludeMatcher, + one_file_system: bool, + logger: &'a JobLogger, + failures: usize, +} + +impl Wipe<'_> { + fn fail(&mut self, what: &str, path: &Path, e: io::Error) { + self.logger + .log("warn", format!("Could not {what} {}: {e}", path.display())); + self.failures += 1; + } + + fn dir(&mut self, dir: &Path, rel: &Path, dir_dev: u64) -> bool { + let entries = match fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) => { + self.fail("read", dir, e); + return true; + } + }; + let mut kept = false; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(e) => { + self.fail("read", dir, e); + kept = true; + continue; + } + }; + let path = entry.path(); + let rel = rel.join(entry.file_name()); + if self.matcher.is_excluded(&rel) { + kept = true; + continue; + } + + let meta = match fs::symlink_metadata(&path) { + Ok(meta) => meta, + Err(e) => { + self.fail("read", &path, e); + kept = true; + continue; + } + }; + let ft = meta.file_type(); + let removed = if ft.is_dir() { + let mount_point = dev(&meta) != dir_dev; + if mount_point && self.one_file_system { + kept = true; + continue; + } + if self.dir(&path, &rel, dev(&meta)) || mount_point { + kept = true; + continue; + } + fs::remove_dir(&path) + } else if ft.is_file() || ft.is_symlink() { + fs::remove_file(&path) + } else { + kept = true; + continue; + }; + if let Err(e) = removed { + self.fail("remove", &path, e); + kept = true; + } + } + kept + } +} diff --git a/src/domain/files/sync/backup.rs b/src/domain/files/sync/backup.rs new file mode 100644 index 0000000..fbca007 --- /dev/null +++ b/src/domain/files/sync/backup.rs @@ -0,0 +1,45 @@ +use super::rclone::{replica_target, sync_dir}; +use super::stats::SyncStats; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::services::storage::providers::local::{LOCAL_STORAGE_USER, local_storage_password}; +use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password}; +use crate::services::storage::providers::rclone::target::rclone_target; +use crate::utils::edge_key::EdgeKey; +use crate::utils::file::folder_or_default; +use anyhow::Result; + +pub fn replica_path(storage: &DatabaseStorage, generated_id: &str) -> String { + format!("{}/sync/{generated_id}/current", folder_or_default(storage.folder_name.as_deref())) +} + +const LOCAL_REMOTE: &str = "pblocal"; + +pub fn local_sync_config(edge_key: &EdgeKey) -> Result { + build_rclone_config( + LOCAL_REMOTE, + &[ + ("type", "webdav".into()), + ("url", format!("{}/storage/sync", edge_key.server_url.trim_end_matches('/'))), + ("vendor", "rclone".into()), + ("user", LOCAL_STORAGE_USER.into()), + ("pass", obscure_password(&local_storage_password(&edge_key.master_key_b64)?)?), + ], + ) +} + +pub async fn replicate(edge_key: &EdgeKey, cfg: &DatabaseConfig, storage: &DatabaseStorage, logger: &JobLogger) -> Result { + if storage.provider == "local" { + let dest = format!("{LOCAL_REMOTE}:{}/current", cfg.generated_id); + return sync_dir(&local_sync_config(edge_key)?, &dest, cfg, logger).await; + } + let target = rclone_target(storage)?; + let dest = replica_target( + &target.remote_name, + &target.base_path, + folder_or_default(storage.folder_name.as_deref()), + &cfg.generated_id, + ); + sync_dir(&target.config_text, &dest, cfg, logger).await +} diff --git a/src/domain/files/sync/excludes.rs b/src/domain/files/sync/excludes.rs new file mode 100644 index 0000000..28728e8 --- /dev/null +++ b/src/domain/files/sync/excludes.rs @@ -0,0 +1,55 @@ +use crate::domain::files::matcher::bracket_negation; + +pub fn sync_excludes(patterns: &[String]) -> Vec { + let mut out = Vec::new(); + for raw in patterns { + let pattern = raw.trim(); + let (anchored, rest) = match pattern.strip_prefix('/') { + Some(rest) => (true, rest), + None => (false, pattern), + }; + let mut rest = rest.trim_end_matches('/'); + if !anchored { + while let Some(stripped) = rest.strip_prefix("**/") { + rest = stripped; + } + } + if rest.is_empty() { + continue; + } + let rest = bracket_negation(rest); + let rest = collapse_double_stars(&rest); + let full = if anchored { format!("/{rest}") } else { rest }; + for variant in zero_dir_variants(&full) { + out.push(variant.clone()); + out.push(format!("{variant}/**")); + } + } + out +} + +fn collapse_double_stars(pattern: &str) -> String { + let mut result = pattern.to_string(); + while result.contains("/**/**/") { + result = result.replace("/**/**/", "/**/"); + } + result +} + +fn zero_dir_variants(pattern: &str) -> Vec { + let parts: Vec<&str> = pattern.split("/**/").collect(); + let gaps = parts.len() - 1; + if gaps > 4 { + return vec![pattern.to_string(), pattern.replace("/**/", "/")]; + } + (0..1u32 << gaps) + .map(|mask| { + let mut variant = parts[0].to_string(); + for (i, part) in parts[1..].iter().enumerate() { + variant.push_str(if mask & (1 << i) == 0 { "/**/" } else { "/" }); + variant.push_str(part); + } + variant + }) + .collect() +} diff --git a/src/domain/files/sync/mod.rs b/src/domain/files/sync/mod.rs new file mode 100644 index 0000000..5a26cb6 --- /dev/null +++ b/src/domain/files/sync/mod.rs @@ -0,0 +1,4 @@ +pub mod backup; +pub mod excludes; +pub mod rclone; +pub mod stats; diff --git a/src/domain/files/sync/rclone.rs b/src/domain/files/sync/rclone.rs new file mode 100644 index 0000000..d85525d --- /dev/null +++ b/src/domain/files/sync/rclone.rs @@ -0,0 +1,121 @@ +use super::excludes::sync_excludes; +use super::stats::{SyncStats, parse_log, progress}; +use crate::domain::files::matcher::{exclude_patterns, one_file_system, source_root}; +use crate::services::backup::logger::JobLogger; +use crate::services::config::DatabaseConfig; +use crate::services::storage::providers::rclone::helpers::{remote_target, write_config}; +use anyhow::{Context, Result, anyhow, bail}; +use std::path::Path; +use std::process::Stdio; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generated_id: &str) -> String { + remote_target(remote_name, base_path, &format!("{folder}/sync/{generated_id}/current")) +} + +pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logger: &JobLogger) -> Result { + let root = source_root(cfg)?; + if std::fs::read_dir(&root) + .with_context(|| format!("cannot read source directory {}", root.display()))? + .next() + .is_none() + { + bail!( + "source directory {} is empty; refusing to sync so an unmounted volume cannot wipe the replica", + root.display() + ); + } + let config = write_config(config_text)?; + + let mut cmd = Command::new("rclone"); + cmd.arg("--config") + .arg(config.path()) + .arg("sync") + .arg(&root) + .arg(dest) + .args([ + "--links", "--delete-excluded", "--use-json-log", "--stats-log-level", "NOTICE", "--retries", "1", + "--contimeout", "30s", "--low-level-retries", "3", + ]); + if one_file_system(cfg) { + cmd.arg("--one-file-system"); + } + for pattern in sync_excludes(&exclude_patterns(cfg)) { + cmd.arg("--exclude").arg(pattern); + } + + let debug = tracing::enabled!(tracing::Level::DEBUG); + cmd.args(["--stats", if debug { "5s" } else { "1h" }]); + + logger.log("info", format!("Syncing {} to {dest}", root.display())); + let mut child = cmd + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => anyhow!("rclone binary not found"), + _ => anyhow!(e).context("failed to start rclone"), + })?; + let mut lines = BufReader::new(child.stderr.take().context("rclone stderr unavailable")?).lines(); + let mut stderr = String::new(); + while let Some(line) = lines.next_line().await.context("failed to read rclone output")? { + if debug { + if let Some(progress) = progress(&line) { + logger.log("debug", format!("progress: {progress}")); + } + } + stderr.push_str(&line); + stderr.push('\n'); + } + let status = child.wait().await.context("failed to wait for rclone")?; + + let log = parse_log(&stderr); + for error in &log.errors { + logger.log("warn", format!("rclone: {error}")); + } + if !status.success() { + let reason = log + .stats + .as_ref() + .and_then(|s| s.last_error.clone()) + .or_else(|| log.errors.last().cloned()) + .unwrap_or_else(|| "no error message".to_string()); + bail!("rclone sync failed ({status}): {reason}; the replica may be partially updated"); + } + let mut stats = log.stats.unwrap_or_default(); + stats.replica_bytes = match remote_size(config.path(), dest).await { + Ok(bytes) => Some(bytes), + Err(e) => { + logger.log("warn", format!("Could not measure the replica size: {e:#}")); + None + } + }; + let replica = stats.replica_bytes.map(|b| format!("; replica holds {b} bytes")).unwrap_or_default(); + logger.log( + "info", + format!("Synced: {} file(s) transferred ({} bytes), {} deleted{replica}", stats.transfers, stats.bytes, stats.deletes), + ); + Ok(stats) +} + +async fn remote_size(config: &Path, target: &str) -> Result { + let out = Command::new("rclone") + .args(["--contimeout", "30s", "--low-level-retries", "3", "--config"]) + .arg(config) + .arg("size") + .arg(target) + .arg("--json") + .stdin(Stdio::null()) + .kill_on_drop(true) + .output() + .await + .context("failed to start rclone")?; + if !out.status.success() { + bail!("rclone size failed ({}): {}", out.status, String::from_utf8_lossy(&out.stderr).trim()); + } + let size: serde_json::Value = serde_json::from_slice(&out.stdout).context("unexpected rclone size output")?; + size["bytes"].as_u64().context("rclone size returned no byte count") +} diff --git a/src/domain/files/sync/stats.rs b/src/domain/files/sync/stats.rs new file mode 100644 index 0000000..bfa925e --- /dev/null +++ b/src/domain/files/sync/stats.rs @@ -0,0 +1,85 @@ +use crate::utils::text::human_size; +use serde::Deserialize; +use serde_json::{Value, json}; + +#[derive(Debug, Default, Deserialize, PartialEq)] +pub struct SyncStats { + #[serde(default)] + pub bytes: u64, + #[serde(default)] + pub transfers: u64, + #[serde(default)] + pub deletes: u64, + #[serde(default)] + pub errors: u64, + #[serde(default, rename = "lastError")] + pub last_error: Option, + #[serde(skip)] + pub replica_bytes: Option, +} + +impl SyncStats { + pub fn report(&self) -> Value { + json!({ + "filesTransferred": self.transfers, + "filesDeleted": self.deletes, + "replicaBytes": self.replica_bytes, + }) + } +} + +#[derive(Debug, Default)] +pub struct SyncLog { + pub stats: Option, + pub errors: Vec, +} + +pub fn parse_log(stderr: &str) -> SyncLog { + let mut log = SyncLog::default(); + for line in stderr.lines().map(str::trim).filter(|l| !l.is_empty()) { + let Ok(value) = serde_json::from_str::(line) else { + if !log.errors.contains(&line.to_string()) { + log.errors.push(line.to_string()); + } + continue; + }; + if let Some(stats) = value.get("stats").and_then(|s| serde_json::from_value(s.clone()).ok()) { + log.stats = Some(stats); + } + if let Some(level) = value.get("level").and_then(Value::as_str) { + if matches!(level, "error" | "critical" | "alert" | "emergency") { + if let Some(msg) = value.get("msg").and_then(Value::as_str) { + let msg_str = msg.to_string(); + if !log.errors.contains(&msg_str) { + log.errors.push(msg_str); + } + } + } + } + } + if log.errors.len() > 20 { + log.errors = log.errors.into_iter().rev().take(20).collect::>(); + log.errors.reverse(); + } + log +} + +pub fn progress(line: &str) -> Option { + let value: Value = serde_json::from_str(line).ok()?; + let stats = value.get("stats")?; + let n = |key: &str| stats[key].as_u64().unwrap_or(0); + let (bytes, total) = (n("bytes"), n("totalBytes")); + let percent = if total > 0 { bytes * 100 / total } else { 100 }; + let eta = stats["eta"].as_u64().map(|s| format!(", ETA {s}s")).unwrap_or_default(); + Some(format!( + "rclone sync: {} / {} ({percent}%), {}/{} transferred, {}/{} checked, {} deleted, {}/s{eta}", + human_size(bytes), + human_size(total), + n("transfers"), + n("totalTransfers"), + n("checks"), + n("totalChecks"), + n("deletes"), + human_size(stats["speed"].as_f64().unwrap_or(0.0) as u64), + )) +} diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 095cbe1..16fcba1 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -1,4 +1,5 @@ pub mod docker_volume; +pub mod files; pub mod factory; pub mod mongodb; pub mod mysql; diff --git a/src/services/api/client.rs b/src/services/api/client.rs index f1e396c..3d734de 100644 --- a/src/services/api/client.rs +++ b/src/services/api/client.rs @@ -1,14 +1,35 @@ #![allow(dead_code)] use crate::services::api::ApiError; -use reqwest::{Client, Method}; +use base64::{Engine as _, engine::general_purpose}; +use hmac::{Hmac, Mac}; +use reqwest::header::CONTENT_TYPE; +use reqwest::{Client, Method, RequestBuilder}; use serde::de::DeserializeOwned; -use std::time::Duration; +use sha2::{Digest, Sha256}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tracing::error; + +type HmacSha256 = Hmac; #[derive(Clone, Debug)] pub struct ApiClient { base_url: String, http: Client, + agent_key: Option>, +} + +pub fn derive_agent_key(master_key: &[u8], agent_id: &str) -> Vec { + let mut mac = HmacSha256::new_from_slice(master_key).expect("HMAC accepts any key length"); + mac.update(format!("portabase-agent-auth:v1:{agent_id}").as_bytes()); + mac.finalize().into_bytes().to_vec() +} + +pub fn sign_request(agent_key: &[u8], method: &str, path: &str, timestamp: &str, body: &[u8]) -> String { + let body_hash = hex::encode(Sha256::digest(body)); + let mut mac = HmacSha256::new_from_slice(agent_key).expect("HMAC accepts any key length"); + mac.update(format!("{method}\n{path}\n{timestamp}\n{body_hash}").as_bytes()); + hex::encode(mac.finalize().into_bytes()) } impl ApiClient { @@ -21,9 +42,37 @@ impl ApiClient { Self { base_url: base_url.into(), http, + agent_key: None, } } + pub fn with_agent_auth(mut self, agent_id: &str, master_key_b64: &str) -> Self { + let master_key = match general_purpose::STANDARD.decode(master_key_b64) { + Ok(key) => key, + Err(_) => { + error!("Invalid EDGE_KEY: master key is not valid base64"); + panic!("Cannot initialize AgentContext due to invalid EDGE_KEY"); + } + }; + self.agent_key = Some(derive_agent_key(&master_key, agent_id)); + self + } + + fn sign(&self, request: RequestBuilder, method: &Method, path: &str, body: &[u8]) -> RequestBuilder { + let Some(agent_key) = &self.agent_key else { + return request; + }; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before 1970") + .as_secs() + .to_string(); + let signature = sign_request(agent_key, method.as_str(), &format!("/api{path}"), ×tamp, body); + request + .header("X-Agent-Timestamp", timestamp) + .header("X-Agent-Signature", signature) + } + pub async fn request( &self, method: Method, @@ -31,7 +80,8 @@ impl ApiClient { ) -> Result, ApiError> { let url = format!("{}{}", self.base_url, path); - let res = self.http.request(method, &url).send().await?; + let request = self.sign(self.http.request(method.clone(), &url), &method, path, b""); + let res = request.send().await?; let status = res.status(); let body = res.text().await.unwrap_or_default(); @@ -57,7 +107,13 @@ impl ApiClient { { let url = format!("{}{}", self.base_url, path); - let res = self.http.request(method, &url).json(body).send().await?; + // Serialize once: the signature covers exactly the bytes that are sent. + let bytes = serde_json::to_vec(body)?; + let request = self + .http + .request(method.clone(), &url) + .header(CONTENT_TYPE, "application/json"); + let res = self.sign(request, &method, path, &bytes).body(bytes).send().await?; let status = res.status(); let body_text = res.text().await.unwrap_or_default(); diff --git a/src/services/api/endpoints/agent/backup/upload/init.rs b/src/services/api/endpoints/agent/backup/upload/init.rs index cd39651..c3840c3 100644 --- a/src/services/api/endpoints/agent/backup/upload/init.rs +++ b/src/services/api/endpoints/agent/backup/upload/init.rs @@ -12,6 +12,9 @@ pub struct InitUploadRequest { pub storage_channel_id: String, #[serde(rename = "backupId")] pub backup_id: String, + /// "restic" for snapshots; omitted for archives (older dashboards ignore it). + #[serde(skip_serializing_if = "Option::is_none")] + pub engine: Option, } impl ApiClient { @@ -21,11 +24,13 @@ impl ApiClient { generated_id: impl Into, storage_channel_id: impl Into, backup_id: impl Into, + engine: Option<&str>, ) -> Result, ApiError> { let body = InitUploadRequest { generated_id: generated_id.into(), storage_channel_id: storage_channel_id.into(), backup_id: backup_id.into(), + engine: engine.map(str::to_string), }; let agent_id = agent_id.into(); diff --git a/src/services/api/endpoints/agent/backup/upload/status.rs b/src/services/api/endpoints/agent/backup/upload/status.rs index 41b1acc..441b49d 100644 --- a/src/services/api/endpoints/agent/backup/upload/status.rs +++ b/src/services/api/endpoints/agent/backup/upload/status.rs @@ -3,6 +3,7 @@ use crate::services::api::{ApiClient, ApiError}; use anyhow::Result; use reqwest::Method; use serde::Serialize; +use serde_json::Value; #[derive(Serialize)] pub struct StatusUploadRequest { @@ -15,6 +16,9 @@ pub struct StatusUploadRequest { pub size: u64, #[serde(rename = "backupId")] pub backup_id: String, + /// Counters of a successful snapshot or sync run, shown by the dashboard. + #[serde(skip_serializing_if = "Option::is_none")] + pub stats: Option, } impl ApiClient { @@ -35,12 +39,40 @@ impl ApiClient { path: remote_path.into(), size: total_size.into(), backup_id: backup_id.into(), + stats: None, }; + self.send_upload_status(agent_id.into(), &body).await + } - let agent_id = agent_id.into(); - let path = format!("/agent/{}/backup/upload/status", agent_id); + /// Success status of one snapshot or sync run, with its counters. + pub async fn backup_upload_success( + &self, + agent_id: impl Into, + generated_id: impl Into, + backup_storage_id: impl Into, + path: String, + size: u64, + stats: Value, + backup_id: impl Into, + ) -> Result, ApiError> { + let body = StatusUploadRequest { + generated_id: generated_id.into(), + backup_storage_id: backup_storage_id.into(), + status: "success".into(), + path, + size, + backup_id: backup_id.into(), + stats: Some(stats), + }; + self.send_upload_status(agent_id.into(), &body).await + } - self.request_with_body(Method::PATCH, path.as_str(), &body) - .await + async fn send_upload_status( + &self, + agent_id: String, + body: &StatusUploadRequest, + ) -> Result, ApiError> { + let path = format!("/agent/{}/backup/upload/status", agent_id); + self.request_with_body(Method::PATCH, path.as_str(), body).await } } diff --git a/src/services/api/endpoints/agent/status.rs b/src/services/api/endpoints/agent/status.rs index 8c7a5ce..73acba0 100644 --- a/src/services/api/endpoints/agent/status.rs +++ b/src/services/api/endpoints/agent/status.rs @@ -12,7 +12,9 @@ pub struct DatabasePayload<'a> { #[serde(rename = "generatedId")] pub generated_id: &'a str, #[serde(rename = "pingStatus")] - pub ping_status: bool + pub ping_status: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub method: Option<&'a str>, } #[derive(Serialize)] diff --git a/src/services/api/models/agent/status.rs b/src/services/api/models/agent/status.rs index 1a5aeab..3a75d7e 100644 --- a/src/services/api/models/agent/status.rs +++ b/src/services/api/models/agent/status.rs @@ -61,14 +61,29 @@ pub struct DatabaseData { pub struct BackupInfo { pub action: bool, pub cron: Option, + /// "archive" | "restic"; absent from dashboards older than P2 (→ archive). + #[serde(default)] + pub engine: Option, } #[derive(Debug, Deserialize)] pub struct RestoreInfo { pub action: bool, + #[serde(default)] pub file: Option, - #[serde(rename = "metaFile")] + #[serde(default, rename = "metaFile")] pub meta_file: Option, #[serde(default, deserialize_with = "string_or_number_to_string")] pub size: Option, + /// "restic" for a snapshot restore; absent for archives. + #[serde(default)] + pub engine: Option, + #[serde(default, rename = "snapshotId")] + pub snapshot_id: Option, + /// AES-GCM JSON array holding the snapshot's storage channel. + #[serde(default, rename = "storageCiphertext")] + pub storage_ciphertext: Option, + /// Filled in memory from `storage_ciphertext`; never on the wire. + #[serde(skip)] + pub storage: Option, } diff --git a/src/services/backup/dispatcher.rs b/src/services/backup/dispatcher.rs index bf9145a..469cc0b 100644 --- a/src/services/backup/dispatcher.rs +++ b/src/services/backup/dispatcher.rs @@ -12,6 +12,7 @@ impl BackupService { method: BackupMethod, storages: &Vec, encrypt: bool, + engine: &str, ) { let Some(cfg) = config .databases @@ -29,10 +30,11 @@ impl BackupService { let db_cfg = cfg.clone(); let storages = storages.clone(); let generated_id = generated_id.clone(); + let engine = engine.to_string(); tokio::spawn(async move { if let Err(e) = service - .execute_backup(generated_id, db_cfg, method, storages, encrypt) + .execute_backup(generated_id, db_cfg, method, storages, encrypt, engine) .await { error!("Backup execution failed: {}", e); diff --git a/src/services/backup/executor.rs b/src/services/backup/executor.rs index 018eed9..269bd3c 100644 --- a/src/services/backup/executor.rs +++ b/src/services/backup/executor.rs @@ -1,7 +1,9 @@ +use super::files_engine::{self, FilesEngine}; use super::logger::JobLogger; +use super::models::BackupResult; use super::service::BackupService; use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::config::DatabaseConfig; +use crate::services::config::{DatabaseConfig, DbType}; use crate::utils::common::BackupMethod; use crate::utils::locks::FileLock; @@ -18,6 +20,7 @@ impl BackupService { method: BackupMethod, storages: Vec, encrypt: bool, + engine: String, ) -> Result<()> { let logger = Arc::new(JobLogger::new()); @@ -30,6 +33,25 @@ impl BackupService { let backup = self.create_backup_record(&generated_id, &method).await?; let backup_id = backup.backup.id; + if let Some(files_engine) = FilesEngine::parse(&engine) { + if matches!(db_cfg.db_type, DbType::Files) { + let uploads = files_engine::run(&self.ctx, &db_cfg, files_engine, &storages, &backup_id, &logger).await; + logger.log("info", "Database backup job finished".to_string()); + let result = BackupResult { + generated_id: generated_id.clone(), + db_type: db_cfg.db_type.clone(), + status: "success".into(), + backup_file: None, + code: None, + }; + let duration_ms = start.elapsed().as_millis() as f64; + let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries(); + self.send_result(result, uploads, &backup_id, logs, duration_ms).await?; + return Ok(()); + } + logger.log("warn", format!("The {engine} method only applies to files sources; making an archive")); + } + let temp_dir = TempDir::new()?; let tmp_path = temp_dir.path(); diff --git a/src/services/backup/files_engine.rs b/src/services/backup/files_engine.rs new file mode 100644 index 0000000..e44389b --- /dev/null +++ b/src/services/backup/files_engine.rs @@ -0,0 +1,166 @@ +use super::logger::JobLogger; +use super::models::UploadResult; +use crate::core::context::Context as CoreContext; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::sync::backup::{replica_path, replicate}; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::Result; +use serde_json::Value; + +#[derive(Clone, Copy)] +pub enum FilesEngine { + Restic, + Sync, +} + +impl FilesEngine { + pub fn parse(engine: &str) -> Option { + match engine { + "restic" => Some(Self::Restic), + "sync" => Some(Self::Sync), + _ => None, + } + } + + fn as_str(self) -> &'static str { + match self { + Self::Restic => "restic", + Self::Sync => "sync", + } + } + + fn label(self) -> &'static str { + match self { + Self::Restic => "Snapshot", + Self::Sync => "Sync", + } + } +} + +struct Transfer { + remote_path: String, + size: u64, + report: Value, + total_size: Option, +} + +pub async fn run( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storages: &[DatabaseStorage], + backup_id: &str, + logger: &JobLogger, +) -> Vec { + if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await { + logger.log("error", format!("{} aborted: {e}", engine.label())); + return Vec::new(); + } + let mut results = Vec::with_capacity(storages.len()); + for storage in storages { + results.push(one_storage(ctx, cfg, engine, storage, backup_id, logger).await); + } + if let Err(e) = FileLock::release(&cfg.generated_id).await { + logger.log("warn", format!("Failed to release the backup lock: {e}")); + } + results +} + +async fn transfer( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storage: &DatabaseStorage, + backup_storage_id: &str, + logger: &JobLogger, +) -> Result { + match engine { + FilesEngine::Restic => { + let repo = ResticRepo::open(storage, &cfg.generated_id, &ctx.edge_key)?; + let summary = snapshot(&repo, cfg, backup_storage_id, logger).await?; + Ok(Transfer { + size: summary.data_added_packed, + report: summary.report(), + total_size: Some(summary.total_bytes_processed), + remote_path: summary.snapshot_id, + }) + } + FilesEngine::Sync => { + let stats = replicate(&ctx.edge_key, cfg, storage, logger).await?; + Ok(Transfer { + remote_path: replica_path(storage, &cfg.generated_id), + size: stats.bytes, + report: stats.report(), + total_size: stats.replica_bytes, + }) + } + } +} + +pub(crate) async fn one_storage( + ctx: &CoreContext, + cfg: &DatabaseConfig, + engine: FilesEngine, + storage: &DatabaseStorage, + backup_id: &str, + logger: &JobLogger, +) -> UploadResult { + let agent_id = ctx.edge_key.agent_id.clone(); + let failed = |error: String| UploadResult { + storage_id: storage.id.clone(), + success: false, + error: Some(error), + remote_file_path: None, + total_size: None, + }; + + let backup_storage_id = match ctx + .api + .backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some(engine.as_str())) + .await + { + Ok(Some(response)) => response.backup_storage.id, + Ok(None) => { + logger.log("error", "Upload init returned empty response"); + return failed("backup_upload_init returned empty response".into()); + } + Err(e) => { + logger.log("error", format!("Upload init failed: {e}")); + return failed("backup_upload_init failed".into()); + } + }; + + match transfer(ctx, cfg, engine, storage, &backup_storage_id, logger).await { + Ok(t) => match ctx + .api + .backup_upload_success(agent_id, cfg.generated_id.clone(), backup_storage_id, t.remote_path.clone(), t.size, t.report, backup_id) + .await + { + Ok(_) => UploadResult { + storage_id: storage.id.clone(), + success: true, + error: None, + remote_file_path: Some(t.remote_path), + total_size: t.total_size, + }, + Err(e) => { + logger.log("error", format!("Upload status update failed for {}: {e}", storage.id)); + failed(e.to_string()) + } + }, + Err(e) => { + logger.log("error", format!("{} to storage {} failed: {e:#}", engine.label(), storage.id)); + if let Err(err) = ctx + .api + .backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id) + .await + { + logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id)); + } + failed(format!("{e:#}")) + } + } +} diff --git a/src/services/backup/mod.rs b/src/services/backup/mod.rs index b761556..aa65590 100644 --- a/src/services/backup/mod.rs +++ b/src/services/backup/mod.rs @@ -2,6 +2,7 @@ pub mod compressor; pub mod logger; pub mod dispatcher; pub mod executor; +pub mod files_engine; pub mod helpers; pub mod models; pub mod result; diff --git a/src/services/backup/result.rs b/src/services/backup/result.rs index e005948..5d2b124 100644 --- a/src/services/backup/result.rs +++ b/src/services/backup/result.rs @@ -21,11 +21,9 @@ impl BackupService { "failed" }; - let file_size = upload_results - .iter() - .filter_map(|r| r.total_size) - .reduce(|a, b| a + b) - .map(|sum| sum / upload_results.len() as u64); + // Average over the uploads that reported a size: failed ones have none. + let sizes: Vec = upload_results.iter().filter_map(|r| r.total_size).collect(); + let file_size = (!sizes.is_empty()).then(|| sizes.iter().sum::() / sizes.len() as u64); self.ctx .api diff --git a/src/services/backup/uploader.rs b/src/services/backup/uploader.rs index 64b6325..c20e8f1 100644 --- a/src/services/backup/uploader.rs +++ b/src/services/backup/uploader.rs @@ -49,6 +49,7 @@ impl BackupService { generated_id.clone(), storage_id.clone(), backup_id, + None, ) .await { diff --git a/src/services/config.rs b/src/services/config.rs index 9819632..bb3c3b9 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -28,6 +28,7 @@ pub enum DbType { Mssql, #[serde(rename = "docker-volume")] DockerVolume, + Files, } impl DbType { @@ -44,6 +45,7 @@ impl DbType { DbType::Firebird => "firebird", DbType::Mssql => "mssql", DbType::DockerVolume => "docker-volume", + DbType::Files => "files", } } } @@ -112,6 +114,17 @@ fn optional(opt: &Option) -> T { opt.clone().unwrap_or_default() } +pub const FILES_METHODS: [&str; 3] = ["archive", "snapshot", "sync"]; + +/// The method a files source declares in `options.method` (`archive` when absent). +pub fn files_method(cfg: &DatabaseConfig) -> &'static str { + match cfg.options.get("method").and_then(|v| v.as_str()) { + Some("snapshot") => "snapshot", + Some("sync") => "sync", + _ => "archive", + } +} + pub fn build_config(db: InputDatabaseConfig) -> Result { if Uuid::parse_str(&db.generated_id).is_err() { return Err(format!("Invalid UUID for database '{}'", db.name)); @@ -143,7 +156,7 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { | DbType::Firebird | DbType::Valkey | DbType::Mssql => required(&db.host, &db.name, "host")?, - DbType::Sqlite | DbType::DockerVolume => optional(&db.host), + DbType::Sqlite | DbType::DockerVolume | DbType::Files => optional(&db.host), }; let port = match db.db_type { @@ -155,11 +168,13 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { | DbType::Firebird | DbType::Valkey | DbType::Mssql => required(&db.port, &db.name, "port")?, - DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), + DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume | DbType::Files => { + db.port.unwrap_or(0) + } }; let database_name = match db.db_type { - DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => { + DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume | DbType::Files => { optional(&db.database) } DbType::PostgresqlCluster => db @@ -170,8 +185,28 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { }; let path_val = match db.db_type { DbType::Sqlite => required(&db.path, &db.name, "path")?, + DbType::Files => { + let p = required(&db.path, &db.name, "path")?; + if !p.starts_with('/') || p.trim_end_matches('/').is_empty() { + return Err(format!( + "Path for files source '{}' must be absolute and not '/'", + db.name + )); + } + p + } _ => optional(&db.path), }; + if matches!(db.db_type, DbType::Files) { + if let Some(method) = db.options.as_ref().and_then(|o| o.get("method")) { + if !method.as_str().is_some_and(|m| FILES_METHODS.contains(&m)) { + return Err(format!( + "Unknown method {method} for files source '{}' (expected archive, snapshot or sync)", + db.name + )); + } + } + } let max_packet_size = match db.db_type { DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()), _ => String::new(), diff --git a/src/services/cron.rs b/src/services/cron.rs index 85ba05f..a74ac39 100644 --- a/src/services/cron.rs +++ b/src/services/cron.rs @@ -51,7 +51,8 @@ impl CronService { let encrypt: bool = database.encrypt; let metadata = json!({ "storages": storages, - "encrypt": encrypt + "encrypt": encrypt, + "engine": database.data.backup.engine.as_deref().unwrap_or("archive"), }); check_and_update_cron( diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs index 851a2c8..babc834 100644 --- a/src/services/dashboard_config.rs +++ b/src/services/dashboard_config.rs @@ -2,6 +2,7 @@ use crate::services::api::models::agent::status::PingResult; use crate::services::config::{DatabaseConfig, DatabasesConfig}; +use std::collections::HashSet; use std::path::Path; pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig { @@ -19,6 +20,14 @@ pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> Database DatabasesConfig { databases } } +pub fn local_only_ids(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> HashSet { + local + .iter() + .filter(|c| !dashboard.iter().any(|d| d.generated_id == c.generated_id)) + .map(|c| c.generated_id.clone()) + .collect() +} + pub fn collect_configs(ping: &PingResult) -> Vec { ping.databases .iter() diff --git a/src/services/restore/archive.rs b/src/services/restore/archive.rs index 3119522..73412ec 100644 --- a/src/services/restore/archive.rs +++ b/src/services/restore/archive.rs @@ -60,6 +60,11 @@ impl RestoreService { archive = decrypted; } + if matches!(db_type, DbType::Files) { + logger.log("info", "Files archive ready for extraction".to_string()); + return Ok(archive); + } + if matches!(db_type, DbType::DockerVolume) { let raw_tar = tmp_path.join("volume.tar"); crate::utils::compress::gunzip_to_file(archive.as_path(), &raw_tar).await?; diff --git a/src/services/restore/dispatcher.rs b/src/services/restore/dispatcher.rs index 61f35fd..2cf7b2f 100644 --- a/src/services/restore/dispatcher.rs +++ b/src/services/restore/dispatcher.rs @@ -15,6 +15,19 @@ impl RestoreService { return; }; + if db.data.restore.engine.as_deref() == Some("restic") { + let snapshot_id = db.data.restore.snapshot_id.clone(); + let storage = db.data.restore.storage.clone(); + let service = Self { ctx: self.ctx.clone() }; + let db_cfg = cfg.clone(); + tokio::spawn(async move { + if let Err(e) = service.execute_restic_restore(db_cfg, snapshot_id, storage).await { + error!("Snapshot restore failed: {}", e); + } + }); + return; + } + let Some(file_to_restore) = db.data.restore.file.clone() else { error!("restore file not found"); return; diff --git a/src/services/restore/downloader.rs b/src/services/restore/downloader.rs index bd70a98..c4ae1e0 100644 --- a/src/services/restore/downloader.rs +++ b/src/services/restore/downloader.rs @@ -2,6 +2,7 @@ use super::service::RestoreService; use crate::services::backup::logger::JobLogger; use crate::utils::retry::{RetryPolicy, retry}; +use crate::utils::text::human_size; use anyhow::Result; use futures::StreamExt; use reqwest::{Client, Url}; @@ -10,16 +11,6 @@ use std::sync::Arc; use std::time::Instant; use tokio::io::AsyncWriteExt; -fn human_size(bytes: u64) -> String { - if bytes >= 1024 * 1024 { - format!("{} MB", bytes / 1024 / 1024) - } else if bytes >= 1024 { - format!("{} KB", bytes / 1024) - } else { - format!("{bytes} B") - } -} - impl RestoreService { pub async fn download_backup( &self, diff --git a/src/services/restore/executor.rs b/src/services/restore/executor.rs index 49fbcd2..d7575cc 100644 --- a/src/services/restore/executor.rs +++ b/src/services/restore/executor.rs @@ -1,7 +1,12 @@ +use super::models::RestoreResult; use super::service::RestoreService; +use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::logger::JobLogger; -use crate::services::config::DatabaseConfig; -use anyhow::Result; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::{Context, Result, bail}; use std::sync::Arc; use std::time::Instant; use tempfile::TempDir; @@ -43,4 +48,60 @@ impl RestoreService { Ok(()) } + + pub async fn execute_restic_restore( + &self, + cfg: DatabaseConfig, + snapshot_id: Option, + storage: Option, + ) -> Result<()> { + let logger = Arc::new(JobLogger::new()); + let start = Instant::now(); + logger.log("info", "Snapshot restore job started".to_string()); + + let outcome = match (snapshot_id, storage) { + (Some(id), Some(storage)) => self.restic_restore(&cfg, &id, &storage, &logger).await, + _ => Err(anyhow::anyhow!( + "incomplete snapshot restore payload (snapshotId or storage missing)" + )), + }; + let status = match outcome { + Ok(()) => { + logger.log("info", "Snapshot restore job finished".to_string()); + "success" + } + Err(e) => { + logger.log("error", format!("Snapshot restore failed: {e:#}")); + "failed" + } + }; + + let duration_ms = start.elapsed().as_millis() as f64; + let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries(); + let result = RestoreResult { generated_id: cfg.generated_id.clone(), status: status.into() }; + self.send_result(result, logs, duration_ms).await?; + Ok(()) + } + + async fn restic_restore( + &self, + cfg: &DatabaseConfig, + snapshot_id: &str, + storage: &DatabaseStorage, + logger: &JobLogger, + ) -> Result<()> { + if !matches!(cfg.db_type, DbType::Files) { + bail!("snapshot restore needs a files source, got {}", cfg.db_type.as_str()); + } + FileLock::acquire(&cfg.generated_id, DbOpLock::Restore.as_str()) + .await + .context("another backup or restore of this source is running")?; + let result = async { + let repo = ResticRepo::open(storage, &cfg.generated_id, &self.ctx.edge_key)?; + restore(&repo, cfg, snapshot_id, logger).await + } + .await; + FileLock::release(&cfg.generated_id).await?; + result + } } diff --git a/src/services/status.rs b/src/services/status.rs index 8779adb..ed85daa 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -6,11 +6,12 @@ use crate::services::api::endpoints::status::DatabasePayload; use crate::services::api::models::agent::status::DatabaseStatus; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::api::models::agent::status::PingResult; -use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig}; +use crate::services::config::{build_config, files_method, DatabaseConfig, DbType, InputDatabaseConfig}; use crate::settings::CONFIG; use crate::utils::file::decrypt_json_gcm; use futures_util::future::try_join_all; use reqwest::Client; +use std::collections::HashSet; use std::error::Error; use std::sync::Arc; use tracing::info; @@ -35,6 +36,25 @@ pub fn resolve_dashboard_config( Ok(()) } +fn decrypt_storage(ciphertext: &str, master_key_b64: &str, what: &str) -> Result, String> { + let plaintext = decrypt_json_gcm(ciphertext, master_key_b64) + .map_err(|e| format!("Failed to decrypt {what} storage: {e}"))?; + let storages: Vec = serde_json::from_slice(&plaintext) + .map_err(|e| format!("Failed to parse {what} storage: {e}"))?; + Ok(storages.into_iter().next()) +} + +pub fn resolve_restore_storage(status: &mut DatabaseStatus, master_key_b64: &str) -> Result<(), String> { + if let Some(ciphertext) = status.data.restore.storage_ciphertext.clone() { + status.data.restore.storage = decrypt_storage(&ciphertext, master_key_b64, "restore")?; + } + Ok(()) +} + +pub fn payload_method(db: &DatabaseConfig, local_ids: &HashSet) -> Option<&'static str> { + (matches!(db.db_type, DbType::Files) && local_ids.contains(&db.generated_id)).then(|| files_method(db)) +} + pub struct StatusService { ctx: Arc, client: Client, @@ -48,7 +68,11 @@ impl StatusService { } } - pub async fn ping(&self, databases: &[DatabaseConfig]) -> Result> { + pub async fn ping( + &self, + databases: &[DatabaseConfig], + local_ids: &HashSet, + ) -> Result> { let edge_key = &self.ctx.edge_key; let databases_payload: Vec = @@ -63,6 +87,7 @@ impl StatusService { dbms: &db.db_type.as_str(), generated_id: &db.generated_id, ping_status: reachable, + method: payload_method(db, local_ids), }) })) .await?; @@ -92,6 +117,9 @@ impl StatusService { if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) { tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id); } + if let Err(e) = resolve_restore_storage(db, &edge_key.master_key_b64) { + tracing::warn!("Snapshot restore storage unreadable for {}: {e}", db.generated_id); + } } Ok(result) } diff --git a/src/services/storage/providers/google_cloud_storage/helpers.rs b/src/services/storage/providers/google_cloud_storage/helpers.rs index 431cdba..eee1689 100644 --- a/src/services/storage/providers/google_cloud_storage/helpers.rs +++ b/src/services/storage/providers/google_cloud_storage/helpers.rs @@ -8,25 +8,26 @@ use google_cloud_storage::client::Storage; use google_cloud_storage::streaming_source::{SizeHint, StreamingSource}; use std::pin::Pin; -pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result { +/// Service-account key JSON (keys in alphabetical order so the rclone mapper +/// emits byte-identical output to the dashboard's). +pub fn service_account_key(cfg: &GoogleCloudStorageProviderConfig) -> serde_json::Value { // Service-account JSON stores the PEM with `\n` escape sequences. When the key is // carried through config as a JSON string those can arrive as literal two-char `\n` - // sequences rather than real newlines, so the PEM parser finds no `-----BEGIN-----` - // line ("no items found"). Normalize them back to real newlines. A PEM that already - // has real newlines contains no literal `\n` pairs, so this is a no-op for it. + // sequences rather than real newlines. Normalize them back to real newlines. let private_key = cfg.private_key.replace("\\n", "\n"); - - let key = serde_json::json!({ - "type": "service_account", - "project_id": cfg.project_id, + serde_json::json!({ "client_email": cfg.client_email, "private_key": private_key, "private_key_id": "", + "project_id": cfg.project_id, "token_uri": "https://oauth2.googleapis.com/token", + "type": "service_account", "universe_domain": "googleapis.com", - }); + }) +} - google_cloud_auth::credentials::service_account::Builder::new(key) +pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result { + google_cloud_auth::credentials::service_account::Builder::new(service_account_key(cfg)) .build() .context("failed to build GCS service account credentials") } diff --git a/src/services/storage/providers/google_cloud_storage/mod.rs b/src/services/storage/providers/google_cloud_storage/mod.rs index 0220a09..fa76cc5 100644 --- a/src/services/storage/providers/google_cloud_storage/mod.rs +++ b/src/services/storage/providers/google_cloud_storage/mod.rs @@ -1,5 +1,5 @@ pub mod helpers; -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/google_drive/mod.rs b/src/services/storage/providers/google_drive/mod.rs index e127414..c8089e6 100644 --- a/src/services/storage/providers/google_drive/mod.rs +++ b/src/services/storage/providers/google_drive/mod.rs @@ -1,5 +1,5 @@ mod helpers; -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/local.rs b/src/services/storage/providers/local.rs index b93920a..2bd923a 100644 --- a/src/services/storage/providers/local.rs +++ b/src/services/storage/providers/local.rs @@ -3,15 +3,23 @@ use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::utils::common::BackupMethod; +use crate::utils::edge_key::derive_key; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; use crate::utils::tus::upload_to_tus_stream_with_headers; +use anyhow::Result; use async_trait::async_trait; use reqwest::header::{HeaderMap, HeaderValue}; use std::sync::Arc; use tokio::fs; use tracing::error; +pub const LOCAL_STORAGE_USER: &str = "portabase"; + +pub fn local_storage_password(master_key_b64: &str) -> Result { + Ok(hex::encode(derive_key(master_key_b64, b"portabase/local-storage/v1")?)) +} + pub struct LocalProvider; #[async_trait] diff --git a/src/services/storage/providers/rclone/helpers.rs b/src/services/storage/providers/rclone/helpers.rs index 41458d6..98a1aeb 100644 --- a/src/services/storage/providers/rclone/helpers.rs +++ b/src/services/storage/providers/rclone/helpers.rs @@ -10,7 +10,9 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; use tracing::info; -const BLOCKED_BACKEND_TYPES: [&str; 13] = [ +/// Entries must not contain spaces: the type is compared with its spaces removed (rclone resolves a +/// backend by name, registered prefix, or name without spaces: `google photos` = `googlephotos`). +const BLOCKED_BACKEND_TYPES: [&str; 14] = [ "local", "alias", "crypt", @@ -24,67 +26,133 @@ const BLOCKED_BACKEND_TYPES: [&str; 13] = [ "memory", "http", "googlephotos", + "gphotos", ]; -fn sections(config_text: &str) -> Vec<(String, Option)> { - let mut out: Vec<(String, Option)> = Vec::new(); - - for line in config_text.lines() { - let line = line.trim(); - - if line.starts_with('[') && line.ends_with(']') && line.len() > 2 { - out.push((line[1..line.len() - 1].trim().to_string(), None)); - continue; - } - - let Some((key, value)) = line.split_once('=') else { - continue; - }; - - if key.trim().eq_ignore_ascii_case("type") - && let Some(current) = out.last_mut() - && current.1.is_none() - { - current.1 = Some(value.trim().to_ascii_lowercase()); - } - } +fn valid_remote_name(name: &str) -> bool { + let mut chars = name.chars(); + chars.next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '+' | '@' | '-')) +} - out +fn valid_key(key: &str) -> bool { + !key.is_empty() && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') } +fn forbidden_key(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + matches!( + key.as_str(), + "ssh" | "env_auth" | "use_msi" | "use_az" | "use_kerberos" | "kerberos_ccache" | "key_use_agent" | "set_env" | "unix_socket" + ) || key.ends_with("_command") + || key.ends_with("_file") + || key.ends_with("_path") +} +/// Strict check for user-pasted rclone configs (never for configs Portabase builds itself). +/// Exactly one `[remote]` of plain `key = value` lines: rclone and a naive parser must read it +/// identically, and nothing may run a local command or read host credentials. pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> { - let sections = sections(config_text); + let mut names = Vec::new(); + let mut pairs = Vec::new(); + let mut before_header = None; - if sections.is_empty() { - bail!("rclone config contains no remote sections"); + for (i, line) in config_text.split('\n').enumerate() { + let (n, line) = (i + 1, line.trim()); + if line.is_empty() || line.starts_with(['#', ';']) { + continue; + } + if line.starts_with('[') && line.ends_with(']') { + names.push(line[1..line.len() - 1].trim()); + } else if let Some((key, value)) = line.split_once('=').filter(|(k, _)| !k.is_empty()) { + if names.is_empty() { + before_header.get_or_insert(n); + } + pairs.push((key.trim(), value.trim())); + } else { + bail!("rclone config line {n} is not a [section] header or a 'key = value' pair"); + } } - for (name, backend) in §ions { - let Some(backend) = backend else { continue }; - if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) { - bail!("rclone backend type '{backend}' is not allowed (remote '{name}')"); + if let Some(n) = before_header { + bail!("rclone config line {n} appears before any [section]"); + } + if names.len() != 1 { + bail!("rclone config must contain exactly one [section] (found {})", names.len()); + } + let name = names[0]; + if !valid_remote_name(name) { + bail!("invalid rclone remote name '{name}'"); + } + for (key, _) in &pairs { + if !valid_key(key) { + bail!("invalid rclone config key '{key}' (remote '{name}')"); } } - - if !sections.iter().any(|(name, _)| name == remote_name) { - let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect(); - bail!( - "remote '{remote_name}' is not defined in the rclone config (available: {})", - available.join(", ") - ); + // rclone (goconfig) unquotes `value` and `"""value"""` and drops what follows the closing quote. + for (key, value) in &pairs { + if value.starts_with('`') || value.starts_with("\"\"\"") { + bail!("rclone config value for key '{key}' must not be quoted (remote '{name}')"); + } + } + let mut seen = std::collections::HashSet::new(); + for (key, _) in &pairs { + if !seen.insert(key.to_ascii_lowercase()) { + bail!("duplicate rclone config key '{key}' (remote '{name}')"); + } + } + for (key, _) in &pairs { + if forbidden_key(key) { + bail!("rclone config key '{key}' is not allowed (remote '{name}')"); + } + } + let Some((_, backend)) = pairs.iter().find(|(k, _)| k.eq_ignore_ascii_case("type")) else { + bail!("rclone remote '{name}' has no type"); + }; + let backend = backend.to_ascii_lowercase(); + if backend.is_empty() || !backend.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == ' ') { + bail!("invalid rclone backend type '{backend}' (remote '{name}')"); + } + let compact = backend.replace(' ', ""); + if BLOCKED_BACKEND_TYPES.contains(&compact.as_str()) { + bail!("rclone backend type '{backend}' is not allowed (remote '{name}')"); + } + // The other OCI providers read the host's OCI config or instance identity. Exact match on + // purpose: rclone 1.75.1 compares the key and the value case-sensitively, and falls back to + // host credentials for `PROVIDER = no_auth` (key ignored) and `provider = NO_AUTH` (unknown value). + if compact == "oracleobjectstorage" { + if !pairs.contains(&("provider", "no_auth")) { + bail!("rclone oracleobjectstorage remotes must use provider = no_auth (remote '{name}')"); + } + } + if remote_name != name { + bail!("remote '{remote_name}' is not defined in the rclone config (available: {name})"); } Ok(()) } +/// `rclone obscure -` reads the password from stdin, so it never shows up in argv. pub fn obscure_password(password: &str) -> Result { - let out = std::process::Command::new("rclone") + let mut child = std::process::Command::new("rclone") .arg("obscure") - .arg(password) - .output() + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .context("failed to spawn rclone (is the binary installed in this image?)")?; + // Dropping the handle closes stdin, which is what ends rclone's read. + child + .stdin + .take() + .context("rclone stdin unavailable")? + .write_all(password.as_bytes()) + .context("failed to write the password to rclone obscure")?; + + let out = child.wait_with_output().context("failed to wait for rclone obscure")?; + if !out.status.success() { bail!( "rclone obscure failed: {}", diff --git a/src/services/storage/providers/rclone/mod.rs b/src/services/storage/providers/rclone/mod.rs index de6df63..0004651 100644 --- a/src/services/storage/providers/rclone/mod.rs +++ b/src/services/storage/providers/rclone/mod.rs @@ -1,5 +1,6 @@ pub mod helpers; pub mod models; +pub mod target; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/rclone/target.rs b/src/services/storage/providers/rclone/target.rs new file mode 100644 index 0000000..9f13292 --- /dev/null +++ b/src/services/storage/providers/rclone/target.rs @@ -0,0 +1,137 @@ +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::storage::providers::azure_blob::models::AzureBlobProviderConfig; +use crate::services::storage::providers::google_cloud_storage::helpers::service_account_key; +use crate::services::storage::providers::google_cloud_storage::models::GoogleCloudStorageProviderConfig; +use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; +use crate::services::storage::providers::rclone::helpers::{build_rclone_config, validate_config}; +use crate::services::storage::providers::rclone::models::RcloneProviderConfig; +use crate::services::storage::providers::s3::models::S3ProviderConfig; +use crate::services::storage::providers::sftp::helpers::build_sftp_config; +use crate::services::storage::providers::sftp::models::SftpProviderConfig; +use anyhow::{Context, Result, bail}; +use tempfile::TempPath; + +/// A storage channel expressed as an rclone remote. Temp files the config refers +/// to (e.g. an sftp key) live as long as this value. +pub struct RcloneTarget { + pub config_text: String, + pub remote_name: String, + pub base_path: String, + _guards: Vec, +} + +impl RcloneTarget { + fn new(config_text: String, remote_name: &str, base_path: String) -> Self { + Self { config_text, remote_name: remote_name.to_string(), base_path, _guards: Vec::new() } + } +} + +pub fn rclone_target(storage: &DatabaseStorage) -> Result { + let config = storage.config.clone(); + match storage.provider.as_str() { + "s3" => s3(config.try_into().context("invalid s3 storage config")?), + "blob" => blob(config.try_into().context("invalid azure blob storage config")?), + "google-cloud-storage" => gcs(config.try_into().context("invalid google cloud storage config")?), + "google-drive" => drive(config.try_into().context("invalid google drive storage config")?), + "rclone" => user_rclone(config.try_into().context("invalid rclone storage config")?), + "sftp" => sftp(config.try_into().context("invalid sftp storage config")?), + "local" => bail!("a local storage channel cannot be used as an rclone target"), + other => bail!("unknown storage provider '{other}'"), + } +} + +fn s3(c: S3ProviderConfig) -> Result { + let scheme = if c.ssl { "https" } else { "http" }; + let endpoint = match c.port.as_deref().map(str::trim).filter(|p| !p.is_empty()) { + Some(port) => format!("{scheme}://{}:{port}", c.end_point_url), + None => format!("{scheme}://{}", c.end_point_url), + }; + let region = c.region.as_deref().map(str::trim).filter(|r| !r.is_empty()).unwrap_or("us-east-1"); + let fields = [ + ("type", "s3".to_string()), + ("provider", "Other".to_string()), + ("access_key_id", c.access_key), + ("secret_access_key", c.secret_key), + ("endpoint", endpoint), + ("region", region.to_string()), + ("force_path_style", "true".to_string()), + ("no_check_bucket", "true".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("s3", &fields)?, "s3", c.bucket_name)) +} + +fn blob(c: AzureBlobProviderConfig) -> Result { + let resolved = c.resolve().context("invalid azure blob storage config")?; + if resolved.account_key.trim().is_empty() { + bail!("azure blob needs an account key to be used as an rclone target (SAS-only connection strings are not supported)"); + } + let fields = [ + ("type", "azureblob".to_string()), + ("account", resolved.account_name), + ("key", resolved.account_key), + ("endpoint", resolved.blob_endpoint), + // Native uploads never create the container; neither should rclone. + ("no_check_container", "true".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("blob", &fields)?, "blob", c.container_name)) +} + +fn gcs(c: GoogleCloudStorageProviderConfig) -> Result { + let mut fields = vec![ + ("type", "google cloud storage".to_string()), + ("project_number", c.project_id.clone()), + ]; + match c.api_endpoint.as_deref().map(str::trim).filter(|e| !e.is_empty()) { + // A custom endpoint means the fake-gcs emulator, which does not verify credentials. + // rclone's `endpoint` is the JSON API base path, not the bare host. + Some(endpoint) => { + fields.push(("anonymous", "true".to_string())); + fields.push(("endpoint", format!("{}/storage/v1/", endpoint.trim_end_matches('/')))); + } + None => fields.push(( + "service_account_credentials", + serde_json::to_string(&service_account_key(&c))?, + )), + } + // Uniform bucket-level access (the GCS default) rejects rclone's default + // `predefinedAcl=private`; and native uploads never create the bucket. + fields.push(("bucket_policy_only", "true".to_string())); + fields.push(("no_check_bucket", "true".to_string())); + Ok(RcloneTarget::new( + build_rclone_config("google-cloud-storage", &fields)?, + "google-cloud-storage", + c.bucket_name, + )) +} + +fn drive(c: GoogleDriveProviderConfig) -> Result { + let token = serde_json::json!({ + "access_token": "", + "expiry": "0001-01-01T00:00:00Z", + "refresh_token": c.refresh_token, + "token_type": "Bearer", + }); + let fields = [ + ("type", "drive".to_string()), + ("client_id", c.client_id), + ("client_secret", c.client_secret), + ("scope", "drive.file".to_string()), + ("token", serde_json::to_string(&token)?), + ("root_folder_id", c.folder_id), + // Deletes (retention, restic prune) must free space, not fill the Drive trash. + ("use_trash", "false".to_string()), + ]; + Ok(RcloneTarget::new(build_rclone_config("google-drive", &fields)?, "google-drive", String::new())) +} + +fn user_rclone(c: RcloneProviderConfig) -> Result { + validate_config(&c.config_text, &c.remote_name)?; + Ok(RcloneTarget::new(c.config_text, &c.remote_name, c.remote_path)) +} + +fn sftp(c: SftpProviderConfig) -> Result { + let (config_text, key_file) = build_sftp_config(&c)?; + let mut target = RcloneTarget::new(config_text, "sftp", c.remote_path); + target._guards.extend(key_file.map(|f| f.into_temp_path())); + Ok(target) +} diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index 4d79f24..653f30f 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -1,4 +1,4 @@ -mod models; +pub mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; diff --git a/src/services/storage/providers/sftp/mod.rs b/src/services/storage/providers/sftp/mod.rs index 322da3c..4e166af 100644 --- a/src/services/storage/providers/sftp/mod.rs +++ b/src/services/storage/providers/sftp/mod.rs @@ -6,8 +6,7 @@ use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::services::storage::providers::rclone::helpers::{rcat, remote_target, write_config}; -use crate::services::storage::providers::sftp::helpers::build_sftp_config; -use crate::services::storage::providers::sftp::models::SftpProviderConfig; +use crate::services::storage::providers::rclone::target::rclone_target; use crate::utils::common::BackupMethod; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; @@ -53,19 +52,13 @@ impl StorageProvider for SftpProvider { } }; - let config: SftpProviderConfig = match storage.clone().config.try_into() { - Ok(c) => c, + // `target` owns the sftp key file: keep it alive until `rcat` returns. + let target = match rclone_target(storage) { + Ok(t) => t, Err(e) => { - error!("sftp config deserialization failed: {}", e); - return failed(&storage_id, e, Some(total_size)); - } - }; - - let (config_text, _key_file) = match build_sftp_config(&config) { - Ok(v) => v, - Err(e) => { - error!("sftp config build failed: {}", e); - return failed(&storage_id, e, Some(total_size)); + // `{:#}` keeps the whole anyhow chain (the serde cause), which is what the user sees. + error!("sftp config build failed: {e:#}"); + return failed(&storage_id, format!("{e:#}"), Some(total_size)); } }; @@ -82,7 +75,7 @@ impl StorageProvider for SftpProvider { let file_name = full_file_name(encrypt); let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref()); - let config_file = match write_config(&config_text) { + let config_file = match write_config(&target.config_text) { Ok(f) => f, Err(e) => { error!("sftp config write failed: {}", e); @@ -90,11 +83,11 @@ impl StorageProvider for SftpProvider { } }; - let target = remote_target("sftp", &config.remote_path, &remote_file_path); + let remote = remote_target(&target.remote_name, &target.base_path, &remote_file_path); - info!("Starting sftp (rclone) upload to {}", target); + info!("Starting sftp (rclone) upload to {}", remote); - match rcat(config_file.path(), &target, upload.stream).await { + match rcat(config_file.path(), &remote, upload.stream).await { Ok(()) => { info!("sftp upload successful: {}", remote_file_path); UploadResult { diff --git a/src/tests/domain/files/mod.rs b/src/tests/domain/files/mod.rs new file mode 100644 index 0000000..4e73066 --- /dev/null +++ b/src/tests/domain/files/mod.rs @@ -0,0 +1,619 @@ +mod restic; +mod sync; + +use crate::domain::files::matcher::ExcludeMatcher; +use std::path::Path; + +fn matcher(patterns: &[&str]) -> ExcludeMatcher { + let owned: Vec = patterns.iter().map(|p| p.to_string()).collect(); + ExcludeMatcher::new(&owned).unwrap() +} + +#[test] +fn matcher_unanchored_matches_any_depth() { + let m = matcher(&["node_modules", "*.log"]); + assert!(m.is_excluded(Path::new("node_modules"))); + assert!(m.is_excluded(Path::new("app/node_modules"))); + assert!(m.is_excluded(Path::new("x.log"))); + assert!(m.is_excluded(Path::new("a/b/x.log"))); + assert!(!m.is_excluded(Path::new("src/main.rs"))); +} + +#[test] +fn matcher_anchored_matches_root_only() { + let m = matcher(&["/cache", "/build/"]); + assert!(m.is_excluded(Path::new("cache"))); + assert!(!m.is_excluded(Path::new("app/cache"))); + assert!(m.is_excluded(Path::new("build"))); +} + +#[test] +fn matcher_star_does_not_cross_separator_but_double_star_does() { + let m = matcher(&["/logs/*.txt", "/data/**/tmp"]); + assert!(m.is_excluded(Path::new("logs/a.txt"))); + assert!(!m.is_excluded(Path::new("logs/sub/a.txt"))); + assert!(m.is_excluded(Path::new("data/tmp"))); + assert!(m.is_excluded(Path::new("data/a/b/tmp"))); +} + +#[test] +fn matcher_ignores_blank_patterns_and_rejects_invalid_ones() { + let m = matcher(&["", " "]); + assert!(!m.is_excluded(Path::new("anything"))); + assert!(ExcludeMatcher::new(&["[".to_string()]).is_err()); +} + +use crate::domain::files::backup; +use crate::services::backup::logger::JobLogger; +use crate::services::config::{DatabaseConfig, DbType}; +use flate2::read::GzDecoder; +use std::collections::HashMap; +use std::fs; +use std::io::Read; +use std::sync::Arc; + +pub(crate) fn files_config(root: &Path, exclude: &[&str]) -> DatabaseConfig { + let mut options = HashMap::new(); + options.insert("exclude".to_string(), serde_json::json!(exclude)); + DatabaseConfig { + name: "files-test".to_string(), + database: String::new(), + db_type: DbType::Files, + username: String::new(), + password: String::new(), + port: 0, + host: String::new(), + generated_id: uuid::Uuid::new_v4().to_string(), + path: root.to_string_lossy().into_owned(), + max_packet_size: String::new(), + volume_name: String::new(), + container_name: None, + options, + } +} + +fn sample_tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::create_dir_all(root.join("app/node_modules")).unwrap(); + fs::write(root.join("app/node_modules/dep.js"), "dep").unwrap(); + fs::write(root.join("app/index.js"), "app").unwrap(); + std::os::unix::fs::symlink("docs/a.txt", root.join("link")).unwrap(); +} + +#[tokio::test] +async fn backup_writes_tar_gz_without_excluded_paths() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let cfg = files_config(src.path(), &["node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + + let archive = backup::run( + cfg.clone(), + out.path().to_path_buf(), + Arc::new(JobLogger::new()), + ) + .await + .unwrap(); + + assert_eq!( + archive.file_name().unwrap().to_string_lossy(), + format!("{}.tar.gz", cfg.generated_id) + ); + + let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(&archive).unwrap())); + let mut files = Vec::new(); + let mut link_is_symlink = false; + let mut a_txt = String::new(); + for entry in ar.entries().unwrap() { + let mut entry = entry.unwrap(); + let name = entry + .path() + .unwrap() + .to_string_lossy() + .trim_end_matches('/') + .to_string(); + if name == "link" { + link_is_symlink = entry.header().entry_type().is_symlink(); + } + if name == "docs/a.txt" { + entry.read_to_string(&mut a_txt).unwrap(); + } + files.push(name); + } + assert!(files.contains(&"docs/a.txt".to_string())); + assert!(files.contains(&"app/index.js".to_string())); + assert!( + !files.iter().any(|n| n.contains("node_modules")), + "excluded path archived: {files:?}" + ); + assert!(link_is_symlink, "symlink must be stored as a link"); + assert_eq!(a_txt, "v1", "file content must be archived"); +} + +fn entry_names(archive: &Path) -> Vec { + let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(archive).unwrap())); + ar.entries() + .unwrap() + .map(|e| { + e.unwrap() + .path() + .unwrap() + .to_string_lossy() + .trim_end_matches('/') + .to_string() + }) + .collect() +} + +#[tokio::test] +async fn backup_does_not_archive_its_own_output() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let out_dir = src.path().join("backups"); + fs::create_dir_all(&out_dir).unwrap(); + let cfg = files_config(src.path(), &[]); + + let archive = backup::run(cfg, out_dir.clone(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + assert!(archive.exists()); + let names = entry_names(&archive); + assert!(names.contains(&"docs/a.txt".to_string())); + assert!( + !names.iter().any(|n| n.starts_with("backups")), + "own output archived: {names:?}" + ); +} + +#[tokio::test] +async fn backup_skips_sockets_with_a_warning() { + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let _listener = std::os::unix::net::UnixListener::bind(src.path().join("sock")).unwrap(); + let cfg = files_config(src.path(), &[]); + let out = tempfile::TempDir::new().unwrap(); + + let archive = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + let names = entry_names(&archive); + assert!(names.contains(&"docs/a.txt".to_string())); + assert!( + !names.iter().any(|n| n == "sock"), + "socket archived: {names:?}" + ); +} + +#[tokio::test] +async fn backup_fails_when_source_is_not_a_directory() { + let src = tempfile::TempDir::new().unwrap(); + let file = src.path().join("plain.txt"); + fs::write(&file, "x").unwrap(); + let cfg = files_config(&file, &[]); + let out = tempfile::TempDir::new().unwrap(); + + let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await; + + assert!( + result.is_err(), + "a file path must not produce an empty archive" + ); +} + +use crate::domain::files::restore; + +#[tokio::test] +async fn restore_is_a_mirror_that_keeps_excluded_paths() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &["node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + fs::create_dir_all(root.join("newdir/sub")).unwrap(); + fs::write(root.join("app/node_modules/dep.js"), "changed").unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert_eq!( + fs::read_to_string(root.join("app/index.js")).unwrap(), + "app" + ); + assert!( + !root.join("new.txt").exists(), + "file created after backup must be deleted" + ); + assert!( + !root.join("newdir").exists(), + "dir created after backup must be deleted" + ); + assert_eq!( + fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(), + "changed", + "excluded path must be left untouched" + ); + assert!( + fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +#[test] +fn ensure_restorable_rejects_dangerous_targets() { + assert!(restore::ensure_restorable(Path::new("/")).is_err()); + assert!(restore::ensure_restorable(Path::new("")).is_err()); + assert!(restore::ensure_restorable(Path::new("relative/dir")).is_err()); + assert!(restore::ensure_restorable(Path::new("/definitely/not/here")).is_err()); + let dir = tempfile::TempDir::new().unwrap(); + assert!(restore::ensure_restorable(dir.path()).is_ok()); + let escapes_to_root = dir.path().join("../../../../../../../../../.."); + assert!(restore::ensure_restorable(&escapes_to_root).is_err()); +} + +#[tokio::test] +async fn restore_refuses_corrupt_archive_without_touching_data() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let len = fs::metadata(&archive).unwrap().len(); + fs::OpenOptions::new() + .write(true) + .open(&archive) + .unwrap() + .set_len(len / 2) + .unwrap(); + fs::write(root.join("keep.txt"), "keep").unwrap(); + + let result = restore::run(cfg, archive, logger).await; + + assert!(result.is_err(), "a truncated archive must be refused"); + assert!( + root.join("keep.txt").exists(), + "data must survive a refused restore" + ); + assert!( + root.join("docs/a.txt").exists(), + "data must survive a refused restore" + ); +} + +#[tokio::test] +async fn ping_reports_directory_reachability() { + let dir = tempfile::TempDir::new().unwrap(); + let ok = files_config(dir.path(), &[]); + assert!(crate::domain::files::ping::run(ok).await.unwrap()); + let missing = files_config(Path::new("/definitely/not/here"), &[]); + assert!(!crate::domain::files::ping::run(missing).await.unwrap()); +} + +#[tokio::test] +async fn restore_survives_archive_stored_inside_the_source() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let tmp_dir = root.join(".tmp-restore"); + fs::create_dir_all(&tmp_dir).unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), tmp_dir.clone(), logger.clone()) + .await + .unwrap(); + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + + restore::run(cfg, archive.clone(), logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert_eq!( + fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(), + "dep" + ); + assert!(!root.join("new.txt").exists()); + assert!( + archive.exists(), + "the directory holding the archive must not be wiped" + ); +} + +#[tokio::test] +async fn restore_survives_archive_stored_directly_in_the_source() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let outside = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let archive = root.join("snapshot.tar.gz"); + fs::copy(&outside, &archive).unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + + restore::run(cfg, archive.clone(), logger).await.unwrap(); + + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); + assert!(!root.join("new.txt").exists()); + assert!(archive.exists(), "the archive itself must not be wiped"); +} + +#[tokio::test] +async fn restore_never_follows_symlinked_directories() { + let outside = tempfile::TempDir::new().unwrap(); + fs::write(outside.path().join("secret.txt"), "outside").unwrap(); + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + std::os::unix::fs::symlink(outside.path(), root.join("old_link")).unwrap(); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + std::os::unix::fs::symlink(outside.path(), root.join("late_link")).unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!( + fs::read_to_string(outside.path().join("secret.txt")).unwrap(), + "outside" + ); + assert!( + fs::symlink_metadata(root.join("late_link")).is_err(), + "link created after backup must be removed" + ); + assert!( + fs::symlink_metadata(root.join("old_link")) + .unwrap() + .file_type() + .is_symlink() + ); +} + +#[tokio::test] +async fn restore_refuses_archive_with_corrupt_gzip_trailer() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + // The last 8 bytes are CRC32 + ISIZE; tar itself never reads them. + let mut bytes = fs::read(&archive).unwrap(); + let idx = bytes.len() - 6; + bytes[idx] ^= 0xff; + fs::write(&archive, bytes).unwrap(); + fs::write(root.join("keep.txt"), "keep").unwrap(); + + assert!(restore::run(cfg, archive, logger).await.is_err()); + assert_eq!(fs::read_to_string(root.join("keep.txt")).unwrap(), "keep"); + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); +} + +#[tokio::test] +async fn restore_anchored_exclude_applies_through_a_symlinked_path() { + let base = tempfile::TempDir::new().unwrap(); + let real = base.path().join("real"); + fs::create_dir_all(real.join("app/node_modules")).unwrap(); + fs::create_dir_all(real.join("node_modules")).unwrap(); + fs::write(real.join("app/node_modules/x"), "x1").unwrap(); + fs::write(real.join("node_modules/y"), "y1").unwrap(); + let link = base.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let cfg = files_config(&link, &["/app/node_modules"]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + fs::write(real.join("app/node_modules/x"), "x2").unwrap(); + fs::write(real.join("node_modules/y"), "y2").unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert_eq!( + fs::read_to_string(real.join("app/node_modules/x")).unwrap(), + "x2", + "anchored exclude must be kept" + ); + assert_eq!( + fs::read_to_string(real.join("node_modules/y")).unwrap(), + "y1", + "top-level node_modules is not excluded" + ); +} + +#[test] +fn wipe_keeps_only_excluded_paths_on_a_single_filesystem() { + for one_file_system in [false, true] { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let m = matcher(&["node_modules"]); + + let failures = + restore::wipe_except_excluded(root, &m, one_file_system, &JobLogger::new()).unwrap(); + + assert_eq!(failures, 0); + + assert!(root.join("app/node_modules/dep.js").exists()); + assert!(!root.join("app/index.js").exists()); + assert!(!root.join("docs").exists()); + assert!(fs::symlink_metadata(root.join("link")).is_err()); + } +} + +use crate::domain::files::platform::is_root; +use std::os::unix::fs::PermissionsExt; + +#[test] +fn unpack_retries_without_metadata_when_ownership_cannot_be_set() { + if is_root() { + return; // root may chown to uid 0, so the first pass would not fail + } + let dir = tempfile::TempDir::new().unwrap(); + let archive = dir.path().join("root-owned.tar.gz"); + let gz = flate2::write::GzEncoder::new( + fs::File::create(&archive).unwrap(), + flate2::Compression::default(), + ); + let mut builder = tar::Builder::new(gz); + for (name, body) in [("a.txt", "alpha"), ("sub/b.txt", "beta")] { + let mut header = tar::Header::new_gnu(); + header.set_path(name).unwrap(); + header.set_uid(0); + header.set_gid(0); + header.set_mode(0o644); + header.set_size(body.len() as u64); + header.set_cksum(); + builder.append(&header, body.as_bytes()).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + let logger = JobLogger::new(); + + restore::unpack( + &fs::File::open(&archive).unwrap(), + target.path(), + true, + &logger, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(target.path().join("a.txt")).unwrap(), + "alpha" + ); + assert_eq!( + fs::read_to_string(target.path().join("sub/b.txt")).unwrap(), + "beta" + ); + assert!( + logger + .into_entries() + .iter() + .any(|e| e.level == "warn" && e.message.contains("retrying without metadata")), + "the chown failure must go through the fallback" + ); +} + +#[tokio::test] +async fn restore_extracts_even_when_some_entries_cannot_be_removed() { + if is_root() { + return; // root ignores directory permissions + } + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + fs::write(root.join("docs/a.txt"), "v2").unwrap(); + fs::write(root.join("new.txt"), "new").unwrap(); + let locked = root.join("locked"); + fs::create_dir(&locked).unwrap(); + fs::write(locked.join("stale.txt"), "stale").unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o555)).unwrap(); + + let result = restore::run(cfg, archive, logger).await; + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap(); + + let err = result.expect_err("an entry that could not be removed must fail the restore"); + assert!( + err.to_string().contains("could not be removed"), + "unexpected error: {err:#}" + ); + assert_eq!( + fs::read_to_string(root.join("docs/a.txt")).unwrap(), + "v1", + "the archive must still be extracted" + ); + assert_eq!( + fs::read_to_string(root.join("app/index.js")).unwrap(), + "app" + ); + assert!( + !root.join("new.txt").exists(), + "removable entries must still be wiped" + ); + assert!(locked.join("stale.txt").exists()); +} + +#[tokio::test] +async fn backup_fails_when_an_entry_cannot_be_read() { + if is_root() { + return; // root reads mode 0o000 files + } + let src = tempfile::TempDir::new().unwrap(); + sample_tree(src.path()); + let secret = src.path().join("secret.txt"); + fs::write(&secret, "secret").unwrap(); + fs::set_permissions(&secret, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(src.path(), &[]); + let out = tempfile::TempDir::new().unwrap(); + + let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await; + fs::set_permissions(&secret, fs::Permissions::from_mode(0o644)).unwrap(); + + let err = result.expect_err("an unreadable entry must fail the backup"); + assert!( + err.to_string().contains("1 entry could not be read"), + "unexpected error: {err:#}" + ); +} + +#[tokio::test] +async fn restore_keeps_special_files() { + let src = tempfile::TempDir::new().unwrap(); + let root = src.path(); + sample_tree(root); + let cfg = files_config(root, &[]); + let out = tempfile::TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + + let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone()) + .await + .unwrap(); + let _listener = std::os::unix::net::UnixListener::bind(root.join("sock")).unwrap(); + + restore::run(cfg, archive, logger).await.unwrap(); + + assert!( + fs::symlink_metadata(root.join("sock")).is_ok(), + "socket must survive a restore" + ); + assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1"); +} diff --git a/src/tests/domain/files/restic.rs b/src/tests/domain/files/restic.rs new file mode 100644 index 0000000..accd825 --- /dev/null +++ b/src/tests/domain/files/restic.rs @@ -0,0 +1,351 @@ +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::logger::JobLogger; +use crate::domain::files::restic::repo::is_snapshot_id; +use crate::domain::files::restic::excludes::{backup_excludes, glob_escape, restore_excludes}; +use crate::domain::files::restic::json::{self, BackupSummary, Line}; +use crate::domain::files::restic::password::derive_password; +use crate::tests::domain::files::files_config; +use crate::utils::edge_key::hkdf_sha256_32; +use serde_json::json; +use std::path::Path; + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|s| s.to_string()).collect() +} + +#[test] +fn snapshot_ids_are_full_lowercase_hex() { + assert!(is_snapshot_id(&"a1".repeat(32))); + for bad in ["--password-command=touch /tmp/pwned", &"A".repeat(64), &"a".repeat(63), &"a".repeat(65), ""] { + assert!(!is_snapshot_id(bad), "{bad}"); + } +} + +#[test] +fn hkdf_matches_rfc5869_case_3_first_block() { + let okm = hkdf_sha256_32(&[0x0b; 22], b""); + assert_eq!( + hex::encode(okm), + "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d" + ); +} + +#[test] +fn password_matches_the_dashboard_vector() { + let password = derive_password( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + "test-generated-id", + ) + .unwrap(); + assert_eq!( + password, + "577e00efe6c953e8583d90e14d78109d805b616164279b8182282db76033b090" + ); +} + +#[test] +fn password_rejects_a_non_base64_master_key() { + assert!(derive_password("not base64!", "x").is_err()); +} + +#[test] +fn backup_excludes_are_rooted_at_the_source() { + let got = backup_excludes( + Path::new("/data/files"), + &strings(&["*.log", "node_modules/", "/build/", "/docs/**/*.md", " ", "/"]), + ) + .unwrap(); + assert_eq!( + got, + strings(&[ + "/data/files/**/*.log", + "/data/files/**/node_modules", + "/data/files/build", + "/data/files/docs/**/*.md", + ]) + ); +} + +#[test] +fn backup_excludes_escape_glob_characters_in_the_root() { + let got = backup_excludes(Path::new("/data/src[1]"), &strings(&["tmp"])).unwrap(); + assert_eq!(got, strings(&["/data/src\\[1\\]/**/tmp"])); +} + +#[test] +fn restore_excludes_are_relative_to_the_snapshot_subfolder() { + let got = restore_excludes(&strings(&["*.log", "/build/", "a/b"])).unwrap(); + assert_eq!(got, strings(&["/**/*.log", "/build", "/**/a/b"])); +} + +#[test] +fn bracket_negation_is_rewritten_for_restic() { + assert_eq!( + backup_excludes(Path::new("/data"), &strings(&["[!a]*.log", "/x/[!0-9]", r"\[!a]", "[a[!]"])).unwrap(), + strings(&["/data/**/[^a]*.log", "/data/x/[^0-9]", r"/data/**/\[!a]", "/data/**/[a[!]"]) + ); + assert_eq!(restore_excludes(&strings(&["/[!a]*", "[!a]"])).unwrap(), strings(&["/[^a]*", "/**/[^a]"])); +} + +#[test] +fn brace_patterns_are_rejected_on_both_sides() { + let patterns = strings(&["{x,y}.dat"]); + let err = backup_excludes(Path::new("/data"), &patterns).unwrap_err(); + assert!(err.to_string().contains("brace patterns are not supported in Snapshots mode")); + assert!(restore_excludes(&patterns).is_err()); +} + +#[test] +fn glob_escape_escapes_every_metacharacter() { + assert_eq!(glob_escape(r"a*b?c[d]e\f"), r"a\*b\?c\[d\]e\\f"); +} + +#[test] +fn json_lines_are_classified() { + let status = r#"{"message_type":"status","percent_done":0.9999997777778271,"total_files":4,"files_done":2,"total_bytes":9000002,"bytes_done":9000000,"error_count":1}"#; + let error = r#"{"message_type":"error","error":{"message":"open /src/u: permission denied"},"during":"archival","item":"/src/u"}"#; + let exit = r#"{"message_type":"exit_error","code":3,"message":"Warning: at least one source file could not be read"}"#; + + assert_eq!(json::parse(status), Line::Status { percent_done: 0.9999997777778272 }); + assert_eq!( + json::parse(error), + Line::Error { message: "open /src/u: permission denied".into(), item: Some("/src/u".into()) } + ); + assert_eq!( + json::parse(exit), + Line::ExitError { code: 3, message: "Warning: at least one source file could not be read".into() } + ); + assert_eq!(json::parse("Fatal: wrong password"), Line::Text("Fatal: wrong password".into())); + assert_eq!(json::parse("[]"), Line::Text("[]".into())); +} + +#[test] +fn backup_summary_deserializes_from_a_recorded_line() { + let line = r#"{"message_type":"summary","files_new":3,"files_changed":0,"files_unmodified":0,"dirs_new":9,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":8,"tree_blobs":10,"data_added":9005859,"data_added_packed":9004808,"total_files_processed":3,"total_bytes_processed":9000000,"total_duration":0.78928,"backup_start":"2026-10-03T16:59:06.569467+02:00","backup_end":"2026-10-03T16:59:07.359124+02:00","snapshot_id":"05ba7db8d08e4e097b869dbc511a8b8936298b2fcb01b221115789491204a470"}"#; + let Line::Summary(value) = json::parse(line) else { panic!("not a summary") }; + let summary: BackupSummary = serde_json::from_value(value).unwrap(); + assert_eq!(summary.snapshot_id.len(), 64); + assert_eq!(summary.data_added_packed, 9004808); + assert_eq!(summary.total_bytes_processed, 9000000); + assert_eq!(summary.files_new, 3); +} + +#[test] +fn removed_files_come_from_the_parent_summary() { + use crate::domain::files::restic::json::{Snapshot, files_removed}; + let listing: Vec = serde_json::from_value(serde_json::json!([ + {"id": "p", "paths": ["/src"], "summary": {"total_files_processed": 3, "files_new": 3}}, + {"id": "c", "paths": ["/src"], "parent": "p"}, + {"id": "orphan", "paths": ["/src"], "parent": "forgotten"}, + {"id": "old", "paths": ["/src"]}, + {"id": "child-of-old", "paths": ["/src"], "parent": "old"}, + ])) + .unwrap(); + assert_eq!(files_removed(&listing, "c", 0, 2), Some(1)); + assert_eq!(files_removed(&listing, "p", 0, 0), Some(0), "no parent"); + assert_eq!(files_removed(&listing, "orphan", 0, 2), None, "parent not listed"); + assert_eq!(files_removed(&listing, "child-of-old", 0, 2), None, "parent without summary"); + assert_eq!(files_removed(&listing, "missing", 0, 2), None); +} + +#[test] +fn a_local_channel_opens_the_dashboard_rest_repository() { + use crate::utils::edge_key::EdgeKey; + let storage: DatabaseStorage = + serde_json::from_value(json!({ "id": "storage-1", "provider": "local", "config": {} })).unwrap(); + let edge_key = EdgeKey { + server_url: "http://dash:8887/".into(), + agent_id: "agent-1".into(), + master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(), + }; + let repo = ResticRepo::open(&storage, "g1", &edge_key).unwrap(); + assert_eq!(repo.repository(), "rest:http://dash:8887/storage/restic/g1/"); + assert!(repo.is_append_only()); +} + +use crate::domain::files::platform::is_root; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; +use std::fs; +use tempfile::TempDir; + +fn local_repo(dir: &Path, generated_id: &str) -> ResticRepo { + fs::create_dir_all(dir).unwrap(); + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + ResticRepo::new(&config, "pb", "", "backups", generated_id, "test-password", dir.join("cache")).unwrap() +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); + fs::create_dir_all(root.join("build")).unwrap(); + fs::write(root.join("build/out.bin"), "out").unwrap(); + fs::write(root.join("app.log"), "log").unwrap(); +} + +fn files(root: &Path) -> Vec { + let mut out: Vec = walkdir::WalkDir::new(root) + .min_depth(1) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| { + let rel = e.path().strip_prefix(root).unwrap().display().to_string(); + format!("{rel}={}", fs::read_to_string(e.path()).unwrap()) + }) + .collect(); + out.sort(); + out +} + +#[tokio::test] +async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_parent() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + + let first = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + assert_eq!(first.snapshot_id.len(), 64); + assert_eq!(first.files_new, 4); + assert_eq!(first.files_removed, Some(0)); + assert!(base.path().join("repo/backups/restic").join(&cfg.generated_id).join("config").exists()); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + let second = snapshot(&repo, &cfg, "bs-2", &logger).await.unwrap(); + assert_eq!(second.files_changed, 1); + assert_eq!(second.files_unmodified, 3); + assert_eq!(second.files_removed, Some(0)); + + let tagged = repo.run(["snapshots", "--json", "--tag", "bs:bs-2"], &logger).await.unwrap(); + let listed: serde_json::Value = serde_json::from_str(tagged.stdout.trim()).unwrap(); + let listed = listed.as_array().unwrap(); + assert_eq!(listed.len(), 1, "{}", tagged.stdout); + assert_eq!(listed[0]["id"], second.snapshot_id.as_str()); + assert_eq!(listed[0]["parent"], first.snapshot_id.as_str()); + + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + let third = snapshot(&repo, &cfg, "bs-3", &logger).await.unwrap(); + assert_eq!((third.files_new, third.files_changed, third.files_unmodified), (0, 0, 3)); + assert_eq!(third.files_removed, Some(1)); + assert_eq!(third.report()["filesRemoved"], 1); +} + +#[tokio::test] +async fn restore_is_a_mirror_that_keeps_excluded_paths() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + fs::write(src.path().join("docs/new.log"), "kept").unwrap(); + fs::write(src.path().join("build/new.bin"), "kept").unwrap(); + + restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap(); + + assert_eq!( + files(src.path()), + vec![ + "app.log=log", + "build/new.bin=kept", + "build/out.bin=out", + "docs/a.txt=v1", + "docs/b.txt=b", + "docs/new.log=kept", + ] + ); +} + +#[tokio::test] +async fn unreadable_file_fails_the_snapshot_and_forgets_it() { + if is_root() { + return; + } + use std::os::unix::fs::PermissionsExt; + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let locked = src.path().join("docs/b.txt"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + + let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("could not be read"), "{err:#}"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert_eq!(listed.stdout.trim(), "[]"); +} + +#[tokio::test] +async fn unanchored_exclude_never_matches_directories_above_the_source() { + let base = TempDir::new().unwrap(); + let src = base.path().join("tmp/src"); + fs::create_dir_all(src.join("tmp")).unwrap(); + fs::write(src.join("kept.txt"), "k").unwrap(); + fs::write(src.join("tmp/dropped.txt"), "d").unwrap(); + let cfg = files_config(&src, &["tmp"]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + + let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + assert_eq!(snap.files_new, 1); +} + +#[tokio::test] +async fn restoring_an_unknown_snapshot_fails_without_touching_the_target() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let repo = local_repo(&base.path().join("repo"), &cfg.generated_id); + let logger = JobLogger::new(); + snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + let before = files(src.path()); + + let err = restore(&repo, &cfg, &"deadbeef".repeat(8), &logger).await.unwrap_err(); + + assert!(err.to_string().contains("not found"), "{err:#}"); + assert_eq!(files(src.path()), before); +} + +#[tokio::test] +async fn wrong_password_is_reported_explicitly() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let dir = base.path().join("repo"); + snapshot(&local_repo(&dir, &cfg.generated_id), &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + let other = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "other", dir.join("cache")).unwrap(); + let err = snapshot(&other, &cfg, "bs-2", &JobLogger::new()).await.unwrap_err(); + assert!(err.to_string().contains("wrong repository password"), "{err:#}"); +} + +#[tokio::test] +async fn an_unusable_cache_dir_runs_restic_without_cache() { + let base = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let dir = base.path().join("repo"); + fs::create_dir_all(&dir).unwrap(); + let blocker = tempfile::NamedTempFile::new().unwrap(); + let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display()); + let repo = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "test-password", blocker.path().join("cache")).unwrap(); + snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); +} diff --git a/src/tests/domain/files/sync.rs b/src/tests/domain/files/sync.rs new file mode 100644 index 0000000..f1baea8 --- /dev/null +++ b/src/tests/domain/files/sync.rs @@ -0,0 +1,275 @@ +use crate::domain::files::sync::excludes::sync_excludes; +use crate::domain::files::sync::stats::{SyncStats, parse_log, progress}; + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|s| s.to_string()).collect() +} + +#[test] +fn sync_excludes_translate_p0_patterns() { + let got = sync_excludes(&strings(&[ + "*.log", "node_modules/", "/build/", "**/tmp", "/docs/**/*.md", "[!a].txt", "{x,y}.dat", " ", "/", + ])); + assert_eq!( + got, + strings(&[ + "*.log", "*.log/**", + "node_modules", "node_modules/**", + "/build", "/build/**", + "tmp", "tmp/**", + "/docs/**/*.md", "/docs/**/*.md/**", "/docs/*.md", "/docs/*.md/**", + "[^a].txt", "[^a].txt/**", + "{x,y}.dat", "{x,y}.dat/**", + ]) + ); +} + +#[test] +fn every_double_star_gap_gets_a_zero_directory_variant() { + assert_eq!( + sync_excludes(&strings(&["a/**/b/**/c"])), + strings(&[ + "a/**/b/**/c", "a/**/b/**/c/**", + "a/b/**/c", "a/b/**/c/**", + "a/**/b/c", "a/**/b/c/**", + "a/b/c", "a/b/c/**", + ]) + ); +} + +#[test] +fn anchored_leading_double_star_also_matches_the_root() { + assert_eq!(sync_excludes(&strings(&["/**/x"])), strings(&["/**/x", "/**/x/**", "/x", "/x/**"])); +} + +#[test] +fn escaped_bracket_is_left_alone() { + assert_eq!(sync_excludes(&strings(&[r"\[!a]"])), strings(&[r"\[!a]", r"\[!a]/**"])); +} + +#[test] +fn parse_log_reads_the_final_stats_and_error_lines() { + let stderr = concat!( + r#"{"level":"error","msg":"Failed to copy: couldn't copy from /src/secret: errno -1","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"not deleting files as there were IO errors","source":"sync/sync.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"notice","msg":"\nTransferred: 0 B\n","source":"accounting/stats.go:1","stats":{"bytes":0,"checks":12,"deletes":0,"errors":1,"fatalError":false,"lastError":"couldn't copy from /src/secret: errno -1","transfers":0},"time":"2026-10-03T19:44:59+02:00"}"#, "\n", + ); + let log = parse_log(stderr); + assert_eq!( + log.stats, + Some(SyncStats { errors: 1, last_error: Some("couldn't copy from /src/secret: errno -1".into()), ..Default::default() }) + ); + assert_eq!(log.errors.len(), 2); + assert_eq!(log.errors[1], "not deleting files as there were IO errors"); +} + +#[test] +fn parse_log_reads_a_successful_run() { + let stderr = r#"{"level":"notice","msg":"\nTransferred: 2 B\n","source":"accounting/stats.go:1","stats":{"bytes":2,"checks":5,"deletes":1,"errors":0,"fatalError":false,"transfers":1},"time":"2026-10-03T19:45:10+02:00"}"#; + let log = parse_log(stderr); + assert_eq!(log.stats, Some(SyncStats { bytes: 2, transfers: 1, deletes: 1, ..Default::default() })); + assert!(log.errors.is_empty()); +} + +#[test] +fn progress_formats_periodic_stats_and_ignores_other_lines() { + let line = r#"{"level":"notice","msg":"stats","stats":{"bytes":536870912,"totalBytes":1073741824,"transfers":1200,"totalTransfers":2924,"checks":3,"totalChecks":10,"deletes":0,"speed":89128960.5,"eta":6}}"#; + assert_eq!( + progress(line).unwrap(), + "rclone sync: 512 MB / 1024 MB (50%), 1200/2924 transferred, 3/10 checked, 0 deleted, 85 MB/s, ETA 6s" + ); + let idle = r#"{"level":"notice","stats":{"bytes":0,"totalBytes":0,"speed":0,"eta":null}}"#; + assert_eq!(progress(idle).unwrap(), "rclone sync: 0 B / 0 B (100%), 0/0 transferred, 0/0 checked, 0 deleted, 0 B/s"); + assert!(progress(r#"{"level":"error","msg":"boom"}"#).is_none()); + assert!(progress("plain text").is_none()); +} + +#[test] +fn parse_log_keeps_plain_text_lines_as_errors() { + let log = parse_log("Error: unknown flag: --bogus\n"); + assert!(log.stats.is_none()); + assert_eq!(log.errors, strings(&["Error: unknown flag: --bogus"])); +} + +#[test] +fn parse_log_accepts_critical_error_level() { + let stderr = r#"{"level":"critical","msg":"Failed to create file system for \":sftp,host=127.0.0.1,port=1,user=x:/tmp/dst\": NewFs: couldn't connect SSH: dial tcp 127.0.0.1:1: connect: connection refused","source":"cmd/cmd.go:148"}"#; + let log = parse_log(stderr); + assert!(log.stats.is_none()); + assert_eq!(log.errors.len(), 1); + assert!(log.errors[0].starts_with("Failed to create file system")); +} + +#[test] +fn parse_log_deduplicates_repeated_error_lines() { + let stderr = concat!( + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n", + ); + let log = parse_log(stderr); + assert_eq!(log.errors.len(), 1); + assert_eq!(log.errors[0], "connection timeout"); +} + +#[test] +fn sync_excludes_collapses_repeated_double_stars() { + assert_eq!( + sync_excludes(&strings(&["a/**/**/b"])), + strings(&["a/**/b", "a/**/b/**", "a/b", "a/b/**"]) + ); +} + +use crate::domain::files::platform::is_root; +use crate::services::backup::logger::JobLogger; +use crate::domain::files::sync::rclone::{replica_target, sync_dir}; +use crate::tests::domain::files::files_config; +use std::fs; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; + +fn alias(dir: &Path) -> String { + fs::create_dir_all(dir).unwrap(); + format!("[pb]\ntype = alias\nremote = {}\n", dir.display()) +} + +fn replica(store: &Path, generated_id: &str) -> PathBuf { + store.join("backups/sync").join(generated_id).join("current") +} + +fn files(root: &Path) -> Vec { + let mut out: Vec = walkdir::WalkDir::new(root) + .min_depth(1) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| format!("{}={}", e.path().strip_prefix(root).unwrap().display(), fs::read_to_string(e.path()).unwrap())) + .collect(); + out.sort(); + out +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("docs/a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); + fs::create_dir_all(root.join("build")).unwrap(); + fs::write(root.join("build/out.bin"), "out").unwrap(); + fs::write(root.join("app.log"), "log").unwrap(); +} + +#[test] +fn replica_target_follows_the_path_convention() { + assert_eq!(replica_target("s3", "bucket", "backups", "g1"), "s3:bucket/backups/sync/g1/current"); + assert_eq!(replica_target("pb", "", "backups", "g1"), "pb:backups/sync/g1/current"); +} + +#[tokio::test] +async fn sync_mirrors_the_source_and_propagates_changes() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let config = alias(store.path()); + let logger = JobLogger::new(); + + let first = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); + assert_eq!(first.transfers, 2); + assert_eq!(first.replica_bytes, Some(3), "excluded files are not counted"); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1", "docs/b.txt=b"]); + + fs::write(src.path().join("docs/a.txt"), "v2").unwrap(); + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + fs::write(src.path().join("new.txt"), "new").unwrap(); + + let second = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap(); + assert_eq!(second.transfers, 2); + assert_eq!(second.deletes, 1); + assert_eq!(second.replica_bytes, Some(5)); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v2", "new.txt=new"]); +} + +#[tokio::test] +async fn a_later_exclude_removes_files_from_the_replica() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &["*.log", "/build"]); + let config = alias(store.path()); + let dest = replica_target("pb", "", "backups", &cfg.generated_id); + sync_dir(&config, &dest, &cfg, &JobLogger::new()).await.unwrap(); + + let mut later = cfg.clone(); + later.options = files_config(src.path(), &["*.log", "/build", "b.txt"]).options; + let stats = sync_dir(&config, &dest, &later, &JobLogger::new()).await.unwrap(); + assert_eq!(stats.deletes, 1); + assert_eq!(stats.replica_bytes, Some(2)); + assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1"]); +} + +#[tokio::test] +async fn failed_sync_keeps_files_the_failed_run_would_delete() { + if is_root() { + return; + } + use std::os::unix::fs::PermissionsExt; + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + tree(src.path()); + let cfg = files_config(src.path(), &[]); + let config = alias(store.path()); + sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap(); + + fs::remove_file(src.path().join("docs/b.txt")).unwrap(); + let locked = src.path().join("docs/a.txt"); + fs::write(&locked, "changed and now unreadable").unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let err = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("rclone sync failed"), "{err:#}"); + assert!(replica(store.path(), &cfg.generated_id).join("docs/b.txt").exists()); +} + +#[tokio::test] +async fn empty_source_is_refused_and_the_replica_is_untouched() { + let store = TempDir::new().unwrap(); + let src = TempDir::new().unwrap(); + let cfg = files_config(src.path(), &[]); + let err = sync_dir(&alias(store.path()), &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err(); + assert!(err.to_string().contains("is empty"), "{err:#}"); + assert!(!store.path().join("backups").exists()); +} + +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::domain::files::sync::backup::replica_path; +use serde_json::json; + +#[test] +fn replica_path_uses_the_channel_folder() { + let custom: DatabaseStorage = + serde_json::from_value(json!({ "id": "s", "provider": "s3", "folderName": "/pb/", "config": {} })).unwrap(); + assert_eq!(replica_path(&custom, "g1"), "pb/sync/g1/current"); + let default: DatabaseStorage = + serde_json::from_value(json!({ "id": "s", "provider": "s3", "config": {} })).unwrap(); + assert_eq!(replica_path(&default, "g1"), "backups/sync/g1/current"); +} + +#[test] +fn local_sync_config_points_at_the_dashboard_webdav_server() { + use crate::domain::files::sync::backup::local_sync_config; + use crate::utils::edge_key::EdgeKey; + let edge_key = EdgeKey { + server_url: "http://dash:8887/".into(), + agent_id: "agent-1".into(), + master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(), + }; + let text = local_sync_config(&edge_key).unwrap(); + assert!( + text.starts_with( + "[pblocal]\ntype = webdav\nurl = http://dash:8887/storage/sync\nvendor = rclone\nuser = portabase\npass = " + ), + "{text}" + ); + assert!(!text.contains("8abfb788"), "the password must be obscured: {text}"); +} diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index e467fdd..61f114b 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -8,3 +8,4 @@ mod valkey; mod firebird; mod mssql; mod docker_volume; +pub(crate) mod files; diff --git a/src/tests/services/api_client_tests.rs b/src/tests/services/api_client_tests.rs new file mode 100644 index 0000000..15275f5 --- /dev/null +++ b/src/tests/services/api_client_tests.rs @@ -0,0 +1,113 @@ +use crate::services::api::ApiClient; +use crate::services::api::client::{derive_agent_key, sign_request}; + +use base64::{Engine as _, engine::general_purpose}; +use reqwest::Method; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const AGENT_ID: &str = "11111111-1111-4111-8111-111111111111"; + +fn vector_master_key() -> Vec { + (0u8..32).collect() +} + +#[test] +fn derived_agent_key_matches_the_dashboard_vector() { + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + + assert_eq!( + hex::encode(key), + "16bee8dcf9d5f497cdd882ca208ff7f630ca02a5ecfe157f34239838fdcd9006" + ); +} + +#[test] +fn request_signature_matches_the_dashboard_vector() { + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + let path = format!("/api/agent/{AGENT_ID}/status"); + + assert_eq!( + sign_request(&key, "POST", &path, "1760968800", br#"{"version":"1.24.0","databases":[]}"#), + "ad1e56a50ac0c4a3f539f3180d2957ebeb9a61bd1df44e57b50f8e59dd269069" + ); + assert_eq!( + sign_request(&key, "POST", &path, "1760968800", "{\"name\":\"Base é\"}".as_bytes()), + "d3e6911c9273e0224b8dbae72cabdd3720973f1866c651fb4c99e1ac03e73231" + ); +} + +#[test] +fn invalid_master_key_logs_an_error_before_panicking() { + #[derive(Clone, Default)] + struct Logs(Arc>>); + impl std::io::Write for Logs { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let logs = Logs::default(); + let writer = logs.clone(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + + let result = tracing::subscriber::with_default(subscriber, || { + std::panic::catch_unwind(|| ApiClient::new("http://dashboard").with_agent_auth(AGENT_ID, "not base64!")) + }); + + assert!(result.is_err(), "an invalid master key must stop the agent"); + let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!(output.contains("ERROR") && output.contains("Invalid EDGE_KEY"), "logs: {output}"); +} + +#[tokio::test] +async fn every_request_is_signed_with_a_fresh_timestamp() { + let server = MockServer::start().await; + let route = format!("/agent/{AGENT_ID}/status"); + Mock::given(method("POST")) + .and(path(route.as_str())) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + + let client = ApiClient::new(server.uri()) + .with_agent_auth(AGENT_ID, &general_purpose::STANDARD.encode(vector_master_key())); + let body = serde_json::json!({"version": "1.24.0", "databases": [{"name": "Base é"}]}); + + for i in 0..2 { + if i > 0 { + tokio::time::sleep(Duration::from_millis(1100)).await; + } + let _: Option = client + .request_with_body(Method::POST, &route, &body) + .await + .expect("signed request accepted by the mock"); + } + + let key = derive_agent_key(&vector_master_key(), AGENT_ID); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + let requests = server.received_requests().await.unwrap(); + let mut timestamps = Vec::new(); + for req in &requests { + let ts = req.headers.get("x-agent-timestamp").unwrap().to_str().unwrap(); + assert!(ts.chars().all(|c| c.is_ascii_digit()), "timestamp {ts}"); + assert!(now.abs_diff(ts.parse::().unwrap()) <= 5); + assert_eq!(req.headers.get("content-type").unwrap(), "application/json"); + assert_eq!( + req.headers.get("x-agent-signature").unwrap().to_str().unwrap(), + sign_request(&key, "POST", &format!("/api{route}"), ts, &req.body) + ); + timestamps.push(ts.to_string()); + } + assert_eq!(requests.len(), 2); + assert_ne!(timestamps[0], timestamps[1], "each call signs with its own timestamp"); +} diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index a4f8e87..5c62f5d 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -228,6 +228,72 @@ fn resolve_dashboard_config_noop_when_not_encrypted() { assert!(status.resolved_config.is_none()); } +#[test] +fn ping_without_engine_fields_defaults_to_none() { + let status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": false, "file": null, "metaFile": null, "size": null } } + })) + .unwrap(); + assert!(status.data.backup.engine.is_none()); + assert!(status.data.restore.engine.is_none()); + assert!(status.data.restore.snapshot_id.is_none()); + assert!(status.data.restore.storage.is_none()); +} + +#[test] +fn resolve_restore_storage_decrypts_the_snapshot_channel() { + use crate::services::status::resolve_restore_storage; + use base64::{engine::general_purpose, Engine}; + + let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]); + let channel = r#"[{"id":"ch-1","provider":"s3","folderName":"backups","config":{"endPointUrl":"s3.example.com","bucketName":"b"}}]"#; + let snapshot_id = "ab".repeat(32); + + let mut status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { + "backup": { "action": false, "cron": "0 * * * *", "engine": "restic" }, + "restore": { "action": true, "file": null, "metaFile": null, "size": null, + "engine": "restic", "snapshotId": snapshot_id, + "storageCiphertext": encrypt_json_gcm(channel.as_bytes(), &master_key_b64) } + } + })) + .unwrap(); + + assert_eq!(status.data.backup.engine.as_deref(), Some("restic")); + assert_eq!(status.data.restore.snapshot_id.as_deref(), Some(snapshot_id.as_str())); + + resolve_restore_storage(&mut status, &master_key_b64).unwrap(); + + let storage = status.data.restore.storage.expect("decrypted"); + assert_eq!(storage.id, "ch-1"); + assert_eq!(storage.provider, "s3"); + assert_eq!(storage.folder_name.as_deref(), Some("backups")); +} + +#[test] +fn resolve_restore_storage_rejects_a_bad_ciphertext() { + use crate::services::status::resolve_restore_storage; + use base64::{engine::general_purpose, Engine}; + + let mut status: DatabaseStatus = serde_json::from_value(json!({ + "dbms": "files", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": true, "engine": "restic", "storageCiphertext": "bm9wZQ==" } } + })) + .unwrap(); + assert!(resolve_restore_storage(&mut status, &general_purpose::STANDARD.encode([7u8; 32])).is_err()); + assert!(status.data.restore.storage.is_none()); +} + fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { use aes_gcm::aead::{Aead, KeyInit}; use aes_gcm::{Aes256Gcm, Key, Nonce}; @@ -243,3 +309,41 @@ fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { data.extend_from_slice(&ct); general_purpose::STANDARD.encode(data) } + +#[test] +fn ping_payload_sends_method_only_when_set() { + use crate::services::api::endpoints::status::DatabasePayload; + let payload = |method: Option<&'static str>| DatabasePayload { + name: "docs", + dbms: "files", + generated_id: "g", + ping_status: true, + method, + }; + let with = serde_json::to_value(payload(Some("sync"))).unwrap(); + assert_eq!(with["method"], "sync"); + let without = serde_json::to_value(payload(None)).unwrap(); + assert!(without.get("method").is_none(), "{without}"); +} + +#[test] +fn upload_status_sends_stats_only_when_set() { + use crate::services::api::endpoints::agent::backup::upload::status::StatusUploadRequest; + use crate::domain::files::sync::stats::SyncStats; + let request = |stats: Option| StatusUploadRequest { + generated_id: "g".into(), + backup_storage_id: "bs".into(), + status: "success".into(), + path: "p".into(), + size: 3, + backup_id: "b".into(), + stats, + }; + let sync = SyncStats { transfers: 2, deletes: 1, replica_bytes: Some(8), ..Default::default() }; + let with = serde_json::to_value(request(Some(sync.report()))).unwrap(); + assert_eq!(with["stats"], serde_json::json!({"filesTransferred": 2, "filesDeleted": 1, "replicaBytes": 8})); + let unmeasured = SyncStats::default().report(); + assert!(unmeasured["replicaBytes"].is_null(), "{unmeasured}"); + let without = serde_json::to_value(request(None)).unwrap(); + assert!(without.get("stats").is_none(), "{without}"); +} diff --git a/src/tests/services/backup_uploader_tests.rs b/src/tests/services/backup_uploader_tests.rs index f23b99f..3588d48 100644 --- a/src/tests/services/backup_uploader_tests.rs +++ b/src/tests/services/backup_uploader_tests.rs @@ -20,7 +20,7 @@ use tempfile::NamedTempFile; use wiremock::matchers::{body_partial_json, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; -fn ctx_pointing_at(base_url: String) -> Context { +pub(crate) fn ctx_pointing_at(base_url: String) -> Context { Context { edge_key: EdgeKey { server_url: String::new(), diff --git a/src/tests/services/config_tests.rs b/src/tests/services/config_tests.rs index 6ceb172..6c29465 100644 --- a/src/tests/services/config_tests.rs +++ b/src/tests/services/config_tests.rs @@ -1,7 +1,7 @@ use crate::core::context::Context; use crate::services::api::ApiClient; use crate::services::config::ConfigService; -use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig}; +use crate::services::config::{DatabasesConfig, InputDatabaseConfig, build_config, files_method}; use crate::utils::edge_key::EdgeKey; use std::io::Write; use std::sync::Arc; @@ -176,24 +176,39 @@ fn keep_ownership_extraction_logic() { // true → keep ownership let mut opts: HashMap = HashMap::new(); opts.insert("keep_ownership".to_string(), Value::Bool(true)); - let keep = opts.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep = opts + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(keep, "should keep ownership when flag is true"); // false → strip let mut opts2: HashMap = HashMap::new(); opts2.insert("keep_ownership".to_string(), Value::Bool(false)); - let keep2 = opts2.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep2 = opts2 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep2, "should strip when flag is false"); // missing → strip let opts3: HashMap = HashMap::new(); - let keep3 = opts3.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + let keep3 = opts3 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep3, "should strip when key absent"); // wrong type → strip let mut opts4: HashMap = HashMap::new(); - opts4.insert("keep_ownership".to_string(), Value::String("yes".to_string())); - let keep4 = opts4.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false); + opts4.insert( + "keep_ownership".to_string(), + Value::String("yes".to_string()), + ); + let keep4 = opts4 + .get("keep_ownership") + .and_then(|v| v.as_bool()) + .unwrap_or(false); assert!(!keep4, "should strip when value is not bool"); } @@ -258,7 +273,9 @@ fn docker_volume_requires_volume_name() { ); let service = ConfigService::new(test_context()); - let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err(); + let err = service + .load(Some(file.path().to_str().unwrap())) + .unwrap_err(); assert!(err.contains("volume_name"), "error was: {err}"); } @@ -323,7 +340,9 @@ fn databases_config_roundtrips_through_serde() { ) .unwrap(); let cfg = build_config(input).unwrap(); - let wrapped = DatabasesConfig { databases: vec![cfg] }; + let wrapped = DatabasesConfig { + databases: vec![cfg], + }; let json = serde_json::to_string(&wrapped).unwrap(); let back: DatabasesConfig = serde_json::from_str(&json).unwrap(); @@ -331,3 +350,77 @@ fn databases_config_roundtrips_through_serde() { assert_eq!(back.databases[0].db_type.as_str(), "postgresql"); assert_eq!(back.databases[0].password, "secret"); } + +#[test] +fn files_type_parses_with_options() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ + "name": "shared", + "type": "files", + "path": "/data/files", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681", + "options": { "exclude": ["node_modules", "/cache"], "one_file_system": true } + }"#, + ) + .unwrap(); + + let cfg = build_config(input).unwrap(); + assert_eq!(cfg.db_type.as_str(), "files"); + assert_eq!(cfg.path, "/data/files"); + assert_eq!( + cfg.options["exclude"], + serde_json::json!(["node_modules", "/cache"]) + ); + assert_eq!(cfg.options["one_file_system"], serde_json::json!(true)); +} + +#[test] +fn files_type_requires_path() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ "name": "shared", "type": "files", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }"#, + ) + .unwrap(); + let err = build_config(input).unwrap_err(); + assert!(err.contains("path"), "unexpected error: {err}"); +} + +#[test] +fn files_type_rejects_relative_and_root_paths() { + for bad in ["data/files", "/", "//"] { + let json = format!( + r#"{{ "name": "shared", "type": "files", "path": "{bad}", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }}"# + ); + let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap(); + let err = build_config(input).unwrap_err(); + assert!(err.contains("absolute"), "path {bad:?} gave: {err}"); + } +} + +fn files_input(options: &str) -> InputDatabaseConfig { + serde_json::from_str(&format!( + r#"{{ "name": "docs", "type": "files", "path": "/data/files", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"{options} }}"# + )) + .unwrap() +} + +#[test] +fn files_method_defaults_to_archive() { + let cfg = build_config(files_input("")).unwrap(); + assert_eq!(files_method(&cfg), "archive"); +} + +#[test] +fn files_method_reads_snapshot_and_sync() { + let snap = build_config(files_input(r#", "options": { "method": "snapshot" }"#)).unwrap(); + assert_eq!(files_method(&snap), "snapshot"); + let sync = build_config(files_input(r#", "options": { "method": "sync" }"#)).unwrap(); + assert_eq!(files_method(&sync), "sync"); +} + +#[test] +fn unknown_files_method_refuses_the_source() { + let err = build_config(files_input(r#", "options": { "method": "mirror" }"#)).unwrap_err(); + assert!(err.contains("Unknown method"), "{err}"); + assert!(err.contains("docs"), "{err}"); +} diff --git a/src/tests/services/dashboard_config_tests.rs b/src/tests/services/dashboard_config_tests.rs index 68027ad..decb577 100644 --- a/src/tests/services/dashboard_config_tests.rs +++ b/src/tests/services/dashboard_config_tests.rs @@ -82,3 +82,38 @@ fn persist_cache_leaves_no_tmp_file() { assert!(!tmp.exists(), "temp file should have been renamed away"); assert!(path.exists()); } + +use crate::services::dashboard_config::local_only_ids; +use crate::services::status::payload_method; + +fn files_cfg(gen_id: &str, method: Option<&str>) -> DatabaseConfig { + let options = method + .map(|m| format!(r#", "options": {{ "method": "{m}" }}"#)) + .unwrap_or_default(); + let json = format!( + r#"{{ "name": "docs", "type": "files", "path": "/data/files", "generated_id": "{gen_id}"{options} }}"# + ); + build_config(serde_json::from_str::(&json).unwrap()).unwrap() +} + +#[test] +fn local_only_ids_skips_sources_replaced_by_the_dashboard() { + let local = vec![cfg("local-a", ID_A, "h"), cfg("local-b", ID_B, "h")]; + let dashboard = vec![cfg("dash-b", ID_B, "h")]; + let ids = local_only_ids(&local, &dashboard); + assert!(ids.contains(ID_A)); + assert!(!ids.contains(ID_B)); +} + +#[test] +fn payload_method_is_sent_for_local_files_sources_only() { + let local = files_cfg(ID_A, Some("sync")); + let ids = local_only_ids(&[local.clone()], &[]); + assert_eq!(payload_method(&local, &ids), Some("sync")); + assert_eq!(payload_method(&files_cfg(ID_A, None), &ids), Some("archive")); + // Replaced by a dashboard config: the dashboard owns the method. + let replaced = local_only_ids(&[local.clone()], &[local.clone()]); + assert_eq!(payload_method(&local, &replaced), None); + // Not a files source. + assert_eq!(payload_method(&cfg("pg", ID_A, "h"), &ids), None); +} diff --git a/src/tests/services/local_storage_tests.rs b/src/tests/services/local_storage_tests.rs new file mode 100644 index 0000000..e44a4f9 --- /dev/null +++ b/src/tests/services/local_storage_tests.rs @@ -0,0 +1,198 @@ +//! Spec B: the dashboard's local storage, served here by `rclone serve restic +//! --append-only` and `rclone serve webdav` on free ports. The restic tests need +//! restic on PATH (test container); the sync test only needs rclone. + +use crate::domain::files::platform::is_root; +use crate::services::backup::logger::JobLogger; +use crate::domain::files::restic::backup::snapshot; +use crate::domain::files::restic::repo::ResticRepo; +use crate::domain::files::restic::restore::restore; +use crate::services::storage::providers::local::local_storage_password; +use crate::tests::domain::files::files_config; +use std::fs; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +pub(crate) const MASTER_KEY_B64: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + +pub(crate) struct RcloneServer { + child: Child, + pub base_url: String, +} + +impl Drop for RcloneServer { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// `rclone serve --baseurl /storage/ ` on a free port, user +/// `portabase`, credentials through the env like the dashboard does. +pub(crate) fn serve(kind: &str, prefix: &str, root: &Path, extra: &[&str], password: &str) -> RcloneServer { + let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); + let child = Command::new("rclone") + .args(["serve", kind, "--addr", &format!("127.0.0.1:{port}"), "--baseurl", &format!("/storage/{prefix}")]) + .args(extra) + .arg(root) + .env("RCLONE_USER", "portabase") + .env("RCLONE_PASS", password) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("rclone binary not found"); + for _ in 0..100 { + if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + RcloneServer { child, base_url: format!("http://127.0.0.1:{port}") } +} + +fn tree(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("a.txt"), "v1").unwrap(); + fs::write(root.join("docs/b.txt"), "b").unwrap(); +} + +fn local_repo(server: &RcloneServer, generated_id: &str, cache: &Path) -> ResticRepo { + ResticRepo::rest(&format!("{}/storage/restic", server.base_url), generated_id, "test-password", "s3cret", cache.to_path_buf()) +} + +#[tokio::test] +async fn local_snapshot_and_restore_go_through_the_append_only_server() { + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + let repo_dir = store.path().join(&cfg.generated_id); + assert!(repo_dir.join("config").exists(), "repository written on the served disk"); + assert_eq!(fs::read_dir(repo_dir.join("locks")).unwrap().count(), 0, "lock removed despite --append-only"); + + fs::write(src.join("a.txt"), "changed").unwrap(); + fs::write(src.join("extra.txt"), "x").unwrap(); + restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap(); + assert_eq!(fs::read_to_string(src.join("a.txt")).unwrap(), "v1"); + assert!(!src.join("extra.txt").exists()); +} + +#[tokio::test] +async fn the_agent_cannot_forget_on_the_local_server() { + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap(); + + let forget = repo.run(["forget", snap.snapshot_id.as_str()], &logger).await.unwrap(); + + assert_ne!(forget.code, 0, "append-only must refuse forget"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert!(listed.stdout.contains(&snap.snapshot_id), "{}", listed.stdout); +} + +#[tokio::test] +async fn unreadable_file_on_a_local_repo_keeps_the_snapshot_for_the_dashboard() { + if is_root() { + return; // root reads a 000 file anyway + } + use std::os::unix::fs::PermissionsExt; + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret"); + let src = work.path().join("src"); + tree(&src); + let locked = src.join("docs/b.txt"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + let cfg = files_config(&src, &[]); + let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache")); + let logger = JobLogger::new(); + + let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap(); + + assert!(err.to_string().contains("the dashboard forgets it"), "{err:#}"); + let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap(); + assert!(listed.stdout.contains("bs:bs-1"), "{}", listed.stdout); +} + +#[tokio::test] +async fn local_sync_mirrors_through_the_webdav_server() { + use crate::core::context::Context; + use crate::services::api::ApiClient; + use crate::services::api::models::agent::status::DatabaseStorage; + use crate::services::backup::files_engine::{FilesEngine, one_storage}; + use crate::utils::edge_key::EdgeKey; + use serde_json::json; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let store = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + let password = local_storage_password(MASTER_KEY_B64).unwrap(); + let server = serve("webdav", "sync", store.path(), &[], &password); + + let api = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/agent/agent-1/backup/upload/init")) + .and(body_partial_json(json!({ "engine": "sync" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&api) + .await; + Mock::given(method("PATCH")) + .and(path("/agent/agent-1/backup/upload/status")) + .and(body_partial_json(json!({ "status": "success" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&api) + .await; + + let ctx = Context { + edge_key: EdgeKey { + server_url: server.base_url.clone(), + agent_id: "agent-1".into(), + master_key_b64: MASTER_KEY_B64.into(), + }, + api: ApiClient::new(api.uri()), + }; + let storage: DatabaseStorage = + serde_json::from_value(json!({ "id": "ch-local", "provider": "local", "folderName": "backups", "config": {} })).unwrap(); + let src = work.path().join("src"); + tree(&src); + let cfg = files_config(&src, &[]); + let replica = store.path().join(&cfg.generated_id).join("current"); + + let first = one_storage(&ctx, &cfg, FilesEngine::Sync, &storage, "backup-1", &JobLogger::new()).await; + assert!(first.success, "{:?}", first.error); + assert_eq!(first.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); + assert_eq!(fs::read_to_string(replica.join("docs/b.txt")).unwrap(), "b"); + + fs::remove_file(src.join("docs/b.txt")).unwrap(); + let second = one_storage(&ctx, &cfg, FilesEngine::Sync, &storage, "backup-2", &JobLogger::new()).await; + assert!(second.success, "{:?}", second.error); + assert!(!replica.join("docs/b.txt").exists(), "deletion propagated"); + assert!(replica.join("a.txt").exists()); +} + +#[test] +fn local_storage_password_matches_the_dashboard_vector() { + assert_eq!( + local_storage_password(MASTER_KEY_B64).unwrap(), + "8abfb788701b8c502658154746af6af768ea08f579381ad73b0f1e7dd43caea2" + ); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index d825608..d7d0ba5 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -1,7 +1,9 @@ +mod api_client_tests; mod api_models_tests; mod backup_runner_tests; -mod backup_uploader_tests; +pub(crate) mod backup_uploader_tests; mod config_tests; mod cron_tests; mod dashboard_config_tests; +pub(crate) mod local_storage_tests; mod restore_downloader_tests; diff --git a/src/tests/storage/fixtures/rclone-target/blob-account-key.ini b/src/tests/storage/fixtures/rclone-target/blob-account-key.ini new file mode 100644 index 0000000..6a11d0b --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-account-key.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = myaccount +key = a2V5LWJhc2U2NA== +endpoint = https://myaccount.blob.core.windows.net +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini b/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini new file mode 100644 index 0000000..5c90e5d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-connection-string.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = devstoreaccount1 +key = a2V5 +endpoint = http://127.0.0.1:10000/devstoreaccount1 +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini b/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini new file mode 100644 index 0000000..5c90e5d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/blob-endpoint-url.ini @@ -0,0 +1,6 @@ +[blob] +type = azureblob +account = devstoreaccount1 +key = a2V5 +endpoint = http://127.0.0.1:10000/devstoreaccount1 +no_check_container = true diff --git a/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini b/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini new file mode 100644 index 0000000..4a08d5f --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-cloud-storage-emulator.ini @@ -0,0 +1,7 @@ +[google-cloud-storage] +type = google cloud storage +project_number = test +anonymous = true +endpoint = http://127.0.0.1:4443/storage/v1/ +bucket_policy_only = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini b/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini new file mode 100644 index 0000000..fdca8af --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-cloud-storage.ini @@ -0,0 +1,6 @@ +[google-cloud-storage] +type = google cloud storage +project_number = my-project +service_account_credentials = {"client_email":"svc@my-project.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEexample\n-----END PRIVATE KEY-----\n","private_key_id":"","project_id":"my-project","token_uri":"https://oauth2.googleapis.com/token","type":"service_account","universe_domain":"googleapis.com"} +bucket_policy_only = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/google-drive.ini b/src/tests/storage/fixtures/rclone-target/google-drive.ini new file mode 100644 index 0000000..d28fe0d --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/google-drive.ini @@ -0,0 +1,8 @@ +[google-drive] +type = drive +client_id = cid.apps.googleusercontent.com +client_secret = csecret +scope = drive.file +token = {"access_token":"","expiry":"0001-01-01T00:00:00Z","refresh_token":"1//refresh","token_type":"Bearer"} +root_folder_id = folder123 +use_trash = false diff --git a/src/tests/storage/fixtures/rclone-target/inputs.json b/src/tests/storage/fixtures/rclone-target/inputs.json new file mode 100644 index 0000000..4d85c98 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/inputs.json @@ -0,0 +1,52 @@ +{ + "s3": { + "provider": "s3", + "config": { "endPointUrl": "s3.example.com", "accessKey": "AKIAEXAMPLE", "secretKey": "secretEXAMPLE", "bucketName": "my-bucket", "ssl": true, "region": "eu-west-1" }, + "expected": { "remoteName": "s3", "basePath": "my-bucket" } + }, + "s3-port-no-ssl": { + "provider": "s3", + "config": { "endPointUrl": "minio.local", "port": 9000, "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "backups", "ssl": false, "region": " " }, + "expected": { "remoteName": "s3", "basePath": "backups" } + }, + "blob-account-key": { + "provider": "blob", + "config": { "authMode": "accountKey", "accountName": "myaccount", "accountKey": "a2V5LWJhc2U2NA==", "containerName": "backups" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "blob-endpoint-url": { + "provider": "blob", + "config": { "authMode": "accountKey", "accountName": "devstoreaccount1", "accountKey": "a2V5", "containerName": "backups", "endpointUrl": "http://127.0.0.1:10000" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "blob-connection-string": { + "provider": "blob", + "config": { "authMode": "connectionString", "connectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=a2V5;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", "containerName": "backups" }, + "expected": { "remoteName": "blob", "basePath": "backups" } + }, + "google-cloud-storage": { + "provider": "google-cloud-storage", + "config": { "projectId": "my-project", "bucketName": "my-gcs-bucket", "clientEmail": "svc@my-project.iam.gserviceaccount.com", "privateKey": "-----BEGIN PRIVATE KEY-----\\nMIIEexample\\n-----END PRIVATE KEY-----\\n", "apiEndpoint": "" }, + "expected": { "remoteName": "google-cloud-storage", "basePath": "my-gcs-bucket" } + }, + "google-cloud-storage-emulator": { + "provider": "google-cloud-storage", + "config": { "projectId": "test", "bucketName": "bucket", "clientEmail": "x@test", "privateKey": "", "apiEndpoint": "http://127.0.0.1:4443" }, + "expected": { "remoteName": "google-cloud-storage", "basePath": "bucket" } + }, + "google-drive": { + "provider": "google-drive", + "config": { "clientId": "cid.apps.googleusercontent.com", "clientSecret": "csecret", "refreshToken": "1//refresh", "folderId": "folder123" }, + "expected": { "remoteName": "google-drive", "basePath": "" } + }, + "rclone": { + "provider": "rclone", + "config": { "configText": "[ovh]\ntype = s3\nprovider = OVHcloud\naccess_key_id = a\nsecret_access_key = b\nendpoint = s3.gra.io.cloud.ovh.net\n", "remoteName": "ovh", "remotePath": "my-bucket" }, + "expected": { "remoteName": "ovh", "basePath": "my-bucket" } + }, + "sftp-key": { + "provider": "sftp", + "config": { "host": "sftp.example.com", "port": 2222, "username": "backup", "privateKey": "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----\n", "remotePath": "/srv/backups" }, + "expected": { "remoteName": "sftp", "basePath": "/srv/backups" } + } +} diff --git a/src/tests/storage/fixtures/rclone-target/rclone.ini b/src/tests/storage/fixtures/rclone-target/rclone.ini new file mode 100644 index 0000000..1d57741 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/rclone.ini @@ -0,0 +1,6 @@ +[ovh] +type = s3 +provider = OVHcloud +access_key_id = a +secret_access_key = b +endpoint = s3.gra.io.cloud.ovh.net diff --git a/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini b/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini new file mode 100644 index 0000000..7a5c1eb --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/s3-port-no-ssl.ini @@ -0,0 +1,9 @@ +[s3] +type = s3 +provider = Other +access_key_id = minioadmin +secret_access_key = minioadmin +endpoint = http://minio.local:9000 +region = us-east-1 +force_path_style = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/s3.ini b/src/tests/storage/fixtures/rclone-target/s3.ini new file mode 100644 index 0000000..e9fd105 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/s3.ini @@ -0,0 +1,9 @@ +[s3] +type = s3 +provider = Other +access_key_id = AKIAEXAMPLE +secret_access_key = secretEXAMPLE +endpoint = https://s3.example.com +region = eu-west-1 +force_path_style = true +no_check_bucket = true diff --git a/src/tests/storage/fixtures/rclone-target/sftp-key.ini b/src/tests/storage/fixtures/rclone-target/sftp-key.ini new file mode 100644 index 0000000..f7afe12 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-target/sftp-key.ini @@ -0,0 +1,6 @@ +[sftp] +type = sftp +host = sftp.example.com +port = 2222 +user = backup +key_file = diff --git a/src/tests/storage/fixtures/rclone-validation/cases.json b/src/tests/storage/fixtures/rclone-validation/cases.json new file mode 100644 index 0000000..69fe4c1 --- /dev/null +++ b/src/tests/storage/fixtures/rclone-validation/cases.json @@ -0,0 +1,345 @@ +[ + { + "name": "ovh-s3", + "configText": "[ovhcloud-rbx]\ntype = s3\nprovider = OVHcloud\naccess_key_id = my_access\nsecret_access_key = my_secret\nregion = rbx\nendpoint = s3.rbx.io.cloud.ovh.net\nacl = private\n", + "remoteName": "ovhcloud-rbx", + "ok": true, + "errorContains": null + }, + { + "name": "comments-crlf", + "configText": "# comment\r\n[r]\r\ntype = s3\r\n; note\r\nprovider = AWS\r\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "empty-value", + "configText": "[r]\ntype = webdav\nurl = https://dav.example.com\nuser =\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "duplicate-type", + "configText": "[x]\ntype = s3\ntype = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'type'" + }, + { + "name": "duplicate-type-case", + "configText": "[x]\ntype = s3\nTYPE = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'TYPE'" + }, + { + "name": "colon-separator", + "configText": "[x]\ntype: local\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 2 is not a [section] header" + }, + { + "name": "on-the-fly", + "configText": "[:local]\ntype = s3\n", + "remoteName": ":local", + "ok": false, + "errorContains": "invalid rclone remote name ':local'" + }, + { + "name": "sftp-ssh", + "configText": "[x]\ntype = sftp\nhost = h\nssh = touch /tmp/pwned\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'ssh' is not allowed" + }, + { + "name": "webdav-command", + "configText": "[x]\ntype = webdav\nurl = http://h\nbearer_token_command = id\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'bearer_token_command' is not allowed" + }, + { + "name": "env-auth", + "configText": "[x]\ntype = s3\nenv_auth = true\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'env_auth' is not allowed" + }, + { + "name": "use-msi", + "configText": "[x]\ntype = azureblob\naccount = a\nuse_msi = true\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_msi' is not allowed" + }, + { + "name": "service-account-file", + "configText": "[x]\ntype = google cloud storage\nservice_account_file = /etc/passwd\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'service_account_file' is not allowed" + }, + { + "name": "blocked-local", + "configText": "[x]\ntype = local\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed" + }, + { + "name": "two-sections", + "configText": "[a]\ntype = s3\n[b]\ntype = s3\n", + "remoteName": "a", + "ok": false, + "errorContains": "exactly one [section] (found 2)" + }, + { + "name": "key-before-section", + "configText": "type = s3\n[x]\ntype = s3\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 1 appears before any [section]" + }, + { + "name": "no-type", + "configText": "[x]\nprovider = AWS\n", + "remoteName": "x", + "ok": false, + "errorContains": "has no type" + }, + { + "name": "remote-mismatch", + "configText": "[a]\ntype = s3\n", + "remoteName": "b", + "ok": false, + "errorContains": "remote 'b' is not defined" + }, + { + "name": "bad-key", + "configText": "[x]\ntype = s3\nbad key = 1\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone config key 'bad key'" + }, + { + "name": "empty-config", + "configText": "", + "remoteName": "r", + "ok": false, + "errorContains": "exactly one [section] (found 0)" + }, + { + "name": "empty-section-name", + "configText": "[]\ntype = s3\n", + "remoteName": "", + "ok": false, + "errorContains": "invalid rclone remote name ''" + }, + { + "name": "name-with-space", + "configText": "[a b]\ntype = s3\n", + "remoteName": "a b", + "ok": false, + "errorContains": "invalid rclone remote name 'a b'" + }, + { + "name": "blocked-type-uppercase", + "configText": "[x]\ntype = LOCAL\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed" + }, + { + "name": "key-file-uppercase", + "configText": "[x]\ntype = sftp\nKEY_FILE = /etc/passwd\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'KEY_FILE' is not allowed" + }, + { + "name": "equals-at-line-start", + "configText": "[x]\n= v\n", + "remoteName": "x", + "ok": false, + "errorContains": "line 2 is not a [section] header" + }, + { + "name": "whitespace-tolerant", + "configText": " [r] \n type=s3 \n\n", + "remoteName": "r", + "ok": true, + "errorContains": null + }, + { + "name": "quoted-type-backtick", + "configText": "[x]\ntype = `local`\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-type-triple", + "configText": "[x]\ntype = \"\"\"local\"\"\"\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-type-trailing-junk", + "configText": "[x]\ntype = `alias` junk\nremote = /\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'type' must not be quoted" + }, + { + "name": "quoted-other-value", + "configText": "[x]\ntype = s3\nendpoint = `x`\n", + "remoteName": "x", + "ok": false, + "errorContains": "value for key 'endpoint' must not be quoted (remote 'x')" + }, + { + "name": "blocked-google-photos", + "configText": "[x]\ntype = google photos\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'google photos' is not allowed" + }, + { + "name": "blocked-gphotos", + "configText": "[x]\ntype = gphotos\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'gphotos' is not allowed" + }, + { + "name": "invalid-type-syntax", + "configText": "[x]\ntype = s3!\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone backend type 's3!' (remote 'x')" + }, + { + "name": "forbidden-use-az", + "configText": "[x]\ntype = azureblob\nuse_az = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_az' is not allowed (remote 'x')" + }, + { + "name": "forbidden-use-kerberos", + "configText": "[x]\ntype = hdfs\nuse_kerberos = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'use_kerberos' is not allowed (remote 'x')" + }, + { + "name": "forbidden-kerberos-ccache", + "configText": "[x]\ntype = hdfs\nkerberos_ccache = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'kerberos_ccache' is not allowed (remote 'x')" + }, + { + "name": "forbidden-key-use-agent", + "configText": "[x]\ntype = sftp\nkey_use_agent = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'key_use_agent' is not allowed (remote 'x')" + }, + { + "name": "forbidden-set-env", + "configText": "[x]\ntype = sftp\nset_env = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'set_env' is not allowed (remote 'x')" + }, + { + "name": "forbidden-unix-socket", + "configText": "[x]\ntype = sftp\nunix_socket = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'unix_socket' is not allowed (remote 'x')" + }, + { + "name": "forbidden-client-certificate-path", + "configText": "[x]\ntype = azureblob\nclient_certificate_path = v\n", + "remoteName": "x", + "ok": false, + "errorContains": "key 'client_certificate_path' is not allowed (remote 'x')" + }, + { + "name": "oracle-env-auth", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = env_auth\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-missing-provider", + "configText": "[x]\ntype = oracleobjectstorage\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-spaced-type", + "configText": "[x]\ntype = oracle object storage\nprovider = instance_principal_auth\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-provider-uppercase-value", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = NO_AUTH\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-provider-uppercase-key", + "configText": "[x]\ntype = oracleobjectstorage\nPROVIDER = no_auth\nnamespace = n\n", + "remoteName": "x", + "ok": false, + "errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')" + }, + { + "name": "oracle-no-auth", + "configText": "[x]\ntype = oracleobjectstorage\nprovider = no_auth\nnamespace = n\nregion = eu-frankfurt-1\n", + "remoteName": "x", + "ok": true, + "errorContains": null + }, + { + "name": "nel-in-type", + "configText": "[x]\ntype = local\u0085\n", + "remoteName": "x", + "ok": false, + "errorContains": "backend type 'local' is not allowed (remote 'x')" + }, + { + "name": "order-duplicate-before-forbidden", + "configText": "[x]\ntype = s3\nssh = a\nssh = b\n", + "remoteName": "x", + "ok": false, + "errorContains": "duplicate rclone config key 'ssh' (remote 'x')" + }, + { + "name": "order-bad-line-before-pair-before-header", + "configText": "type = s3\n[x]\nfoo\n", + "remoteName": "x", + "ok": false, + "errorContains": "rclone config line 3 is not a [section] header" + }, + { + "name": "order-invalid-key-before-duplicate", + "configText": "[x]\ntype = s3\nbad key = 1\nbad key = 2\n", + "remoteName": "x", + "ok": false, + "errorContains": "invalid rclone config key 'bad key' (remote 'x')" + } +] diff --git a/src/tests/storage/google_cloud_storage.rs b/src/tests/storage/google_cloud_storage.rs index 9a5142d..0b01da6 100644 --- a/src/tests/storage/google_cloud_storage.rs +++ b/src/tests/storage/google_cloud_storage.rs @@ -12,7 +12,7 @@ use testcontainers::{GenericImage, ImageExt}; const BUCKET: &str = "portabase"; -async fn start_fake_gcs() -> (testcontainers::ContainerAsync, String) { +pub(super) async fn start_fake_gcs() -> (testcontainers::ContainerAsync, String) { // Natural random host port (no port-80 pin). The provider forces a single-shot // upload for custom endpoints, which issues one request to this endpoint and never // follows a server-built `Location` — so it works on any port, unlike the resumable diff --git a/src/tests/storage/mod.rs b/src/tests/storage/mod.rs index f6c9152..96840a0 100644 --- a/src/tests/storage/mod.rs +++ b/src/tests/storage/mod.rs @@ -1,4 +1,5 @@ mod azure_blob; mod google_cloud_storage; mod rclone; +mod rclone_target; mod sftp; diff --git a/src/tests/storage/rclone.rs b/src/tests/storage/rclone.rs index 6827cf7..9d40389 100644 --- a/src/tests/storage/rclone.rs +++ b/src/tests/storage/rclone.rs @@ -63,6 +63,41 @@ fn validate_config_accepts_the_target_remote() { assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok()); } +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct ValidationCase { + name: String, + config_text: String, + remote_name: String, + ok: bool, + error_contains: Option, +} + +/// The same file is verified against the dashboard validator: keep both in sync. +#[test] +fn validate_config_matches_the_shared_test_vectors() { + let cases: Vec = + serde_json::from_str(include_str!("fixtures/rclone-validation/cases.json")).unwrap(); + assert!(!cases.is_empty()); + + let mut failures = Vec::new(); + for case in &cases { + match (validate_config(&case.config_text, &case.remote_name), case.ok) { + (Ok(()), true) => {} + (Ok(()), false) => failures.push(format!("[{}] expected an error, got Ok", case.name)), + (Err(e), true) => failures.push(format!("[{}] expected Ok, got: {e:#}", case.name)), + (Err(e), false) => { + let msg = format!("{e:#}"); + let want = case.error_contains.as_deref().expect("ok:false needs errorContains"); + if !msg.contains(want) { + failures.push(format!("[{}] error {msg:?} does not contain {want:?}", case.name)); + } + } + } + } + assert!(failures.is_empty(), "{} case(s) failed:\n{}", failures.len(), failures.join("\n")); +} + #[test] fn validate_config_rejects_an_unknown_remote_name() { let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string(); @@ -85,23 +120,28 @@ fn validate_config_rejects_alias_backend() { } #[test] -fn validate_config_rejects_a_blocked_backend_in_a_chained_section() { +fn validate_config_rejects_chained_sections() { + // A blocked backend hidden behind an allowed (or wrapping) remote can no longer be expressed: + // anything beyond a single [section] is refused outright. let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n"; let err = validate_config(cfg, "secret").unwrap_err().to_string(); - assert!(err.contains("crypt"), "unexpected error: {err}"); - assert!(err.contains("secret"), "error should name the offending remote: {err}"); + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n"; let err = validate_config(cfg, "outer").unwrap_err().to_string(); - assert!(err.contains("local"), "unexpected error: {err}"); - assert!(err.contains("disk"), "error should name the offending remote: {err}"); -} + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); -#[test] -fn validate_config_rejects_crypt_even_over_an_allowed_remote() { let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}"); let err = validate_config(&cfg, "secret").unwrap_err().to_string(); + assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}"); +} + +#[test] +fn validate_config_rejects_crypt_over_an_allowed_remote_name() { + let cfg = "[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n"; + let err = validate_config(cfg, "secret").unwrap_err().to_string(); assert!(err.contains("crypt"), "unexpected error: {err}"); + assert!(err.contains("secret"), "error should name the offending remote: {err}"); } #[test] @@ -179,7 +219,7 @@ use testcontainers::{GenericImage, ImageExt}; const BUCKET: &str = "portabase"; -async fn start_minio() -> (testcontainers::ContainerAsync, String) { +pub(super) async fn start_minio() -> (testcontainers::ContainerAsync, String) { let container = GenericImage::new("coollabsio/minio", "latest") .with_exposed_port(9000.tcp()) .with_wait_for(WaitFor::message_on_stderr("API:")) @@ -208,7 +248,7 @@ fn minio_config(endpoint: &str) -> String { ) } -fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec { +pub(super) fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec { let out = Command::new("rclone") .arg("--config") .arg(config_path) diff --git a/src/tests/storage/rclone_target.rs b/src/tests/storage/rclone_target.rs new file mode 100644 index 0000000..190c726 --- /dev/null +++ b/src/tests/storage/rclone_target.rs @@ -0,0 +1,302 @@ +use crate::services::api::models::agent::status::DatabaseStorage; +use super::google_cloud_storage::start_fake_gcs; +use super::rclone::{rclone_ok, start_minio}; +use crate::services::storage::providers::rclone::helpers::{obscure_password, write_config}; +use crate::services::storage::providers::rclone::target::rclone_target; +use serde_json::{Value, json}; + +const INPUTS: &str = include_str!("fixtures/rclone-target/inputs.json"); + +pub(super) fn expected_ini(name: &str) -> &'static str { + match name { + "s3" => include_str!("fixtures/rclone-target/s3.ini"), + "s3-port-no-ssl" => include_str!("fixtures/rclone-target/s3-port-no-ssl.ini"), + "blob-account-key" => include_str!("fixtures/rclone-target/blob-account-key.ini"), + "blob-endpoint-url" => include_str!("fixtures/rclone-target/blob-endpoint-url.ini"), + "blob-connection-string" => include_str!("fixtures/rclone-target/blob-connection-string.ini"), + "google-cloud-storage" => include_str!("fixtures/rclone-target/google-cloud-storage.ini"), + "google-cloud-storage-emulator" => { + include_str!("fixtures/rclone-target/google-cloud-storage-emulator.ini") + } + "google-drive" => include_str!("fixtures/rclone-target/google-drive.ini"), + "rclone" => include_str!("fixtures/rclone-target/rclone.ini"), + "sftp-key" => include_str!("fixtures/rclone-target/sftp-key.ini"), + other => panic!("no expected fixture for {other}"), + } +} + +pub(super) fn storage(provider: &str, config: Value) -> DatabaseStorage { + serde_json::from_value(json!({ + "id": "storage-1", + "provider": provider, + "folderName": "backups", + "config": config, + })) + .unwrap() +} + +/// Temp-file paths differ per run; normalize them before comparing. Keeps the exact +/// trailing newline so a missing final `\n` fails the golden test. +pub(super) fn normalize(config_text: &str) -> String { + config_text + .split('\n') + .map(|l| { + if l.starts_with("key_file = ") { + "key_file = ".to_string() + } else { + l.to_string() + } + }) + .collect::>() + .join("\n") +} + +#[test] +fn rclone_target_matches_golden_fixtures() { + let inputs: Value = serde_json::from_str(INPUTS).unwrap(); + for (name, case) in inputs.as_object().unwrap() { + let target = rclone_target(&storage( + case["provider"].as_str().unwrap(), + case["config"].clone(), + )) + .unwrap_or_else(|e| panic!("{name}: {e:#}")); + + assert_eq!(normalize(&target.config_text), expected_ini(name), "{name}: config_text"); + assert_eq!(target.remote_name, case["expected"]["remoteName"].as_str().unwrap(), "{name}: remote_name"); + assert_eq!(target.base_path, case["expected"]["basePath"].as_str().unwrap(), "{name}: base_path"); + } +} + +#[test] +fn rclone_target_rejects_local_and_unknown_providers() { + let err = rclone_target(&storage("local", json!({}))).err().unwrap(); + assert!(format!("{err:#}").contains("a local storage channel cannot be used as an rclone target")); + + let err = rclone_target(&storage("ftp-of-doom", json!({}))).err().unwrap(); + assert!(format!("{err:#}").contains("ftp-of-doom")); +} + +#[test] +fn rclone_target_rejects_azure_without_account_key() { + let err = rclone_target(&storage( + "blob", + json!({ + "authMode": "connectionString", + "connectionString": "BlobEndpoint=https://acct.blob.core.windows.net;SharedAccessSignature=sv=2022&sig=x", + "containerName": "c" + }), + )) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("account key"), "{err:#}"); +} + +#[test] +fn rclone_target_applies_the_rclone_backend_blocklist() { + let err = rclone_target(&storage( + "rclone", + json!({ "configText": "[disk]\ntype = local\n", "remoteName": "disk", "remotePath": "" }), + )) + .err() + .unwrap(); + assert!(format!("{err:#}").contains("not allowed"), "{err:#}"); +} + +#[test] +fn sftp_key_file_lives_exactly_as_long_as_the_target() { + let target = rclone_target(&storage( + "sftp", + json!({ "host": "h", "username": "u", "privateKey": "KEY", "remotePath": "" }), + )) + .unwrap(); + let key_path = target + .config_text + .lines() + .find_map(|l| l.strip_prefix("key_file = ")) + .unwrap() + .to_string(); + assert_eq!(std::fs::read_to_string(&key_path).unwrap(), "KEY"); + drop(target); + assert!(!std::path::Path::new(&key_path).exists(), "key file must be removed with the target"); +} + +#[test] +fn sftp_config_error_keeps_the_root_cause() { + let err = rclone_target(&storage("sftp", json!({ "username": "u", "password": "p" }))) + .err() + .unwrap(); + let text = format!("{err:#}"); + assert!(text.contains("invalid sftp storage config"), "{text}"); + assert!(text.contains("host"), "{text}"); +} + +#[test] +fn sftp_password_only_has_pass_and_no_key_file() { + let target = rclone_target(&storage( + "sftp", + json!({ "host": "h", "username": "u", "password": "secret", "remotePath": "" }), + )) + .unwrap(); + assert!(!target.config_text.contains("key_file"), "{}", target.config_text); + assert!(target.config_text.lines().any(|l| l.starts_with("pass = ")), "{}", target.config_text); +} + +#[test] +fn obscured_password_round_trips_through_rclone_reveal() { + let obscured = obscure_password("s3cr3t").unwrap(); + assert_ne!(obscured, "s3cr3t"); + + let out = std::process::Command::new("rclone").args(["reveal", &obscured]).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "s3cr3t"); +} + +#[tokio::test] +async fn s3_target_reaches_minio_through_rclone() { + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + + let target = rclone_target(&storage( + "s3", + json!({ "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }), + )) + .unwrap(); + let config = write_config(&target.config_text).unwrap(); + + // The mapper sets `no_check_bucket = true`, so create the bucket with a per-call override. + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap(); + assert!(listing.contains("portabase"), "lsd output: {listing}"); +} + +#[tokio::test] +async fn gcs_emulator_target_reaches_fake_gcs_through_rclone() { + let (_container, endpoint) = start_fake_gcs().await; // "http://host:port" + + let target = rclone_target(&storage( + "google-cloud-storage", + json!({ "projectId": "test", "bucketName": "bucket1", "clientEmail": "x@test", + "privateKey": "", "apiEndpoint": endpoint }), + )) + .unwrap(); + let config = write_config(&target.config_text).unwrap(); + + // Same per-call override as the s3 test: the mapper never creates buckets on its own. + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap(); + assert!(listing.contains("bucket1"), "lsd output: {listing}"); +} + +#[tokio::test] +async fn s3_target_hosts_a_restic_repository() { + use crate::services::backup::logger::JobLogger; + use crate::domain::files::restic::backup::snapshot; + use crate::domain::files::restic::repo::ResticRepo; + use base64::{Engine as _, engine::general_purpose}; + + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + let channel = storage( + "s3", + json!({ "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }), + ); + let target = rclone_target(&channel).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + + let src = tempfile::TempDir::new().unwrap(); + std::fs::write(src.path().join("a.txt"), "a").unwrap(); + let cfg = crate::tests::domain::files::files_config(src.path(), &[]); + let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]); + + let edge_key = crate::utils::edge_key::EdgeKey { + server_url: String::new(), + agent_id: "agent-1".into(), + master_key_b64: master_key_b64.clone(), + }; + let repo = ResticRepo::open(&channel, &cfg.generated_id, &edge_key).unwrap(); + assert_eq!(repo.repository(), format!("rclone:s3:portabase/backups/restic/{}", cfg.generated_id)); + let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap(); + + let listed = String::from_utf8(rclone_ok( + config.path(), + &["lsf", &format!("s3:portabase/backups/restic/{}/snapshots", cfg.generated_id)], + )) + .unwrap(); + assert!(listed.contains(&snap.snapshot_id), "{listed}"); +} + +#[tokio::test] +async fn s3_targets_host_one_sync_replica_per_channel() { + use crate::services::backup::logger::JobLogger; + use crate::services::backup::files_engine::{FilesEngine, one_storage}; + use crate::tests::services::backup_uploader_tests::ctx_pointing_at; + use wiremock::matchers::{body_partial_json, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let (_container, endpoint) = start_minio().await; // "http://host:port" + let host_port = endpoint.trim_start_matches("http://"); + let (host, port) = host_port.rsplit_once(':').unwrap(); + let channel = |id: &str, bucket: &str| -> DatabaseStorage { + serde_json::from_value(json!({ + "id": id, "provider": "s3", "folderName": "backups", + "config": { "endPointUrl": host, "port": port.parse::().unwrap(), "ssl": false, + "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": bucket } + })) + .unwrap() + }; + let channels = [channel("ch-a", "portabase"), channel("ch-b", "portabase2")]; + for c in &channels { + let target = rclone_target(c).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path); + rclone_ok(config.path(), &["mkdir", &create]); + } + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/agent/agent-1/backup/upload/init")) + .and(body_partial_json(json!({ "engine": "sync" }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&server) + .await; + Mock::given(method("PATCH")) + .and(path("/agent/agent-1/backup/upload/status")) + .and(body_partial_json(json!({ "status": "success", "size": 2, "stats": { "filesTransferred": 2, "filesDeleted": 0, "replicaBytes": 2 } }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } }))) + .expect(2) + .mount(&server) + .await; + + let src = tempfile::TempDir::new().unwrap(); + std::fs::create_dir_all(src.path().join("docs")).unwrap(); + std::fs::write(src.path().join("a.txt"), "a").unwrap(); + std::fs::write(src.path().join("docs/b.txt"), "b").unwrap(); + let cfg = crate::tests::domain::files::files_config(src.path(), &[]); + let ctx = ctx_pointing_at(server.uri()); + + for c in &channels { + let result = one_storage(&ctx, &cfg, FilesEngine::Sync, c, "backup-1", &JobLogger::new()).await; + assert!(result.success, "{:?}", result.error); + assert_eq!(result.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str())); + assert_eq!(result.total_size, Some(2), "backups.file_size is the replica size"); + } + for (c, bucket) in channels.iter().zip(["portabase", "portabase2"]) { + let target = rclone_target(c).unwrap(); + let config = write_config(&target.config_text).unwrap(); + let listed = String::from_utf8(rclone_ok( + config.path(), + &["lsf", "-R", &format!("s3:{bucket}/backups/sync/{}/current", cfg.generated_id)], + )) + .unwrap(); + assert!(listed.contains("a.txt") && listed.contains("docs/b.txt"), "{bucket}: {listed}"); + } +} diff --git a/src/utils/edge_key.rs b/src/utils/edge_key.rs index d18b87d..7a8875d 100644 --- a/src/utils/edge_key.rs +++ b/src/utils/edge_key.rs @@ -1,6 +1,9 @@ +use anyhow::Context; use base64::{Engine as _, engine::general_purpose}; +use hmac::{Hmac, Mac}; use serde::Deserialize; use serde_json::Value; +use sha2::Sha256; use thiserror::Error; #[derive(Debug, Deserialize)] @@ -45,3 +48,21 @@ pub fn decode_edge_key(edge_key: &str) -> Result { Err(EdgeKeyError::InvalidKey) } } + +pub fn derive_key(master_key_b64: &str, info: &[u8]) -> anyhow::Result<[u8; 32]> { + let master_key = general_purpose::STANDARD + .decode(master_key_b64.trim()) + .context("master key is not valid base64")?; + Ok(hkdf_sha256_32(&master_key, info)) +} + +pub fn hkdf_sha256_32(ikm: &[u8], info: &[u8]) -> [u8; 32] { + let mut extract = Hmac::::new_from_slice(&[]).expect("HMAC takes any key length"); + extract.update(ikm); + let prk = extract.finalize().into_bytes(); + + let mut expand = Hmac::::new_from_slice(&prk).expect("HMAC takes any key length"); + expand.update(info); + expand.update(&[1]); + expand.finalize().into_bytes().into() +} diff --git a/src/utils/file.rs b/src/utils/file.rs index 10bac58..1168f45 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -46,14 +46,15 @@ pub fn full_file_name(encrypt: bool) -> String { } } -pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String { - let folder_name = folder_name - .map(str::trim) - .map(|folder_name| folder_name.trim_matches(char::from(47))) +pub fn folder_or_default(folder_name: Option<&str>) -> &str { + folder_name + .map(|folder_name| folder_name.trim().trim_matches('/')) .filter(|folder_name| !folder_name.is_empty()) - .unwrap_or("backups"); + .unwrap_or("backups") +} - format!("{}/{}/{}", folder_name, chrono::Local::now().format("%Y-%m-%d"), file_name) +pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String { + format!("{}/{}/{}", folder_or_default(folder_name), chrono::Local::now().format("%Y-%m-%d"), file_name) } const CHUNK_SIZE: usize = 16 * 1024 * 1024; diff --git a/src/utils/task_manager/scheduler.rs b/src/utils/task_manager/scheduler.rs index c1d248c..8e59b01 100644 --- a/src/utils/task_manager/scheduler.rs +++ b/src/utils/task_manager/scheduler.rs @@ -32,10 +32,8 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) { let mut conn_clone = conn.clone(); tokio::spawn(async move { - info!( - "Executing task={} args={:?} metadata={:?}", - task_clone.task, task_clone.args, task_clone.metadata - ); + // Never log metadata: it carries decrypted storage channels (secrets). + info!("Executing task={} args={:?}", task_clone.task, task_clone.args); if let Err(e) = execute_task( task_clone.task.as_str(), task_clone.args, @@ -65,6 +63,14 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) { } } +/// Local config merged with the dashboard-pushed sources, as the agent loop sees them. +fn merged_config(ctx: &Arc) -> crate::services::config::DatabasesConfig { + let local = ConfigService::new(ctx.clone()).load_optional(None); + let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path).join("dashboard_databases.json"); + let dashboard = crate::services::dashboard_config::load_cache(&cache_path); + crate::services::dashboard_config::merge(&local.databases, &dashboard) +} + pub async fn execute_task( task: &str, args: Vec, @@ -77,14 +83,9 @@ pub async fn execute_task( info!("{} | {}", generated_id, dbms); let ctx = Arc::new(Context::new()); - let config_service = ConfigService::new(ctx.clone()); let backup_service = BackupService::new(ctx.clone()); - let local = config_service.load_optional(None); - let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path) - .join("dashboard_databases.json"); - let dashboard = crate::services::dashboard_config::load_cache(&cache_path); - let config = crate::services::dashboard_config::merge(&local.databases, &dashboard); + let config = merged_config(&ctx); let metadata_obj = metadata .into_iter() @@ -101,6 +102,8 @@ pub async fn execute_task( let storages: Vec = serde_json::from_value(storages_value.clone())?; let encrypt: bool = serde_json::from_value(encrypt_value.clone())?; + // Tasks stored before P2 have no engine: archive. + let engine = metadata_obj.get("engine").and_then(Value::as_str).unwrap_or("archive"); backup_service .dispatch( @@ -109,6 +112,7 @@ pub async fn execute_task( BackupMethod::Automatic, &storages, encrypt, + engine, ) .await; diff --git a/src/utils/text.rs b/src/utils/text.rs index a8814ea..e3ef593 100644 --- a/src/utils/text.rs +++ b/src/utils/text.rs @@ -53,3 +53,13 @@ fn convert_dow_part(part: &str) -> String { None => converted, } } + +pub fn human_size(bytes: u64) -> String { + if bytes >= 1024 * 1024 { + format!("{} MB", bytes / 1024 / 1024) + } else if bytes >= 1024 { + format!("{} KB", bytes / 1024) + } else { + format!("{bytes} B") + } +}