1.1.1

Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.44"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
 
 [[package]]
 name = "atty"
@@ -30,9 +30,9 @@ dependencies = [
 
 [[package]]
 name = "autocfg"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "bitflags"
@@ -42,7 +42,7 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "butterup"
-version = "0.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "chrono",
@@ -54,15 +54,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.70"
+version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
-
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 
 [[package]]
 name = "cfg-if"
@@ -85,27 +79,26 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "3.0.0-beta.4"
+version = "3.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcd70aa5597dbc42f7217a543f9ef2768b2ef823ba29036072d30e1d88e98406"
+checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
 dependencies = [
  "atty",
  "bitflags",
  "clap_derive",
+ "clap_lex",
  "indexmap",
  "lazy_static",
- "os_str_bytes",
  "strsim",
  "termcolor",
  "textwrap",
- "vec_map",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "3.0.0-beta.4"
+version = "3.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b5bb0d655624a0b8770d1c178fb8ffcb1f91cc722cb08f451e3dc72465421ac"
+checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -115,12 +108,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "cloudabi"
+name = "clap_lex"
-version = "0.0.3"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
 dependencies = [
- "bitflags",
+ "os_str_bytes",
 ]
 
 [[package]]
@@ -144,12 +137,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 
 [[package]]
 name = "heck"
-version = "0.3.3"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9"
-dependencies = [
- "unicode-segmentation",
-]
 
 [[package]]
 name = "hermit-abi"
@@ -171,14 +161,23 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.7.0"
+version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"
 dependencies = [
  "autocfg",
  "hashbrown",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -187,15 +186,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.103"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
 
 [[package]]
 name = "libssh2-sys"
-version = "0.2.21"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0186af0d8f171ae6b9c4c90ec51898bad5d08a2d5e470903a50d9ad8959cbee"
+checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca"
 dependencies = [
  "cc",
  "libc",
@@ -207,9 +206,9 @@ dependencies = [
 
 [[package]]
 name = "libz-sys"
-version = "1.1.3"
+version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
 dependencies = [
  "cc",
  "libc",
@@ -219,33 +218,34 @@ dependencies = [
 
 [[package]]
 name = "lock_api"
-version = "0.3.4"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
+checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
 dependencies = [
+ "autocfg",
  "scopeguard",
 ]
 
 [[package]]
 name = "log"
-version = "0.4.14"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
 name = "memchr"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "num-integer"
-version = "0.1.44"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
+checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
 dependencies = [
  "autocfg",
  "num-traits",
@@ -253,50 +253,61 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
-name = "openssl-sys"
+name = "openssl-src"
-version = "0.9.67"
+version = "111.20.0+1.1.1o"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
+checksum = "92892c4f87d56e376e469ace79f1128fdaded07646ddf73aa0be4706ff712dec"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0"
 dependencies = [
  "autocfg",
  "cc",
  "libc",
+ "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
 
 [[package]]
 name = "os_str_bytes"
-version = "3.1.0"
+version = "6.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d"
+checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa"
 
 [[package]]
 name = "parking_lot"
-version = "0.10.2"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
 dependencies = [
+ "instant",
  "lock_api",
  "parking_lot_core",
 ]
 
 [[package]]
 name = "parking_lot_core"
-version = "0.7.2"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
+checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if",
- "cloudabi",
+ "instant",
  "libc",
  "redox_syscall",
  "smallvec",
@@ -305,9 +316,9 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.20"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
 
 [[package]]
 name = "pretty_env_logger"
@@ -345,11 +356,11 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.29"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
@@ -360,24 +371,27 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quote"
-version = "1.0.9"
+version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.1.57"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
+checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
+dependencies = [
+ "bitflags",
+]
 
 [[package]]
 name = "regex"
-version = "1.5.4"
+version = "1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -386,9 +400,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.6.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
 
 [[package]]
 name = "scopeguard"
@@ -398,15 +412,15 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "smallvec"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
+checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
 
 [[package]]
 name = "ssh2"
-version = "0.9.1"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d876d4d57f6bbf2245d43f7ec53759461f801a446d3693704aa6d27b257844d7"
+checksum = "269343e64430067a14937ae0e3c4ec604c178fb896dde0964b1acd22b3e2eeb1"
 dependencies = [
  "bitflags",
  "libc",
@@ -422,32 +436,29 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.77"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5239bc68e0fef57495900cfea4e8dc75596d9a319d7e16b1e0a440d24e6fe0a0"
+checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
 dependencies = [
  "winapi-util",
 ]
 
 [[package]]
 name = "textwrap"
-version = "0.14.2"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
+checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
-dependencies = [
- "unicode-width",
-]
 
 [[package]]
 name = "time"
@@ -461,22 +472,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "unicode-segmentation"
+name = "unicode-ident"
-version = "1.8.0"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
 
 [[package]]
 name = "vcpkg"
@@ -484,17 +483,11 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
 [[package]]
 name = "version_check"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasi"

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "butterup"
 description = "Backup btrfs snapshots over SSH"
-version = "0.1.0"
+version = "1.1.1"
 authors = ["Joakim Hulthe <joakim@hulthe.net>"]
 license = "MPL-2.0"
 edition = "2018"
@@ -9,7 +9,13 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0.44"
 chrono = "0.4.19"
-clap = "3.0.0-beta.4"
 log = "0.4"
 pretty_env_logger = "0.4"
-ssh2 = "0.9.1"
+
+[dependencies.clap]
+version = "3.1.0"
+features = ["derive", "cargo"]
+
+[dependencies.ssh2]
+version = "0.9.1"
+features = ["vendored-openssl"]

README.md

@@ -0,0 +1,10 @@
+# butterup
+Backup btrfs snapshots over ssh.
+
+## Usage
+Point butterup at a folder containing btrfs subvolumes named according to RFC 3339, it will upload them to a remote host using ssh.
+
+For detailed usage:
+```sh
+butterup help
+```

src/actions/show_plan.rs

@@ -1,17 +1,17 @@
 use crate::{local, planner, remote, Opt};
 
-pub fn run(opt: &Opt) -> anyhow::Result<()> {
+pub fn run(opt: &Opt, include_all: bool) -> anyhow::Result<()> {
     info!("showing backup plan");
 
     let local_list = local::file_list(opt)?;
     let session = remote::connect(opt)?;
     let remote_list = remote::file_list(opt, &session)?;
 
-    let plan = planner::plan(&local_list, &remote_list);
+    let plan = planner::plan(&local_list, &remote_list, include_all);
     let presence = planner::presence(&local_list, &remote_list);
 
     println!(
-        "found {} out of {} folders that need backup",
+        "found that {} out of {} folders need backup",
         plan.transfers.len(),
         presence.len(),
     );
@@ -19,7 +19,7 @@ pub fn run(opt: &Opt) -> anyhow::Result<()> {
     if !plan.transfers.is_empty() {
         println!("plan:");
         for (item, item_plan) in plan.transfers {
-            println!("- {}: {:?}", item, item_plan);
+            println!("- {:?}: {:?}", item, item_plan);
         }
     }
 

src/actions/sync.rs

@@ -1,25 +1,24 @@
 use crate::local;
 use crate::planner::{self, TransferKind};
 use crate::remote;
+use crate::util::{format_duration, path_as_utf8};
 use crate::Opt;
 use ssh2::Session;
-use std::io::{self, Read};
+use std::io::{self, Read, Write};
 use std::process::{Command, Stdio};
+use std::sync::mpsc;
+use std::thread;
+use std::time::Instant;
 
-pub fn run(opt: &Opt, sync_all: bool) -> anyhow::Result<()> {
+const TMP_FOLDER: &str = ".tmp";
-    // TODO: currently we only sync the latest local files
-    // --all will force a sync of ALL files on local which does not exist on remote
-    if sync_all {
-        error!("backup --all is not yet implemented");
-        unimplemented!();
-    }
 
+pub fn run(opt: &Opt, include_all: bool) -> anyhow::Result<()> {
     info!("generating backup plan");
     let local_list = local::file_list(opt)?;
     let session = remote::connect(opt)?;
     let remote_list = remote::file_list(opt, &session)?;
 
-    let plan = planner::plan(&local_list, &remote_list);
+    let plan = planner::plan(&local_list, &remote_list, include_all);
 
     if plan.transfers.is_empty() {
         info!("nothing to do");
@@ -44,13 +43,81 @@ pub fn run(opt: &Opt, sync_all: bool) -> anyhow::Result<()> {
     Ok(())
 }
 
+struct CmdOutput {
+    exit_status: i32,
+    stdout: String,
+    stderr: String,
+}
+
+fn do_cmd(session: &Session, cmd: &str) -> anyhow::Result<CmdOutput> {
+    let mut ch = session.channel_session()?;
+
+    ch.exec(cmd)?;
+    ch.send_eof()?;
+
+    let mut stdout = String::new();
+    let mut stderr = String::new();
+
+    ch.stderr().read_to_string(&mut stderr)?;
+    ch.read_to_string(&mut stdout)?;
+
+    ch.close()?;
+    ch.wait_close()?;
+    let exit_status = ch.exit_status()?;
+
+    Ok(CmdOutput {
+        stdout,
+        stderr,
+        exit_status,
+    })
+}
+
+fn clear_tmp_dir(opt: &Opt, session: &Session) -> anyhow::Result<bool> {
+    let tmp_path = opt.remote.path.join(TMP_FOLDER);
+    let tmp_path = path_as_utf8(&tmp_path)?;
+    let cmd = format!(r#"rm -r "{}""#, tmp_path);
+    let result = do_cmd(session, &cmd)?;
+
+    let success = result.exit_status == 0;
+
+    Ok(success)
+}
+
+fn create_tmp_dir(opt: &Opt, session: &Session) -> anyhow::Result<()> {
+    let tmp_path = opt.remote.path.join(TMP_FOLDER);
+    let tmp_path = path_as_utf8(&tmp_path)?;
+    let cmd = format!(r#"mkdir "{}""#, tmp_path);
+    let result = do_cmd(session, &cmd)?;
+
+    if result.exit_status != 0 {
+        anyhow::bail!("failed to create {} dir on remote", TMP_FOLDER);
+    }
+
+    Ok(())
+}
+
 fn send_snapshot(
     opt: &Opt,
     session: &Session,
     snapshot: &str,
     parent: Option<&str>,
 ) -> anyhow::Result<()> {
-    info!("[{}] transmitting delta", snapshot);
+    if parent.is_none() {
+        info!("[{}] sending full snapshot data", snapshot);
+    } else {
+        info!("[{}] sending snapshot delta", snapshot);
+    }
+
+    if clear_tmp_dir(opt, session)? {
+        warn!(
+            "[{}] {} dir did already exist, it is likely that a previous upload failed.",
+            snapshot, TMP_FOLDER
+        );
+    }
+
+    create_tmp_dir(opt, session)?;
+
+    let start_time = Instant::now();
 
     // spawn btrfs send
     let mut send = Command::new("btrfs")
@@ -59,7 +126,7 @@ fn send_snapshot(
         .arg(snapshot)
         .current_dir(&opt.path)
         .stdin(Stdio::null())
-        .stderr(Stdio::null())
+        .stderr(Stdio::piped())
         .stdout(Stdio::piped())
         .spawn()?;
 
@@ -69,35 +136,83 @@ fn send_snapshot(
         .take()
         .ok_or_else(|| anyhow::format_err!("failed to take stdout"))?;
 
-    // start btrfs receive
+    // #### UPLOAD SNAPSHOT FILE ####
-    let remote_path = opt
+    const CHUNK_SIZE: usize = 1024 * 1024 * 100; // 100MiB
-        .remote
-        .path
-        .to_str()
-        .ok_or_else(|| anyhow::format_err!("path not utf-8"))?;
-    let mut receive = session.channel_session()?;
-    receive.exec(&format!(r#"btrfs receive "{}""#, remote_path,))?;
 
-    // pipe send to receive
+    let (data_tx, data_rx) = mpsc::sync_channel(10);
-    let num_bytes = io::copy(&mut send_stdout, &mut receive)?;
+    let tmp_path = opt.remote.path.join(TMP_FOLDER);
-    info!("[{}] sent {} bytes", snapshot, num_bytes);
 
-    // wait for send to complete
+    // spawn a thread to stream data from `btrfs send` in chunks
-    let local_out = send.wait_with_output()?;
+    thread::spawn(move || -> io::Result<()> {
-    if !local_out.status.success() {
+        'outer: for _chunk in 0.. {
-        let stderr = std::str::from_utf8(&local_out.stderr)
+            let mut buf: Vec<u8> = vec![0u8; CHUNK_SIZE];
-            .unwrap_or("failed to parse stderr, not valid utf8");
+            let mut len = 0;
-        anyhow::bail!("btrfs send failed\nstderr:\n{}", stderr);
+            loop {
+                let free = &mut buf[len..];
+                let n = send_stdout.read(free)?;
+                len += n;
+
+                if n == 0 || n == free.len() {
+                    buf.truncate(len);
+                    if data_tx.send(buf).is_err() {
+                        break 'outer;
+                    }
+
+                    // check if we reached EOF
+                    if n == 0 {
+                        break 'outer;
+                    } else {
+                        continue 'outer;
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    });
+
+    let mut byte_count = 0;
+    let mut i = 0;
+    while let Ok(data) = data_rx.recv() {
+        byte_count += data.len();
+        info!("[{}] uploading {} bytes...", snapshot, byte_count);
+        let snapshot_file = tmp_path.join(format!("{:016}", i));
+        let mut ch = session.scp_send(&snapshot_file, 0o600, data.len() as u64, None)?;
+        ch.write_all(&data)?;
+        i += 1;
     }
 
-    // wait for receive to complete
+    info!(
-    receive.send_eof()?;
+        "[{}] re-creating snapshot (this can take a while)",
-    let mut remote_err = String::new();
+        snapshot
-    receive.stderr().read_to_string(&mut remote_err)?;
+    );
-    receive.wait_close()?;
+    let remote_path = path_as_utf8(&opt.remote.path)?;
-    let status = receive.exit_status()?;
+    let tmp_path = path_as_utf8(&tmp_path)?;
-    if status != 0 {
+    let cmd = format!(
-        anyhow::bail!("btrfs receive failed\nstderr:\n{}", remote_err);
+        r#"cat "{}"/* | btrfs receive -e "{}""#,
+        tmp_path, remote_path
+    );
+    let out = do_cmd(session, &cmd)?;
+
+    let time_elapsed = start_time.elapsed();
+
+    if out.exit_status != 0 {
+        anyhow::bail!(
+            "btrfs receive failed\nstdout:\n{}\nstderr:\n{}",
+            out.stdout,
+            out.stderr
+        );
+    }
+
+    info!(
+        "[{}] snapshot was {} bytes, time taken was {}",
+        snapshot,
+        byte_count,
+        format_duration(time_elapsed)
+    );
+
+    if !clear_tmp_dir(opt, session)? {
+        anyhow::bail!("failed to remove {} dir", TMP_FOLDER);
     }
 
     Ok(())

src/local.rs

@@ -16,7 +16,7 @@ pub fn file_list(opt: &Opt) -> anyhow::Result<FileList> {
 
         let name = match entry.file_name().into_string() {
             Ok(name) => name,
-            Err(_) => continue,
+            Err(_) => continue, // ignore names that aren't valid utf-8
         };
 
         let date = DateTime::parse_from_rfc3339(&name)?;

src/main.rs

@@ -5,11 +5,12 @@ mod actions;
 mod local;
 mod planner;
 mod remote;
-mod snapshot;
+mod util;
 
 use actions::{list, show_plan, sync};
 use chrono::{DateTime, FixedOffset};
-use clap::{AppSettings, Clap};
+use clap::{crate_version, Parser};
+use log::LevelFilter;
 use remote::Remote;
 use std::collections::BTreeMap;
 use std::path::PathBuf;
@@ -18,9 +19,17 @@ pub type TimeStamp = DateTime<FixedOffset>;
 pub type FileList = BTreeMap<TimeStamp, String>;
 
 /// Backup btrfs snapshots over SSH
-#[derive(Clap)]
+#[derive(Parser)]
-#[clap(setting = AppSettings::ColoredHelp)]
+#[clap(version = crate_version!())]
 pub struct Opt {
+    /// Log more stuff
+    #[clap(long, short, parse(from_occurrences))]
+    verbose: u8,
+
+    /// Do not output anything but errors.
+    #[clap(long, short)]
+    quiet: bool,
+
     /// The path of the backup directory on the local filesystem
     #[clap(short = 'l', long)]
     path: PathBuf,
@@ -41,16 +50,21 @@ pub struct Opt {
     action: Action,
 }
 
-#[derive(Clap)]
+#[derive(Parser)]
 pub enum Action {
     /// Perform a backup
     Backup {
+        /// Backup all files, not just the most recent ones
         #[clap(long)]
         all: bool,
     },
 
     /// Generate and show a backup plan
-    ShowPlan,
+    ShowPlan {
+        /// Backup all files, not just the most recent ones
+        #[clap(long)]
+        all: bool,
+    },
 
     /// List all backups, and where they reside
     List,
@@ -59,11 +73,20 @@ pub enum Action {
 fn main() -> anyhow::Result<()> {
     let opt = Opt::parse();
 
-    pretty_env_logger::init();
+    let log_level = match opt.verbose {
+        0 if opt.quiet => LevelFilter::Error,
+        0 => LevelFilter::Info,
+        1 => LevelFilter::Debug,
+        2.. => LevelFilter::Trace,
+    };
+
+    pretty_env_logger::formatted_builder()
+        .filter(None, log_level)
+        .init();
 
     match opt.action {
         Action::Backup { all } => sync::run(&opt, all)?,
-        Action::ShowPlan => show_plan::run(&opt)?,
+        Action::ShowPlan { all } => show_plan::run(&opt, all)?,
         Action::List => list::run(&opt)?,
     }
 

src/planner.rs

@@ -8,7 +8,7 @@ pub enum Presence {
     LocalAndRemote,
 }
 
-/// For every local file that is not in the remote, return a backup plan.
+/// Check which files exist on remote, local, or both
 pub fn presence(local: &FileList, remote: &FileList) -> BTreeMap<TimeStamp, Presence> {
     let mut presence = BTreeMap::new();
 
@@ -41,17 +41,24 @@ pub struct Plan {
     pub transfers: BTreeMap<TimeStamp, TransferKind>,
 }
 
-/// For every local file that is not in the remote, return a backup plan.
+/// For every trailing local file that is not in the remote, return a backup plan.
 ///
 /// The backup plans may depend on each other, so they must be executed in order.
-pub fn plan(local: &FileList, remote: &FileList) -> Plan {
+///
-    // go through the local files in order, starting with the latest
+/// Set `include_all` to include all local files, not just the most recent.
-    let upload_list: BTreeSet<_> = local
+pub fn plan(local: &FileList, remote: &FileList, include_all: bool) -> Plan {
-        .keys()
+    let upload_list: BTreeSet<_> = {
-        .rev()
+        // go through the local files in order, starting with the most recent
-        // keep going while the file doesn't exist in the remote
+        let local = local.keys().rev();
-        .take_while(|ts| !remote.contains_key(ts))
+
-        .collect();
+        if include_all {
+            // take all files that doesn't exist in the remote
+            local.filter(|ts| !remote.contains_key(ts)).collect()
+        } else {
+            // take only the most recent files that doesn't exist in the remote
+            local.take_while(|ts| !remote.contains_key(ts)).collect()
+        }
+    };
 
     // find the closest parent file of the first planned upload
     let head_item = upload_list.iter().next().copied();

src/remote.rs

@@ -15,14 +15,15 @@ pub struct Remote {
 }
 
 pub fn connect(opt: &Opt) -> anyhow::Result<Session> {
-    let stream = TcpStream::connect(&opt.remote.remote)?;
-    let mut session = Session::new()?;
-    session.set_tcp_stream(stream);
-    session.handshake()?;
     info!(
         r#"connecting to {}@{}"#,
         opt.remote.username, opt.remote.remote,
     );
+
+    let stream = TcpStream::connect(&opt.remote.remote)?;
+    let mut session = Session::new()?;
+    session.set_tcp_stream(stream);
+    session.handshake()?;
     session.userauth_pubkey_file(
         &opt.remote.username,
         None,
@@ -32,6 +33,7 @@ pub fn connect(opt: &Opt) -> anyhow::Result<Session> {
     if !session.authenticated() {
         anyhow::bail!("ssh not authenticated");
     }
+    session.set_allow_sigpipe(true);
 
     Ok(session)
 }
@@ -39,7 +41,7 @@ pub fn connect(opt: &Opt) -> anyhow::Result<Session> {
 pub fn file_list(opt: &Opt, session: &Session) -> anyhow::Result<FileList> {
     let mut channel = session.channel_session()?;
     channel.exec(&format!(
-        r#"ls -1N "{}""#,
+        r#"ls -1NU "{}""#,
         opt.remote
             .path
             .to_str()
@@ -51,7 +53,6 @@ pub fn file_list(opt: &Opt, session: &Session) -> anyhow::Result<FileList> {
     let mut list = BTreeMap::new();
     for file in output.lines() {
         let date = DateTime::parse_from_rfc3339(file)?;
-        //let path = PathBuf::from(file);
         list.insert(date, file.to_string());
     }
 

src/util.rs

@@ -0,0 +1,20 @@
+use std::path::Path;
+use std::time::Duration;
+
+pub fn format_duration(d: Duration) -> String {
+    let seconds = d.as_secs_f32() % 60.0;
+    let minutes = d.as_secs() / 60 % 60;
+    let hours = d.as_secs() / 60 / 60;
+
+    match (hours, minutes) {
+        (0, 0) => format!("{:.2}s", seconds),
+        (0, _) => format!("{}m {:.2}s", minutes, seconds),
+        (_, 0) => format!("{}h {:.2}s", hours, seconds),
+        (_, _) => format!("{}h {}m {:.2}s", hours, minutes, seconds),
+    }
+}
+
+pub fn path_as_utf8(path: &Path) -> anyhow::Result<&str> {
+    path.to_str()
+        .ok_or_else(|| anyhow::format_err!("path not utf-8"))
+}