Merge pull request #37 from ikatson/refactor-split-up-torrent-state

A ton of new features + huge refactor

.cargo/config

@@ -1,2 +1,6 @@
 [target.arm-unknown-linux-gnueabihf]
 rustflags = ["-l", "atomic"]
+
+[target.armv7-unknown-linux-gnueabihf]
+# Workaround for: https://github.com/rust-lang/compiler-builtins/issues/420
+rustflags = ["-C", "link-arg=-Wl,--allow-multiple-definition"]

.github/workflows/release.yml

@@ -29,7 +29,7 @@ jobs:
     - name: install linux cross compiler
       run:
         brew tap messense/macos-cross-toolchains &&
-        brew install x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu arm-unknown-linux-gnueabihf
+        brew install x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu arm-unknown-linux-gnueabihf armv7-unknown-linux-gnueabihf
 
     - name: Make a directory for output artifacts
       run:
@@ -41,12 +41,18 @@ jobs:
         make release-linux-x86_64 &&
         mv target/x86_64-unknown-linux-gnu/release-github/rqbit target/artifacts/rqbit-linux-static-x86_64
 
-    - name: Build release linux arm32bit binary
+    - name: Build release linux armv6 binary
       run:
         rustup target install arm-unknown-linux-gnueabihf &&
         make release-linux-armv6 &&
         mv target/arm-unknown-linux-gnueabihf/release-github/rqbit target/artifacts/rqbit-linux-static-arm32
 
+    - name: Build release linux armv7 binary
+      run:
+        rustup target install armv7-unknown-linux-gnueabihf &&
+        make release-linux-armv7 &&
+        mv target/armv7-unknown-linux-gnueabihf/release-github/rqbit target/artifacts/rqbit-linux-static-arm32
+
     - name: Build release linux aarch64 binary
       run:
         rustup target install aarch64-unknown-linux-gnu &&

Cargo.lock

@@ -80,6 +80,28 @@ version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
+[[package]]
+name = "async-stream"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.74"
@@ -317,6 +339,43 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787"
+dependencies = [
+ "futures-core",
+ "prost",
+ "prost-types",
+ "tonic",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e"
+dependencies = [
+ "console-api",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "futures-task",
+ "hdrhistogram",
+ "humantime",
+ "prost-types",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -342,6 +401,34 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -371,7 +458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if",
- "hashbrown",
+ "hashbrown 0.14.2",
  "lock_api",
  "once_cell",
  "parking_lot_core",
@@ -451,6 +538,16 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
+[[package]]
+name = "flate2"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -624,19 +721,38 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 2.1.0",
  "slab",
  "tokio",
  "tokio-util",
  "tracing",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "hashbrown"
 version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
 
+[[package]]
+name = "hdrhistogram"
+version = "7.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
+dependencies = [
+ "base64",
+ "byteorder",
+ "flate2",
+ "nom",
+ "num-traits",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -701,6 +817,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "0.14.27"
@@ -739,6 +861,18 @@ dependencies = [
  "tokio-rustls",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.5.0"
@@ -762,6 +896,16 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -769,7 +913,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
 dependencies = [
  "equivalent",
- "hashbrown",
+ "hashbrown 0.14.2",
 ]
 
 [[package]]
@@ -787,6 +931,15 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.12.0"
@@ -847,7 +1000,7 @@ dependencies = [
 
 [[package]]
 name = "librqbit"
-version = "3.3.0"
+version = "4.0.0-beta.0"
 dependencies = [
  "anyhow",
  "axum",
@@ -860,7 +1013,7 @@ dependencies = [
  "futures",
  "hex 0.4.3",
  "http",
- "itertools",
+ "itertools 0.12.0",
  "librqbit-bencode",
  "librqbit-buffers",
  "librqbit-clone-to-owned",
@@ -913,29 +1066,31 @@ version = "2.2.1"
 
 [[package]]
 name = "librqbit-core"
-version = "3.0.0"
+version = "3.1.0"
 dependencies = [
  "anyhow",
  "hex 0.4.3",
- "itertools",
+ "itertools 0.12.0",
  "librqbit-bencode",
  "librqbit-buffers",
  "librqbit-clone-to-owned",
  "parking_lot",
  "serde",
+ "tokio",
+ "tracing",
  "url",
  "uuid",
 ]
 
 [[package]]
 name = "librqbit-dht"
-version = "3.1.0"
+version = "3.2.0"
 dependencies = [
  "anyhow",
  "directories",
  "futures",
  "hex 0.4.3",
- "indexmap",
+ "indexmap 2.1.0",
  "leaky-bucket",
  "librqbit-bencode",
  "librqbit-clone-to-owned",
@@ -952,7 +1107,7 @@ dependencies = [
 
 [[package]]
 name = "librqbit-peer-protocol"
-version = "3.0.0"
+version = "3.1.0"
 dependencies = [
  "anyhow",
  "bincode",
@@ -1023,6 +1178,12 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -1061,6 +1222,16 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -1279,7 +1450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap",
+ "indexmap 2.1.0",
 ]
 
 [[package]]
@@ -1335,6 +1506,38 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
+dependencies = [
+ "anyhow",
+ "itertools 0.11.0",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
@@ -1503,10 +1706,11 @@ dependencies = [
 
 [[package]]
 name = "rqbit"
-version = "3.3.0"
+version = "4.0.0-beta.0"
 dependencies = [
  "anyhow",
  "clap",
+ "console-subscriber",
  "futures",
  "librqbit",
  "librqbit-dht",
@@ -1888,9 +2092,20 @@ dependencies = [
  "pin-project-lite",
  "socket2 0.5.5",
  "tokio-macros",
+ "tracing",
  "windows-sys",
 ]
 
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-macros"
 version = "2.2.0"
@@ -1948,6 +2163,33 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64",
+ "bytes",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
 [[package]]
 name = "tower"
 version = "0.4.13"
@@ -1956,9 +2198,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
+ "rand",
+ "slab",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
  "tracing",

Makefile

@@ -75,7 +75,7 @@ release-linux-current-target: target/openssl-linux/$(TARGET)/lib/libssl.a
 	cargo build  --profile release-github --target=$(TARGET)
 
 @PHONY: release-linux
-release-linux: release-linux-x86_64 release-linux-aarch64 release-linux-armv6
+release-linux: release-linux-x86_64 release-linux-aarch64 release-linux-armv6 release-linux-armv7
 
 @PHONY: release-linux-x86_64
 release-linux-x86_64:
@@ -105,6 +105,16 @@ release-linux-armv6:
 	LDFLAGS=-latomic \
 	$(MAKE) release-linux-current-target
 
+# armv7-unknown-linux-gnueabihf
+@PHONY: release-linux-armv7
+release-linux-armv7:
+	TARGET=armv7-unknown-linux-gnueabihf \
+	TARGET_SNAKE_CASE=armv7_unknown_linux_gnueabihf \
+	TARGET_SNAKE_UPPER_CASE=ARMV7_UNKNOWN_LINUX_GNUEABIHF \
+	CROSS_COMPILE_PREFIX=armv7-linux-gnueabihf \
+	OPENSSL_CONFIGURE_NAME=linux-generic32 \
+	$(MAKE) release-linux-current-target
+
 
 @PHONY: release-all
 release-all: release-windows release-linux release-macos-universal

TODO.md

@@ -5,17 +5,31 @@
 - [x] tracing instead of logging. Debugging peers: RUST_LOG=[{peer=.*}]=debug
   test-log for tests
 - [x] reopen read only is bugged
-- [ ] initializing/checking
+- [x] initializing/checking
-  - [ ] blocks the whole process. Need to break it up. On slower devices (rpi) just hangs for a good while
+  - [x] blocks the whole process. Need to break it up. On slower devices (rpi) just hangs for a good while
-  - [ ] checking torrents should be visible right away
+  - [x] checking torrents should be visible right away
 - [ ] server persistence
   - [ ] it would be nice to restart the server and keep the state
-- [ ] torrent actions
+- [x] torrent actions
-  - [ ] pause/unpause
+  - [x] pause/unpause
-  - [ ] remove including from disk
+  - [x] remove including from disk
 - [ ] DHT
-  - [ ] for torrents with a few seeds might be cool to re-query DHT once in a while
+  - [ ] for torrents with a few seeds might be cool to re-query DHT once in a while.
-  - [ ] it's sending many requests now way too fast, locks up Mac OS UI annoyingly
+  - [x] it's sending many requests now way too fast, locks up Mac OS UI annoyingly
 
 someday:
-- [ ] cancellation from the client-side for the lib (i.e. stop the torrent manager)
+- [x] cancellation from the client-side for the lib (i.e. stop the torrent manager)
+
+
+refactor:
+- [x] where are peers stored
+- [x] http api pause/unpause etc
+- [x] when a live torrent fails writing to disk, it should transition to error state
+- [x] something is wrong when unpausing - can't finish. Recalculate needed/have from chunk tracker.
+- [x] silence this: WARN torrent{id=0}:external_peer_adder: librqbit::spawn_utils: finished with error: no longer live
+
+- [x] start from error state should be possible from UI
+- [ ] if the torrent was completed, not need to re-check it
+- [x] checking is very slow on raspberry
+  checked. nothing much can be done here. Even if raspberry's own libssl.so is used it's still super slow (sha1)
+- [ ] .rqbit-session.json file has 0 bytes when disk full. I guess fs::rename does this when disk is full? at least on linux

crates/dht/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "librqbit-dht"
-version = "3.1.0"
+version = "3.2.0"
 edition = "2021"
 description = "DHT implementation, used in rqbit torrent client."
 license = "Apache-2.0"
@@ -33,7 +33,7 @@ indexmap = "2"
 directories = "5"
 
 clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
-librqbit-core = {path="../librqbit_core", version = "3.0.0"}
+librqbit-core = {path="../librqbit_core", version = "3.1.0"}
 
 [dev-dependencies]
 tracing-subscriber = "0.3"

crates/dht/examples/dht.rs

@@ -17,7 +17,7 @@ async fn main() -> anyhow::Result<()> {
     tracing_subscriber::fmt::init();
 
     let dht = Dht::new().await.context("error initializing DHT")?;
-    let mut stream = dht.get_peers(info_hash).await?;
+    let mut stream = dht.get_peers(info_hash)?;
 
     let stats_printer = async {
         loop {

crates/dht/src/dht.rs

@@ -18,7 +18,7 @@ use bencode::ByteString;
 use futures::{stream::FuturesUnordered, Stream, StreamExt};
 use indexmap::IndexSet;
 use leaky_bucket::RateLimiter;
-use librqbit_core::{id20::Id20, peer_id::generate_peer_id};
+use librqbit_core::{id20::Id20, peer_id::generate_peer_id, spawn_utils::spawn};
 use parking_lot::RwLock;
 use rand::Rng;
 use serde::Serialize;
@@ -27,7 +27,7 @@ use tokio::{
     sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
 };
 use tokio_stream::wrappers::{errors::BroadcastStreamRecvError, BroadcastStream};
-use tracing::{debug, info, trace, warn};
+use tracing::{debug, debug_span, error_span, info, trace, warn, Instrument};
 
 #[derive(Debug, Serialize)]
 pub struct DhtStats {
@@ -513,7 +513,7 @@ impl DhtWorker {
         bootstrap_addrs: &[String],
     ) -> anyhow::Result<()> {
         let (out_tx, mut out_rx) = channel(1);
-        let framer = run_framer(&self.socket, in_rx, out_tx);
+        let framer = run_framer(&self.socket, in_rx, out_tx).instrument(debug_span!("dht_framer"));
 
         let bootstrap = async {
             let mut futs = FuturesUnordered::new();
@@ -521,34 +521,40 @@ impl DhtWorker {
             for addr in bootstrap_addrs.iter() {
                 let this = &self;
                 let in_tx = &in_tx;
-                futs.push(async move {
+                futs.push(
-                    match tokio::net::lookup_host(addr).await {
+                    async move {
-                        Ok(addrs) => {
+                        match tokio::net::lookup_host(addr).await {
-                            for addr in addrs {
+                            Ok(addrs) => {
-                                let request = this
+                                for addr in addrs {
-                                    .state
+                                    let request = this
-                                    .write()
+                                        .state
-                                    .create_request(Request::FindNode(this.peer_id), addr);
+                                        .write()
-                                in_tx.send((request, addr))?;
+                                        .create_request(Request::FindNode(this.peer_id), addr);
+                                    in_tx.send((request, addr))?;
+                                }
+                            }
+                            Err(e) => {
+                                warn!("error looking up {}: {}", addr, e);
+                                return Err(e.into());
                             }
                         }
-                        Err(e) => warn!("error looking up {}: {}", addr, e),
+                        Ok::<_, anyhow::Error>(())
                     }
-                    Ok::<_, anyhow::Error>(())
+                    .instrument(error_span!("dht_bootstrap", addr = addr)),
-                });
+                );
             }
             let mut successes = 0;
             while let Some(resp) = futs.next().await {
-                match resp {
+                if resp.is_ok() {
-                    Ok(_) => successes += 1,
+                    successes += 1
-                    Err(e) => warn!("error in one of the bootstrappers: {}", e),
                 }
             }
             if successes == 0 {
                 anyhow::bail!("bootstrapping did not succeed")
             }
             Ok(())
-        };
+        }
+        .instrument(debug_span!("dht_bootstrapper"));
         let mut bootstrap_done = false;
 
         let response_reader = {
@@ -563,7 +569,8 @@ impl DhtWorker {
                     "closed response reader, nowhere to send results to, DHT closed"
                 ))
             }
-        };
+        }
+        .instrument(debug_span!("dht_responese_reader"));
 
         tokio::pin!(framer);
         tokio::pin!(bootstrap);
@@ -676,7 +683,7 @@ impl Dht {
             listen_addr,
         )));
 
-        tokio::spawn({
+        spawn(error_span!("dht"), {
             let state = state.clone();
             async move {
                 let worker = DhtWorker {
@@ -684,13 +691,14 @@ impl Dht {
                     peer_id,
                     state,
                 };
-                let result = worker.start(in_tx, in_rx, &bootstrap_addrs).await;
+                worker.start(in_tx, in_rx, &bootstrap_addrs).await?;
-                warn!("DHT worker finished with {:?}", result);
+                Ok(())
             }
         });
         Ok(Dht { state })
     }
-    pub async fn get_peers(
+
+    pub fn get_peers(
         &self,
         info_hash: Id20,
     ) -> anyhow::Result<impl Stream<Item = SocketAddr> + Unpin> {

crates/dht/src/persistence.rs

@@ -1,5 +1,6 @@
 // TODO: this now stores only the routing table, but we also need AT LEAST the same socket address...
 
+use librqbit_core::spawn_utils::spawn;
 use serde::{Deserialize, Serialize};
 use std::fs::OpenOptions;
 use std::io::{BufReader, BufWriter};
@@ -8,8 +9,7 @@ use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use anyhow::Context;
-use tokio::spawn;
+use tracing::{debug, error, error_span, info, trace, warn};
-use tracing::{debug, error, info, trace, warn};
 
 use crate::dht::{Dht, DhtConfig};
 use crate::routing_table::RoutingTable;
@@ -110,7 +110,7 @@ impl PersistentDht {
         };
         let dht = Dht::with_config(dht_config).await?;
 
-        spawn({
+        spawn(error_span!("dht_persistence"), {
             let dht = dht.clone();
             let dump_interval = config
                 .dump_interval

crates/librqbit/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "librqbit"
-version = "3.3.0"
+version = "4.0.0-beta.0"
 authors = ["Igor Katson <igor.katson@gmail.com>"]
 edition = "2021"
 description = "The main library used by rqbit torrent client. The binary is just a small wrapper on top of it."
@@ -24,11 +24,11 @@ rust-tls = ["reqwest/rustls-tls"]
 [dependencies]
 bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
 buffers = {path = "../buffers", package="librqbit-buffers", version = "2.2.1"}
-librqbit-core = {path = "../librqbit_core", version = "3.0.0"}
+librqbit-core = {path = "../librqbit_core", version = "3.1.0"}
 clone_to_owned = {path = "../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
-peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.0.0"}
+peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.1.0"}
 sha1w = {path = "../sha1w", default-features=false, package="librqbit-sha1-wrapper", version="2.2.1"}
-dht = {path = "../dht", package="librqbit-dht", version="3.1.0"}
+dht = {path = "../dht", package="librqbit-dht", version="3.2.0"}
 
 tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
 axum = {version = "0.6"}

crates/librqbit/examples/ubuntu.rs

@@ -16,6 +16,10 @@ const MAGNET_LINK: &str = "magnet:?xt=urn:btih:cab507494d02ebb1178b38f2e9d7be299
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
     // Output logs to console.
+    match std::env::var("RUST_LOG") {
+        Ok(_) => {}
+        Err(_) => std::env::set_var("RUST_LOG", "info"),
+    }
     tracing_subscriber::fmt::init();
 
     let output_dir = std::env::args()
@@ -32,31 +36,29 @@ async fn main() -> Result<(), anyhow::Error> {
         .add_torrent(
             AddTorrent::from_url(MAGNET_LINK),
             Some(AddTorrentOptions {
-                // Set this to true to allow writing on top of existing files.
+                // Allow writing on top of existing files.
-                // If the file is partially downloaded, librqbit will only download the
+                overwrite: true,
-                // missing pieces.
-                //
-                // Otherwise it will throw an error that the file exists.
-                overwrite: false,
                 ..Default::default()
             }),
         )
         .await
         .context("error adding torrent")?
     {
-        AddTorrentResponse::Added(handle) => handle,
+        AddTorrentResponse::Added(_, handle) => handle,
         // For a brand new session other variants won't happen.
         _ => unreachable!(),
     };
 
+    info!("Details: {:?}", &handle.info().info);
+
     // Print stats periodically.
     tokio::spawn({
         let handle = handle.clone();
         async move {
             loop {
                 tokio::time::sleep(Duration::from_secs(1)).await;
-                let stats = handle.torrent_state().stats_snapshot();
+                let stats = handle.stats();
-                info!("stats: {stats:?}");
+                info!("{stats:}");
             }
         }
     });

crates/librqbit/src/chunk_tracker.rs

@@ -1,6 +1,6 @@
 use librqbit_core::lengths::{ChunkInfo, Lengths, ValidPieceIndex};
 use peer_binary_protocol::Piece;
-use tracing::{debug, info};
+use tracing::{debug, trace};
 
 use crate::type_aliases::BF;
 
@@ -75,9 +75,11 @@ impl ChunkTracker {
             priority_piece_ids,
         }
     }
-    pub fn get_needed_pieces(&self) -> &BF {
+
-        &self.needed_pieces
+    pub fn get_lengths(&self) -> &Lengths {
+        &self.lengths
     }
+
     pub fn get_have_pieces(&self) -> &BF {
         &self.have
     }
@@ -85,6 +87,16 @@ impl ChunkTracker {
         self.needed_pieces.set(index.get() as usize, false)
     }
 
+    pub fn calc_have_bytes(&self) -> u64 {
+        self.have
+            .iter_ones()
+            .filter_map(|piece_id| {
+                let piece_id = self.lengths.validate_piece_index(piece_id as u32)?;
+                Some(self.lengths.piece_length(piece_id) as u64)
+            })
+            .sum()
+    }
+
     pub fn iter_needed_pieces(&self) -> impl Iterator<Item = usize> + '_ {
         self.priority_piece_ids
             .iter()
@@ -117,7 +129,7 @@ impl ChunkTracker {
     }
 
     pub fn mark_piece_broken(&mut self, index: ValidPieceIndex) -> bool {
-        info!("remarking piece={} as broken", index);
+        debug!("remarking piece={} as broken", index);
         self.needed_pieces.set(index.get() as usize, true);
         self.chunk_status
             .get_mut(self.lengths.chunk_range(index))
@@ -132,13 +144,6 @@ impl ChunkTracker {
         self.have.set(idx.get() as usize, true);
     }
 
-    pub fn is_chunk_downloaded(&self, chunk: &ChunkInfo) -> bool {
-        *self
-            .chunk_status
-            .get(chunk.absolute_index as usize)
-            .unwrap()
-    }
-
     pub fn is_chunk_ready_to_upload(&self, chunk: &ChunkInfo) -> bool {
         self.have
             .get(chunk.piece_index.get() as usize)
@@ -165,9 +170,11 @@ impl ChunkTracker {
             return Some(ChunkMarkingResult::PreviouslyCompleted);
         }
         chunk_range.set(chunk_info.chunk_index as usize, true);
-        debug!(
+        trace!(
             "piece={}, chunk_info={:?}, bits={:?}",
-            piece.index, chunk_info, chunk_range,
+            piece.index,
+            chunk_info,
+            chunk_range,
         );
 
         if chunk_range.all() {

crates/librqbit/src/dht_utils.rs

@@ -107,7 +107,7 @@ mod tests {
 
         let info_hash = Id20::from_str("cf3ea75e2ebbd30e0da6e6e215e2226bf35f2e33").unwrap();
         let dht = Dht::new().await.unwrap();
-        let peer_rx = dht.get_peers(info_hash).await.unwrap();
+        let peer_rx = dht.get_peers(info_hash).unwrap();
         let peer_id = generate_peer_id();
         match read_metainfo_from_peer_receiver(peer_id, info_hash, peer_rx, None).await {
             ReadMetainfoResult::Found { info, .. } => dbg!(info),

crates/librqbit/src/file_ops.rs

@@ -2,7 +2,10 @@ use std::{
     fs::File,
     io::{Read, Seek, SeekFrom, Write},
     marker::PhantomData,
-    sync::Arc,
+    sync::{
+        atomic::{AtomicU64, Ordering},
+        Arc,
+    },
 };
 
 use anyhow::Context;
@@ -11,14 +14,14 @@ use librqbit_core::{
     lengths::{ChunkInfo, Lengths, ValidPieceIndex},
     torrent_metainfo::{FileIteratorName, TorrentMetaV1Info},
 };
-use tracing::{debug, trace, warn};
 use parking_lot::Mutex;
 use peer_binary_protocol::Piece;
 use sha1w::ISha1;
+use tracing::{debug, trace, warn};
 
 use crate::type_aliases::{PeerHandle, BF};
 
-pub struct InitialCheckResults {
+pub(crate) struct InitialCheckResults {
     pub needed_pieces: BF,
     pub have_pieces: BF,
     pub have_bytes: u64,
@@ -43,7 +46,7 @@ pub fn update_hash_from_file<Sha1: ISha1>(
     Ok(())
 }
 
-pub struct FileOps<'a, Sha1> {
+pub(crate) struct FileOps<'a, Sha1> {
     torrent: &'a TorrentMetaV1Info<ByteString>,
     files: &'a [Arc<Mutex<File>>],
     lengths: &'a Lengths,
@@ -67,6 +70,7 @@ impl<'a, Sha1Impl: ISha1> FileOps<'a, Sha1Impl> {
     pub fn initial_check(
         &self,
         only_files: Option<&[usize]>,
+        progress: &AtomicU64,
     ) -> anyhow::Result<InitialCheckResults> {
         let mut needed_pieces = BF::from_vec(vec![0u8; self.lengths.piece_bitfield_bytes()]);
         let mut have_pieces = BF::from_vec(vec![0u8; self.lengths.piece_bitfield_bytes()]);
@@ -125,6 +129,7 @@ impl<'a, Sha1Impl: ISha1> FileOps<'a, Sha1Impl> {
             let mut piece_remaining = piece_info.len as usize;
             let mut some_files_broken = false;
             let mut at_least_one_file_required = current_file.full_file_required;
+            progress.fetch_add(piece_info.len as u64, Ordering::Relaxed);
 
             while piece_remaining > 0 {
                 let mut to_read_in_file =

crates/librqbit/src/http_api.rs

@@ -2,29 +2,28 @@ use anyhow::Context;
 use axum::body::Bytes;
 use axum::extract::{Path, Query, State};
 use axum::response::IntoResponse;
-use axum::routing::get;
+use axum::routing::{get, post};
 use buffers::ByteString;
-use dht::{Dht, DhtStats};
+use dht::DhtStats;
 use http::StatusCode;
 use itertools::Itertools;
 use librqbit_core::id20::Id20;
 use librqbit_core::torrent_metainfo::TorrentMetaV1Info;
-use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
 use std::net::SocketAddr;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use tokio::sync::mpsc::UnboundedSender;
 use tracing::{info, warn};
 
 use axum::Router;
 
 use crate::http_api_error::{ApiError, ApiErrorExt};
-use crate::peer_state::PeerStatsFilter;
 use crate::session::{
-    AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session,
+    AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session, TorrentId,
 };
-use crate::torrent_manager::TorrentManagerHandle;
+use crate::torrent_state::peer::stats::snapshot::{PeerStatsFilter, PeerStatsSnapshot};
-use crate::torrent_state::StatsSnapshot;
+use crate::torrent_state::stats::{LiveStats, TorrentStats};
+use crate::torrent_state::ManagedTorrentHandle;
 
 // Public API
 #[derive(Clone)]
@@ -33,16 +32,17 @@ pub struct HttpApi {
 }
 
 impl HttpApi {
-    pub fn new(session: Arc<Session>) -> Self {
+    pub fn new(session: Arc<Session>, rust_log_reload_tx: Option<UnboundedSender<String>>) -> Self {
         Self {
-            inner: Arc::new(ApiInternal::new(session)),
+            inner: Arc::new(ApiInternal::new(session, rust_log_reload_tx)),
         }
     }
-    pub fn add_torrent_handle(&self, handle: TorrentManagerHandle) -> usize {
-        self.inner.add_torrent_handle(handle)
-    }
 
-    pub async fn make_http_api_and_run(self, addr: SocketAddr) -> anyhow::Result<()> {
+    pub async fn make_http_api_and_run(
+        self,
+        addr: SocketAddr,
+        read_only: bool,
+    ) -> anyhow::Result<()> {
         let state = self.inner;
 
         async fn api_root() -> impl IntoResponse {
@@ -54,11 +54,14 @@ impl HttpApi {
                     "GET /torrents": "List torrents (default torrent is 0)",
                     "GET /torrents/{index}": "Torrent details",
                     "GET /torrents/{index}/haves": "The bitfield of have pieces",
-                    "GET /torrents/{index}/stats": "Torrent stats",
+                    "GET /torrents/{index}/stats/v1": "Torrent stats",
                     "GET /torrents/{index}/peer_stats": "Per peer stats",
-                    // This is kind of not secure as it just reads any local file that it has access to,
+                    "POST /torrents/{index}/pause": "Pause torrent",
-                    // or any URL, but whatever, ok for our purposes / threat model.
+                    "POST /torrents/{index}/start": "Resume torrent",
+                    "POST /torrents/{index}/forget": "Forget about the torrent, keep the files",
+                    "POST /torrents/{index}/delete": "Forget about the torrent, remove the files",
                     "POST /torrents": "Add a torrent here. magnet: or http:// or a local file.",
+                    "POST /rust_log": "Set RUST_LOG to this post launch (for debugging)",
                     "GET /web/": "Web UI",
                 },
                 "server": "rqbit",
@@ -104,11 +107,18 @@ impl HttpApi {
             state.api_dump_haves(idx)
         }
 
-        async fn torrent_stats(
+        async fn torrent_stats_v0(
             State(state): State<ApiState>,
             Path(idx): Path<usize>,
         ) -> Result<impl IntoResponse> {
-            state.api_stats(idx).map(axum::Json)
+            state.api_stats_v0(idx).map(axum::Json)
+        }
+
+        async fn torrent_stats_v1(
+            State(state): State<ApiState>,
+            Path(idx): Path<usize>,
+        ) -> Result<impl IntoResponse> {
+            state.api_stats_v1(idx).map(axum::Json)
         }
 
         async fn peer_stats(
@@ -119,17 +129,62 @@ impl HttpApi {
             state.api_peer_stats(idx, filter).map(axum::Json)
         }
 
-        #[allow(unused_mut)]
+        async fn torrent_action_pause(
+            State(state): State<ApiState>,
+            Path(idx): Path<usize>,
+        ) -> Result<impl IntoResponse> {
+            state.api_torrent_action_pause(idx).map(axum::Json)
+        }
+
+        async fn torrent_action_start(
+            State(state): State<ApiState>,
+            Path(idx): Path<usize>,
+        ) -> Result<impl IntoResponse> {
+            state.api_torrent_action_start(idx).map(axum::Json)
+        }
+
+        async fn torrent_action_forget(
+            State(state): State<ApiState>,
+            Path(idx): Path<usize>,
+        ) -> Result<impl IntoResponse> {
+            state.api_torrent_action_forget(idx).map(axum::Json)
+        }
+
+        async fn torrent_action_delete(
+            State(state): State<ApiState>,
+            Path(idx): Path<usize>,
+        ) -> Result<impl IntoResponse> {
+            state.api_torrent_action_delete(idx).map(axum::Json)
+        }
+
+        async fn set_rust_log(
+            State(state): State<ApiState>,
+            new_value: String,
+        ) -> Result<impl IntoResponse> {
+            state.api_set_rust_log(new_value).map(axum::Json)
+        }
+
         let mut app = Router::new()
             .route("/", get(api_root))
+            .route("/rust_log", post(set_rust_log))
             .route("/dht/stats", get(dht_stats))
             .route("/dht/table", get(dht_table))
-            .route("/torrents", get(torrents_list).post(torrents_post))
+            .route("/torrents", get(torrents_list))
             .route("/torrents/:id", get(torrent_details))
             .route("/torrents/:id/haves", get(torrent_haves))
-            .route("/torrents/:id/stats", get(torrent_stats))
+            .route("/torrents/:id/stats", get(torrent_stats_v0))
+            .route("/torrents/:id/stats/v1", get(torrent_stats_v1))
             .route("/torrents/:id/peer_stats", get(peer_stats));
 
+        if !read_only {
+            app = app
+                .route("/torrents", post(torrents_post))
+                .route("/torrents/:id/pause", post(torrent_action_pause))
+                .route("/torrents/:id/start", post(torrent_action_start))
+                .route("/torrents/:id/forget", post(torrent_action_forget))
+                .route("/torrents/:id/delete", post(torrent_action_delete));
+        }
+
         #[cfg(feature = "webui")]
         {
             let webui_router = Router::new()
@@ -184,27 +239,6 @@ impl HttpApi {
 
 type Result<T> = std::result::Result<T, ApiError>;
 
-#[derive(Serialize)]
-struct Speed {
-    mbps: f64,
-    human_readable: String,
-}
-
-impl Speed {
-    fn new(mbps: f64) -> Self {
-        Self {
-            mbps,
-            human_readable: format!("{mbps:.2} MiB/s"),
-        }
-    }
-}
-
-impl From<f64> for Speed {
-    fn from(mbps: f64) -> Self {
-        Self::new(mbps)
-    }
-}
-
 #[derive(Serialize)]
 struct TorrentListResponseItem {
     id: usize,
@@ -223,41 +257,15 @@ pub struct TorrentDetailsResponseFile {
     pub included: bool,
 }
 
+#[derive(Default, Serialize)]
+struct EmptyJsonResponse {}
+
 #[derive(Serialize, Deserialize)]
 pub struct TorrentDetailsResponse {
     pub info_hash: String,
     pub files: Vec<TorrentDetailsResponseFile>,
 }
 
-struct DurationWithHumanReadable(Duration);
-
-impl Serialize for DurationWithHumanReadable {
-    fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        #[derive(Serialize)]
-        struct Tmp {
-            duration: Duration,
-            human_readable: String,
-        }
-        Tmp {
-            duration: self.0,
-            human_readable: format!("{:?}", self.0),
-        }
-        .serialize(serializer)
-    }
-}
-
-#[derive(Serialize)]
-struct StatsResponse {
-    snapshot: StatsSnapshot,
-    average_piece_download_time: Option<Duration>,
-    download_speed: Speed,
-    all_time_download_speed: Speed,
-    time_remaining: Option<DurationWithHumanReadable>,
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct ApiAddTorrentResponse {
     pub id: Option<usize>,
@@ -330,69 +338,94 @@ impl TorrentAddQueryParams {
 }
 
 // Private HTTP API internals. Agnostic of web framework.
-pub struct ApiInternal {
+struct ApiInternal {
-    dht: Option<Dht>,
-    startup_time: Instant,
-    torrent_managers: RwLock<Vec<TorrentManagerHandle>>,
     session: Arc<Session>,
+    rust_log_reload_tx: Option<UnboundedSender<String>>,
 }
 
 type ApiState = Arc<ApiInternal>;
 
 impl ApiInternal {
-    pub fn new(session: Arc<Session>) -> Self {
+    pub fn new(session: Arc<Session>, rust_log_reload_tx: Option<UnboundedSender<String>>) -> Self {
         Self {
-            dht: session.get_dht(),
-            startup_time: Instant::now(),
-            torrent_managers: RwLock::new(Vec::new()),
             session,
+            rust_log_reload_tx,
         }
     }
 
-    fn add_torrent_handle(&self, handle: TorrentManagerHandle) -> usize {
+    fn mgr_handle(&self, idx: TorrentId) -> Result<ManagedTorrentHandle> {
-        let mut g = self.torrent_managers.write();
+        self.session
-        let idx = g.len();
-        g.push(handle);
-        idx
-    }
-
-    fn mgr_handle(&self, idx: usize) -> Result<TorrentManagerHandle> {
-        self.torrent_managers
-            .read()
             .get(idx)
-            .cloned()
             .ok_or(ApiError::torrent_not_found(idx))
     }
 
     fn api_torrent_list(&self) -> TorrentListResponse {
-        TorrentListResponse {
+        let items = self.session.with_torrents(|torrents| {
-            torrents: self
+            torrents
-                .torrent_managers
-                .read()
-                .iter()
-                .enumerate()
                 .map(|(id, mgr)| TorrentListResponseItem {
                     id,
-                    info_hash: mgr.torrent_state().info_hash().as_string(),
+                    info_hash: mgr.info().info_hash.as_string(),
                 })
-                .collect(),
+                .collect()
-        }
+        });
+        TorrentListResponse { torrents: items }
     }
 
-    fn api_torrent_details(&self, idx: usize) -> Result<TorrentDetailsResponse> {
+    fn api_torrent_details(&self, idx: TorrentId) -> Result<TorrentDetailsResponse> {
         let handle = self.mgr_handle(idx)?;
-        let info_hash = handle.torrent_state().info_hash();
+        let info_hash = handle.info().info_hash;
         let only_files = handle.only_files();
-        make_torrent_details(&info_hash, handle.torrent_state().info(), only_files)
+        make_torrent_details(&info_hash, &handle.info().info, only_files.as_deref())
     }
 
-    fn api_peer_stats(
+    fn api_peer_stats(&self, idx: TorrentId, filter: PeerStatsFilter) -> Result<PeerStatsSnapshot> {
-        &self,
-        idx: usize,
-        filter: PeerStatsFilter,
-    ) -> Result<crate::peer_state::PeerStatsSnapshot> {
         let handle = self.mgr_handle(idx)?;
-        Ok(handle.torrent_state().per_peer_stats_snapshot(filter))
+        Ok(handle
+            .live()
+            .context("not live")?
+            .per_peer_stats_snapshot(filter))
+    }
+
+    fn api_torrent_action_pause(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
+        let handle = self.mgr_handle(idx)?;
+        handle
+            .pause()
+            .context("error pausing torrent")
+            .with_error_status_code(StatusCode::BAD_REQUEST)?;
+        Ok(Default::default())
+    }
+
+    fn api_torrent_action_start(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
+        let handle = self.mgr_handle(idx)?;
+        self.session
+            .unpause(&handle)
+            .context("error unpausing torrent")
+            .with_error_status_code(StatusCode::BAD_REQUEST)?;
+        Ok(Default::default())
+    }
+
+    fn api_torrent_action_forget(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
+        self.session
+            .delete(idx, false)
+            .context("error forgetting torrent")?;
+        Ok(Default::default())
+    }
+
+    fn api_torrent_action_delete(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
+        self.session
+            .delete(idx, true)
+            .context("error deleting torrent with files")?;
+        Ok(Default::default())
+    }
+
+    fn api_set_rust_log(&self, new_value: String) -> Result<EmptyJsonResponse> {
+        let tx = self
+            .rust_log_reload_tx
+            .as_ref()
+            .context("rust_log_reload_tx was not set")?;
+        tx.send(new_value)
+            .context("noone is listening to RUST_LOG changes")?;
+        Ok(Default::default())
     }
 
     pub async fn api_add_torrent(
@@ -407,11 +440,12 @@ impl ApiInternal {
             .context("error adding torrent")
             .with_error_status_code(StatusCode::BAD_REQUEST)?
         {
-            AddTorrentResponse::AlreadyManaged(managed) => {
+            AddTorrentResponse::AlreadyManaged(id, managed) => {
                 return Err(anyhow::anyhow!(
-                    "{:?} is already managed, downloaded to {:?}",
+                    "{:?} is already managed, id={}, downloaded to {:?}",
-                    managed.info_hash,
+                    managed.info_hash(),
-                    managed.output_folder
+                    id,
+                    &managed.info().out_dir
                 ))
                 .with_error_status_code(StatusCode::CONFLICT);
             }
@@ -424,14 +458,13 @@ impl ApiInternal {
                 details: make_torrent_details(&info_hash, &info, only_files.as_deref())
                     .context("error making torrent details")?,
             },
-            AddTorrentResponse::Added(handle) => {
+            AddTorrentResponse::Added(id, handle) => {
                 let details = make_torrent_details(
-                    &handle.torrent_state().info_hash(),
+                    &handle.info_hash(),
-                    handle.torrent_state().info(),
+                    &handle.info().info,
-                    handle.only_files(),
+                    handle.only_files().as_deref(),
                 )
                 .context("error making torrent details")?;
-                let id = self.add_torrent_handle(handle);
                 ApiAddTorrentResponse {
                     id: Some(id),
                     details,
@@ -442,45 +475,32 @@ impl ApiInternal {
     }
 
     fn api_dht_stats(&self) -> Result<DhtStats> {
-        self.dht
+        self.session
+            .get_dht()
             .as_ref()
             .map(|d| d.stats())
             .ok_or(ApiError::dht_disabled())
     }
 
     fn api_dht_table(&self) -> Result<impl Serialize> {
-        let dht = self.dht.as_ref().ok_or(ApiError::dht_disabled())?;
+        let dht = self.session.get_dht().ok_or(ApiError::dht_disabled())?;
         Ok(dht.with_routing_table(|r| r.clone()))
     }
 
-    fn api_stats(&self, idx: usize) -> Result<StatsResponse> {
+    fn api_stats_v0(&self, idx: TorrentId) -> Result<LiveStats> {
         let mgr = self.mgr_handle(idx)?;
-        let snapshot = mgr.torrent_state().stats_snapshot();
+        let live = mgr.live().context("torrent not live")?;
-        let estimator = mgr.speed_estimator();
+        Ok(LiveStats::from(&*live))
+    }
 
-        // Poor mans download speed computation
+    fn api_stats_v1(&self, idx: TorrentId) -> Result<TorrentStats> {
-        let elapsed = self.startup_time.elapsed();
+        let mgr = self.mgr_handle(idx)?;
-        let downloaded_bytes = snapshot.downloaded_and_checked_bytes;
+        Ok(mgr.stats())
-        let downloaded_mb = downloaded_bytes as f64 / 1024f64 / 1024f64;
-
-        Ok(StatsResponse {
-            average_piece_download_time: snapshot.average_piece_download_time(),
-            snapshot,
-            all_time_download_speed: (downloaded_mb / elapsed.as_secs_f64()).into(),
-            download_speed: estimator.download_mbps().into(),
-            time_remaining: estimator.time_remaining().map(DurationWithHumanReadable),
-        })
     }
 
     fn api_dump_haves(&self, idx: usize) -> Result<String> {
         let mgr = self.mgr_handle(idx)?;
-        Ok(format!(
+        Ok(mgr.with_chunk_tracker(|chunks| format!("{:?}", chunks.get_have_pieces()))?)
-            "{:?}",
-            mgr.torrent_state()
-                .lock_read("api_dump_haves")
-                .chunks
-                .get_have_pieces(),
-        ))
     }
 }
 

crates/librqbit/src/http_api_error.rs

@@ -19,6 +19,15 @@ impl ApiError {
         }
     }
 
+    #[allow(dead_code)]
+    pub fn not_implemented(msg: &str) -> Self {
+        Self {
+            status: Some(StatusCode::INTERNAL_SERVER_ERROR),
+            kind: ApiErrorKind::Other(anyhow::anyhow!("{}", msg)),
+            plaintext: false,
+        }
+    }
+
     pub const fn dht_disabled() -> Self {
         Self {
             status: Some(StatusCode::NOT_FOUND),

crates/librqbit/src/lib.rs

@@ -6,10 +6,8 @@ pub mod http_api_client;
 mod http_api_error;
 pub mod peer_connection;
 pub mod peer_info_reader;
-pub mod peer_state;
 pub mod session;
 pub mod spawn_utils;
-pub mod torrent_manager;
 pub mod torrent_state;
 pub mod tracker_comms;
 pub mod type_aliases;

crates/librqbit/src/peer_connection.rs

@@ -20,7 +20,7 @@ use crate::spawn_utils::BlockingSpawner;
 pub trait PeerConnectionHandler {
     fn on_connected(&self, _connection_time: Duration) {}
     fn get_have_bytes(&self) -> u64;
-    fn serialize_bitfield_message_to_buf(&self, buf: &mut Vec<u8>) -> Option<usize>;
+    fn serialize_bitfield_message_to_buf(&self, buf: &mut Vec<u8>) -> anyhow::Result<usize>;
     fn on_handshake(&self, handshake: Handshake) -> anyhow::Result<()>;
     fn on_extended_handshake(
         &self,
@@ -204,15 +204,13 @@ impl<H: PeerConnectionHandler> PeerConnection<H> {
                 .unwrap_or_else(|| Duration::from_secs(120));
 
             if self.handler.get_have_bytes() > 0 {
-                if let Some(len) = self
+                let len = self
                     .handler
-                    .serialize_bitfield_message_to_buf(&mut write_buf)
+                    .serialize_bitfield_message_to_buf(&mut write_buf)?;
-                {
+                with_timeout(rwtimeout, write_half.write_all(&write_buf[..len]))
-                    with_timeout(rwtimeout, write_half.write_all(&write_buf[..len]))
+                    .await
-                        .await
+                    .context("error writing bitfield to peer")?;
-                        .context("error writing bitfield to peer")?;
+                debug!("sent bitfield");
-                    debug!("sent bitfield");
-                }
             }
 
             loop {

crates/librqbit/src/peer_info_reader/mod.rs

@@ -141,8 +141,8 @@ impl PeerConnectionHandler for Handler {
         0
     }
 
-    fn serialize_bitfield_message_to_buf(&self, _buf: &mut Vec<u8>) -> Option<usize> {
+    fn serialize_bitfield_message_to_buf(&self, _buf: &mut Vec<u8>) -> anyhow::Result<usize> {
-        None
+        Ok(0)
     }
 
     fn on_handshake(&self, handshake: Handshake) -> anyhow::Result<()> {

crates/librqbit/src/peer_state.rs

@@ -1,349 +0,0 @@
-use std::collections::HashSet;
-use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
-use std::sync::Arc;
-use std::time::Duration;
-
-use anyhow::Context;
-use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
-use librqbit_core::id20::Id20;
-use librqbit_core::lengths::{ChunkInfo, ValidPieceIndex};
-use serde::Serialize;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
-
-use crate::peer_connection::WriterRequest;
-use crate::type_aliases::BF;
-
-#[derive(Debug, Hash, PartialEq, Eq)]
-pub struct InflightRequest {
-    pub piece: ValidPieceIndex,
-    pub chunk: u32,
-}
-
-impl From<&ChunkInfo> for InflightRequest {
-    fn from(c: &ChunkInfo) -> Self {
-        Self {
-            piece: c.piece_index,
-            chunk: c.chunk_index,
-        }
-    }
-}
-
-// TODO: Arc can be removed probably, as UnboundedSender should be clone + it can be downgraded to weak.
-pub type PeerRx = UnboundedReceiver<WriterRequest>;
-pub type PeerTx = UnboundedSender<WriterRequest>;
-
-pub trait SendMany {
-    fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()>;
-}
-
-impl SendMany for PeerTx {
-    fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()> {
-        requests
-            .into_iter()
-            .try_for_each(|r| self.send(r))
-            .context("peer dropped")
-    }
-}
-
-#[derive(Default, Debug)]
-pub struct PeerCounters {
-    pub fetched_bytes: AtomicU64,
-    pub total_time_connecting_ms: AtomicU64,
-    pub connection_attempts: AtomicU32,
-    pub connections: AtomicU32,
-    pub errors: AtomicU32,
-    pub fetched_chunks: AtomicU32,
-    pub downloaded_and_checked_pieces: AtomicU32,
-    pub downloaded_and_checked_bytes: AtomicU64,
-}
-
-#[derive(Debug)]
-pub struct PeerStats {
-    pub counters: Arc<PeerCounters>,
-    pub backoff: ExponentialBackoff,
-}
-
-impl Default for PeerStats {
-    fn default() -> Self {
-        Self {
-            counters: Arc::new(Default::default()),
-            backoff: ExponentialBackoffBuilder::new()
-                .with_initial_interval(Duration::from_secs(10))
-                .with_multiplier(6.)
-                .with_max_interval(Duration::from_secs(3600))
-                .with_max_elapsed_time(Some(Duration::from_secs(86400)))
-                .build(),
-        }
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct Peer {
-    pub state: PeerStateNoMut,
-    pub stats: PeerStats,
-}
-
-#[derive(Debug, Default, Serialize)]
-pub struct AggregatePeerStatsAtomic {
-    pub queued: AtomicU32,
-    pub connecting: AtomicU32,
-    pub live: AtomicU32,
-    pub seen: AtomicU32,
-    pub dead: AtomicU32,
-    pub not_needed: AtomicU32,
-}
-
-pub fn atomic_inc(c: &AtomicU32) -> u32 {
-    c.fetch_add(1, Ordering::Relaxed)
-}
-
-pub fn atomic_dec(c: &AtomicU32) -> u32 {
-    c.fetch_sub(1, Ordering::Relaxed)
-}
-
-impl AggregatePeerStatsAtomic {
-    pub fn counter(&self, state: &PeerState) -> &AtomicU32 {
-        match state {
-            PeerState::Connecting(_) => &self.connecting,
-            PeerState::Live(_) => &self.live,
-            PeerState::Queued => &self.queued,
-            PeerState::Dead => &self.dead,
-            PeerState::NotNeeded => &self.not_needed,
-        }
-    }
-
-    pub fn inc(&self, state: &PeerState) {
-        atomic_inc(self.counter(state));
-    }
-
-    pub fn dec(&self, state: &PeerState) {
-        atomic_dec(self.counter(state));
-    }
-
-    pub fn incdec(&self, old: &PeerState, new: &PeerState) {
-        self.dec(old);
-        self.inc(new);
-    }
-}
-
-#[derive(Debug, Default)]
-pub enum PeerState {
-    #[default]
-    // Will be tried to be connected as soon as possible.
-    Queued,
-    Connecting(PeerTx),
-    Live(LivePeerState),
-    // There was an error, and it's waiting for exponential backoff.
-    Dead,
-    // We don't need to do anything with the peer any longer.
-    // The peer has the full torrent, and we have the full torrent, so no need
-    // to keep talking to it.
-    NotNeeded,
-}
-
-impl std::fmt::Display for PeerState {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.name())
-    }
-}
-
-impl PeerState {
-    pub fn name(&self) -> &'static str {
-        match self {
-            PeerState::Queued => "queued",
-            PeerState::Connecting(_) => "connecting",
-            PeerState::Live(_) => "live",
-            PeerState::Dead => "dead",
-            PeerState::NotNeeded => "not needed",
-        }
-    }
-
-    pub fn take_live_no_counters(self) -> Option<LivePeerState> {
-        match self {
-            PeerState::Live(l) => Some(l),
-            _ => None,
-        }
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct PeerStateNoMut(PeerState);
-
-impl PeerStateNoMut {
-    pub fn get(&self) -> &PeerState {
-        &self.0
-    }
-
-    pub fn take(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
-        self.set(Default::default(), counters)
-    }
-
-    pub fn set(&mut self, new: PeerState, counters: &AggregatePeerStatsAtomic) -> PeerState {
-        counters.incdec(&self.0, &new);
-        std::mem::replace(&mut self.0, new)
-    }
-
-    pub fn get_live(&self) -> Option<&LivePeerState> {
-        match &self.0 {
-            PeerState::Live(l) => Some(l),
-            _ => None,
-        }
-    }
-
-    pub fn get_live_mut(&mut self) -> Option<&mut LivePeerState> {
-        match &mut self.0 {
-            PeerState::Live(l) => Some(l),
-            _ => None,
-        }
-    }
-
-    pub fn queued_to_connecting(
-        &mut self,
-        counters: &AggregatePeerStatsAtomic,
-    ) -> Option<(PeerRx, PeerTx)> {
-        if let PeerState::Queued = &self.0 {
-            let (tx, rx) = unbounded_channel();
-            let tx_2 = tx.clone();
-            self.set(PeerState::Connecting(tx), counters);
-            Some((rx, tx_2))
-        } else {
-            None
-        }
-    }
-    pub fn connecting_to_live(
-        &mut self,
-        peer_id: Id20,
-        counters: &AggregatePeerStatsAtomic,
-    ) -> Option<&mut LivePeerState> {
-        if let PeerState::Connecting(_) = &self.0 {
-            let tx = match self.take(counters) {
-                PeerState::Connecting(tx) => tx,
-                _ => unreachable!(),
-            };
-            self.set(PeerState::Live(LivePeerState::new(peer_id, tx)), counters);
-            self.get_live_mut()
-        } else {
-            None
-        }
-    }
-
-    pub fn to_dead(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
-        self.set(PeerState::Dead, counters)
-    }
-
-    pub fn to_not_needed(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
-        self.set(PeerState::NotNeeded, counters)
-    }
-}
-
-#[derive(Debug)]
-pub struct LivePeerState {
-    pub peer_id: Id20,
-    pub peer_interested: bool,
-
-    // This is used to track the pieces the peer has.
-    pub bitfield: BF,
-
-    // When the peer sends us data this is used to track if we asked for it.
-    pub inflight_requests: HashSet<InflightRequest>,
-
-    // The main channel to send requests to peer.
-    pub tx: PeerTx,
-}
-
-impl LivePeerState {
-    pub fn new(peer_id: Id20, tx: PeerTx) -> Self {
-        LivePeerState {
-            peer_id,
-            peer_interested: false,
-            bitfield: BF::new(),
-            inflight_requests: Default::default(),
-            tx,
-        }
-    }
-
-    pub fn has_full_torrent(&self, total_pieces: usize) -> bool {
-        self.bitfield
-            .get(0..total_pieces)
-            .map_or(false, |s| s.all())
-    }
-}
-
-mod peer_stats_snapshot {
-    use std::{collections::HashMap, sync::atomic::Ordering};
-
-    use serde::{Deserialize, Serialize};
-
-    use crate::peer_state::PeerState;
-
-    #[derive(Serialize, Deserialize)]
-    pub struct PeerCounters {
-        pub fetched_bytes: u64,
-        pub total_time_connecting_ms: u64,
-        pub connection_attempts: u32,
-        pub connections: u32,
-        pub errors: u32,
-        pub fetched_chunks: u32,
-        pub downloaded_and_checked_pieces: u32,
-    }
-
-    #[derive(Serialize, Deserialize)]
-    pub struct PeerStats {
-        pub counters: PeerCounters,
-        pub state: &'static str,
-    }
-
-    impl From<&crate::peer_state::PeerCounters> for PeerCounters {
-        fn from(counters: &crate::peer_state::PeerCounters) -> Self {
-            Self {
-                fetched_bytes: counters.fetched_bytes.load(Ordering::Relaxed),
-                total_time_connecting_ms: counters.total_time_connecting_ms.load(Ordering::Relaxed),
-                connection_attempts: counters.connection_attempts.load(Ordering::Relaxed),
-                connections: counters.connections.load(Ordering::Relaxed),
-                errors: counters.errors.load(Ordering::Relaxed),
-                fetched_chunks: counters.fetched_chunks.load(Ordering::Relaxed),
-                downloaded_and_checked_pieces: counters
-                    .downloaded_and_checked_pieces
-                    .load(Ordering::Relaxed),
-            }
-        }
-    }
-
-    impl From<&crate::peer_state::Peer> for PeerStats {
-        fn from(peer: &crate::peer_state::Peer) -> Self {
-            Self {
-                counters: peer.stats.counters.as_ref().into(),
-                state: peer.state.get().name(),
-            }
-        }
-    }
-
-    #[derive(Serialize)]
-    pub struct PeerStatsSnapshot {
-        pub peers: HashMap<String, PeerStats>,
-    }
-
-    #[derive(Clone, Copy, Default, Deserialize)]
-    pub enum PeerStatsFilterState {
-        All,
-        #[default]
-        Live,
-    }
-
-    impl PeerStatsFilterState {
-        pub fn matches(&self, s: &PeerState) -> bool {
-            match (self, s) {
-                (Self::All, _) => true,
-                (Self::Live, PeerState::Live(_)) => true,
-                _ => false,
-            }
-        }
-    }
-
-    #[derive(Default, Deserialize)]
-    pub struct PeerStatsFilter {
-        pub state: PeerStatsFilterState,
-    }
-}
-
-pub use peer_stats_snapshot::{PeerStatsFilter, PeerStatsSnapshot};

crates/librqbit/src/session.rs

@@ -1,4 +1,13 @@
-use std::{borrow::Cow, io::Read, net::SocketAddr, path::PathBuf, time::Duration};
+use std::{
+    borrow::Cow,
+    collections::{HashMap, HashSet},
+    io::{BufReader, BufWriter, Read},
+    net::SocketAddr,
+    path::PathBuf,
+    str::FromStr,
+    sync::Arc,
+    time::Duration,
+};
 
 use anyhow::{bail, Context};
 use buffers::ByteString;
@@ -10,64 +19,78 @@ use librqbit_core::{
 };
 use parking_lot::RwLock;
 use reqwest::Url;
+use serde::{Deserialize, Serialize};
 use tokio_stream::StreamExt;
-use tracing::{debug, info, span, warn, Level};
+use tracing::{debug, error, error_span, info, warn};
 
 use crate::{
     dht_utils::{read_metainfo_from_peer_receiver, ReadMetainfoResult},
     peer_connection::PeerConnectionOptions,
     spawn_utils::{spawn, BlockingSpawner},
-    torrent_manager::{TorrentManagerBuilder, TorrentManagerHandle},
+    torrent_state::{ManagedTorrentBuilder, ManagedTorrentHandle, ManagedTorrentState},
 };
 
 pub const SUPPORTED_SCHEMES: [&str; 3] = ["http:", "https:", "magnet:"];
 
-#[derive(Clone)]
+pub type TorrentId = usize;
-pub enum ManagedTorrentState {
-    Initializing,
-    Running(TorrentManagerHandle),
-}
-
-#[derive(Clone)]
-pub struct ManagedTorrent {
-    pub info_hash: Id20,
-    pub output_folder: PathBuf,
-    pub state: ManagedTorrentState,
-}
-
-impl PartialEq for ManagedTorrent {
-    fn eq(&self, other: &Self) -> bool {
-        self.info_hash == other.info_hash && self.output_folder == other.output_folder
-    }
-}
 
 #[derive(Default)]
-pub struct SessionLocked {
+pub struct SessionDatabase {
-    torrents: Vec<ManagedTorrent>,
+    next_id: usize,
+    torrents: HashMap<usize, ManagedTorrentHandle>,
 }
 
-enum SessionLockedAddTorrentResult {
+impl SessionDatabase {
-    AlreadyManaged(ManagedTorrent),
+    fn add_torrent(&mut self, torrent: ManagedTorrentHandle) -> TorrentId {
-    Added(usize),
+        let idx = self.next_id;
-}
+        self.torrents.insert(idx, torrent);
-
+        self.next_id += 1;
-impl SessionLocked {
+        idx
-    fn add_torrent(&mut self, torrent: ManagedTorrent) -> SessionLockedAddTorrentResult {
-        if let Some(handle) = self.torrents.iter().find(|t| **t == torrent) {
-            return SessionLockedAddTorrentResult::AlreadyManaged(handle.clone());
-        }
-        let idx = self.torrents.len();
-        self.torrents.push(torrent);
-        SessionLockedAddTorrentResult::Added(idx)
     }
+
+    fn serialize(&self) -> SerializedSessionDatabase {
+        SerializedSessionDatabase {
+            torrents: self
+                .torrents
+                .values()
+                .map(|torrent| SerializedTorrent {
+                    trackers: torrent
+                        .info()
+                        .trackers
+                        .iter()
+                        .map(|u| u.to_string())
+                        .collect(),
+                    info_hash: torrent.info_hash().as_string(),
+                    only_files: torrent.only_files.clone(),
+                    is_paused: torrent.with_state(|s| matches!(s, ManagedTorrentState::Paused(_))),
+                    output_folder: torrent.info().out_dir.clone(),
+                })
+                .collect(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+struct SerializedTorrent {
+    info_hash: String,
+    trackers: HashSet<String>,
+    output_folder: PathBuf,
+    only_files: Option<Vec<usize>>,
+    is_paused: bool,
+}
+
+#[derive(Serialize, Deserialize)]
+struct SerializedSessionDatabase {
+    torrents: Vec<SerializedTorrent>,
 }
 
 pub struct Session {
     peer_id: Id20,
     dht: Option<Dht>,
+    persistence_filename: PathBuf,
     peer_opts: PeerConnectionOptions,
     spawner: BlockingSpawner,
-    locked: RwLock<SessionLocked>,
+    db: RwLock<SessionDatabase>,
     output_folder: PathBuf,
 }
 
@@ -107,6 +130,7 @@ fn compute_only_files<ByteBuf: AsRef<[u8]>>(
 
 #[derive(Default, Clone)]
 pub struct AddTorrentOptions {
+    pub paused: bool,
     pub only_files_regex: Option<String>,
     pub only_files: Option<Vec<usize>>,
     pub overwrite: bool,
@@ -124,9 +148,9 @@ pub struct ListOnlyResponse {
 }
 
 pub enum AddTorrentResponse {
-    AlreadyManaged(ManagedTorrent),
+    AlreadyManaged(TorrentId, ManagedTorrentHandle),
     ListOnly(ListOnlyResponse),
-    Added(TorrentManagerHandle),
+    Added(TorrentId, ManagedTorrentHandle),
 }
 
 pub fn read_local_file_including_stdin(filename: &str) -> anyhow::Result<Vec<u8>> {
@@ -185,20 +209,26 @@ impl<'a> AddTorrent<'a> {
 pub struct SessionOptions {
     pub disable_dht: bool,
     pub disable_dht_persistence: bool,
+    pub persistence: bool,
+    // Will default to output_folder/.rqbit-session.json
+    pub persistence_filename: Option<PathBuf>,
     pub dht_config: Option<PersistentDhtConfig>,
     pub peer_id: Option<Id20>,
     pub peer_opts: Option<PeerConnectionOptions>,
 }
 
 impl Session {
-    pub async fn new(output_folder: PathBuf, spawner: BlockingSpawner) -> anyhow::Result<Self> {
+    pub async fn new(
+        output_folder: PathBuf,
+        spawner: BlockingSpawner,
+    ) -> anyhow::Result<Arc<Self>> {
         Self::new_with_opts(output_folder, spawner, SessionOptions::default()).await
     }
     pub async fn new_with_opts(
         output_folder: PathBuf,
         spawner: BlockingSpawner,
         opts: SessionOptions,
-    ) -> anyhow::Result<Self> {
+    ) -> anyhow::Result<Arc<Self>> {
         let peer_id = opts.peer_id.unwrap_or_else(generate_peer_id);
         let dht = if opts.disable_dht {
             None
@@ -212,31 +242,147 @@ impl Session {
             Some(dht)
         };
         let peer_opts = opts.peer_opts.unwrap_or_default();
-
+        let persistence_filename = opts
-        Ok(Self {
+            .persistence_filename
+            .unwrap_or_else(|| output_folder.join(".rqbit-session.json"));
+        let session = Arc::new(Self {
+            persistence_filename,
             peer_id,
             dht,
             peer_opts,
             spawner,
             output_folder,
-            locked: RwLock::new(SessionLocked::default()),
+            db: RwLock::new(Default::default()),
-        })
+        });
+
+        if opts.persistence {
+            if let Some(parent) = session.persistence_filename.parent() {
+                std::fs::create_dir_all(parent).with_context(|| {
+                    format!("couldn't create directory {:?} for session storage", parent)
+                })?;
+            }
+            let session = session.clone();
+            spawn(
+                "session persistene",
+                error_span!("session_persistence"),
+                async move {
+                    // Populate initial from the state filename
+                    if let Err(e) = session.populate_from_stored().await {
+                        error!("could not populate session from stored file: {:?}", e);
+                    }
+
+                    let session = Arc::downgrade(&session);
+
+                    loop {
+                        tokio::time::sleep(Duration::from_secs(10)).await;
+                        let session = match session.upgrade() {
+                            Some(s) => s,
+                            None => break,
+                        };
+                        if let Err(e) = session.dump_to_disk() {
+                            error!("error dumping session to disk: {:?}", e);
+                        }
+                    }
+
+                    Ok(())
+                },
+            );
+        }
+
+        Ok(session)
     }
-    pub fn get_dht(&self) -> Option<Dht> {
+    pub fn get_dht(&self) -> Option<&Dht> {
-        self.dht.clone()
+        self.dht.as_ref()
     }
-    pub fn with_torrents<F>(&self, callback: F)
+
-    where
+    async fn populate_from_stored(self: &Arc<Self>) -> anyhow::Result<()> {
-        F: Fn(&[ManagedTorrent]),
+        let mut rdr = match std::fs::File::open(&self.persistence_filename) {
-    {
+            Ok(f) => BufReader::new(f),
-        callback(&self.locked.read().torrents)
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+            Err(e) => {
+                return Err(e).context(format!(
+                    "error opening session file {:?}",
+                    self.persistence_filename
+                ))
+            }
+        };
+        let db: SerializedSessionDatabase =
+            serde_json::from_reader(&mut rdr).context("error deserializing session database")?;
+        let mut futures = Vec::new();
+        for storrent in db.torrents.into_iter() {
+            let magnet = Magnet {
+                info_hash: Id20::from_str(&storrent.info_hash)
+                    .context("error deserializing info_hash")?,
+                trackers: storrent.trackers.into_iter().collect(),
+            };
+            futures.push({
+                let session = self.clone();
+                async move {
+                    session
+                        .add_torrent(
+                            AddTorrent::Url(Cow::Owned(magnet.to_string())),
+                            Some(AddTorrentOptions {
+                                paused: storrent.is_paused,
+                                output_folder: Some(
+                                    storrent
+                                        .output_folder
+                                        .to_str()
+                                        .context("broken path")?
+                                        .to_owned(),
+                                ),
+                                only_files: storrent.only_files,
+                                overwrite: true,
+                                ..Default::default()
+                            }),
+                        )
+                        .await
+                        .map_err(|e| {
+                            error!("error adding torrent from stored session: {:?}", e);
+                            e
+                        })
+                }
+            });
+        }
+        futures::future::join_all(futures).await;
+        Ok(())
     }
+
+    fn dump_to_disk(&self) -> anyhow::Result<()> {
+        let tmp_filename = format!("{}.tmp", self.persistence_filename.to_str().unwrap());
+        let mut tmp = BufWriter::new(
+            std::fs::OpenOptions::new()
+                .create(true)
+                .truncate(true)
+                .write(true)
+                .open(&tmp_filename)
+                .with_context(|| format!("error opening {:?}", tmp_filename))?,
+        );
+        let serialized = self.db.read().serialize();
+        serde_json::to_writer(&mut tmp, &serialized).context("error serializing")?;
+        drop(tmp);
+
+        std::fs::rename(&tmp_filename, &self.persistence_filename)
+            .context("error renaming persistence file")?;
+        debug!("wrote persistence to {:?}", &self.persistence_filename);
+        Ok(())
+    }
+
+    pub fn with_torrents<R>(
+        &self,
+        callback: impl Fn(&mut dyn Iterator<Item = (TorrentId, &ManagedTorrentHandle)>) -> R,
+    ) -> R {
+        callback(&mut self.db.read().torrents.iter().map(|(id, t)| (*id, t)))
+    }
+
     pub async fn add_torrent(
         &self,
         add: impl Into<AddTorrent<'_>>,
         opts: Option<AddTorrentOptions>,
     ) -> anyhow::Result<AddTorrentResponse> {
         // Magnet links are different in that we first need to discover the metadata.
+        let span = error_span!("add_torrent");
+        let _ = span.enter();
+
         let opts = opts.unwrap_or_default();
 
         let (info_hash, info, dht_rx, trackers, initial_peers) = match add.into() {
@@ -250,8 +396,7 @@ impl Session {
                     .dht
                     .as_ref()
                     .context("magnet links without DHT are not supported")?
-                    .get_peers(info_hash)
+                    .get_peers(info_hash)?;
-                    .await?;
 
                 let trackers = trackers
                     .into_iter()
@@ -264,6 +409,7 @@ impl Session {
                     })
                     .collect();
 
+                debug!("querying DHT for {:?}", info_hash);
                 let (info, dht_rx, initial_peers) = match read_metainfo_from_peer_receiver(
                     self.peer_id,
                     info_hash,
@@ -277,6 +423,7 @@ impl Session {
                         anyhow::bail!("DHT died, no way to discover torrent metainfo")
                     }
                 };
+                debug!("received result from DHT: {:?}", info);
                 (info_hash, info, Some(dht_rx), trackers, initial_peers)
             }
             other => {
@@ -300,7 +447,7 @@ impl Session {
                 let dht_rx = match self.dht.as_ref() {
                     Some(dht) => {
                         debug!("reading peers for {:?} from DHT", torrent.info_hash);
-                        Some(dht.get_peers(torrent.info_hash).await?)
+                        Some(dht.get_peers(torrent.info_hash)?)
                     }
                     None => None,
                 };
@@ -404,24 +551,13 @@ impl Session {
             .unwrap_or_else(|| self.output_folder.clone())
             .join(sub_folder);
 
-        let managed_torrent = ManagedTorrent {
+        let mut builder = ManagedTorrentBuilder::new(info, info_hash, output_folder.clone());
-            info_hash,
-            output_folder: output_folder.clone(),
-            state: ManagedTorrentState::Initializing,
-        };
-
-        match self.locked.write().add_torrent(managed_torrent) {
-            SessionLockedAddTorrentResult::AlreadyManaged(managed) => {
-                return Ok(AddTorrentResponse::AlreadyManaged(managed))
-            }
-            SessionLockedAddTorrentResult::Added(_) => {}
-        }
-
-        let mut builder = TorrentManagerBuilder::new(info, info_hash, output_folder.clone());
         builder
             .overwrite(opts.overwrite)
             .spawner(self.spawner)
-            .peer_id(self.peer_id);
+            .peer_id(self.peer_id)
+            .trackers(trackers);
+
         if let Some(only_files) = only_files {
             builder.only_files(only_files);
         }
@@ -437,61 +573,79 @@ impl Session {
             builder.peer_read_write_timeout(t);
         }
 
-        let handle = match builder
+        let (managed_torrent, id) = {
-            .start_manager()
+            let mut g = self.db.write();
-            .context("error starting torrent manager")
+            if let Some((id, handle)) = g.torrents.iter().find(|(_, t)| t.info_hash() == info_hash)
-        {
+            {
-            Ok(handle) => {
+                return Ok(AddTorrentResponse::AlreadyManaged(*id, handle.clone()));
-                let mut g = self.locked.write();
-                let m = g
-                    .torrents
-                    .iter_mut()
-                    .find(|t| t.info_hash == info_hash && t.output_folder == output_folder)
-                    .unwrap();
-                m.state = ManagedTorrentState::Running(handle.clone());
-                handle
-            }
-            Err(error) => {
-                let mut g = self.locked.write();
-                let idx = g
-                    .torrents
-                    .iter()
-                    .position(|t| t.info_hash == info_hash && t.output_folder == output_folder)
-                    .unwrap();
-                g.torrents.remove(idx);
-                return Err(error);
             }
+            let next_id = g.torrents.len();
+            let managed_torrent =
+                builder.build(error_span!(parent: None, "torrent", id = next_id))?;
+            let id = g.add_torrent(managed_torrent.clone());
+            (managed_torrent, id)
         };
+
         {
-            let mut g = self.locked.write();
+            let span = managed_torrent.info.span.clone();
-            let m = g
+            let _ = span.enter();
-                .torrents
+            managed_torrent
-                .iter_mut()
+                .start(initial_peers, dht_peer_rx, opts.paused)
-                .find(|t| t.info_hash == info_hash && t.output_folder == output_folder)
+                .context("error starting torrent")?;
-                .unwrap();
-            m.state = ManagedTorrentState::Running(handle.clone());
         }
 
-        for url in trackers {
+        Ok(AddTorrentResponse::Added(id, managed_torrent))
-            handle.add_tracker(url);
+    }
-        }
-        for peer in initial_peers {
-            handle.add_peer(peer);
-        }
 
-        if let Some(mut dht_peer_rx) = dht_peer_rx {
+    pub fn get(&self, id: TorrentId) -> Option<ManagedTorrentHandle> {
-            spawn(span!(Level::INFO, "dht_peer_adder"), {
+        self.db.read().torrents.get(&id).cloned()
-                let handle = handle.clone();
+    }
-                async move {
+
-                    while let Some(peer) = dht_peer_rx.next().await {
+    pub fn delete(&self, id: TorrentId, delete_files: bool) -> anyhow::Result<()> {
-                        handle.add_peer(peer);
+        let removed = self
+            .db
+            .write()
+            .torrents
+            .remove(&id)
+            .with_context(|| format!("torrent with id {} did not exist", id))?;
+
+        let paused = removed
+            .with_state_mut(|s| {
+                let paused = match s.take() {
+                    ManagedTorrentState::Paused(p) => p,
+                    ManagedTorrentState::Live(l) => l.pause()?,
+                    _ => return Ok(None),
+                };
+                Ok::<_, anyhow::Error>(Some(paused))
+            })
+            .context("error pausing torrent");
+
+        match (paused, delete_files) {
+            (Err(e), true) => Err(e).context("torrent deleted, but could not delete files"),
+            (Err(e), false) => {
+                warn!("could not delete torrent files: {:?}", e);
+                Ok(())
+            }
+            (Ok(Some(paused)), true) => {
+                drop(paused.files);
+                for file in paused.filenames {
+                    if let Err(e) = std::fs::remove_file(&file) {
+                        warn!("could not delete file {:?}: {:?}", file, e);
                     }
-                    warn!("dht was closed");
-                    Ok(())
                 }
-            });
+                Ok(())
+            }
+            _ => Ok(()),
         }
+    }
 
-        Ok(AddTorrentResponse::Added(handle))
+    pub fn unpause(&self, handle: &ManagedTorrentHandle) -> anyhow::Result<()> {
+        let peer_rx = self
+            .dht
+            .as_ref()
+            .map(|dht| dht.get_peers(handle.info_hash()))
+            .transpose()?;
+        handle.start(Default::default(), peer_rx, false)?;
+        Ok(())
     }
 }

crates/librqbit/src/spawn_utils.rs

@@ -1,22 +1,9 @@
-use tracing::{debug, error, trace, Instrument};
-
 pub fn spawn(
+    _name: &str,
     span: tracing::Span,
     fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
-) {
+) -> tokio::task::JoinHandle<()> {
-    let fut = async move {
+    librqbit_core::spawn_utils::spawn(span, fut)
-        trace!("started");
-        match fut.await {
-            Ok(_) => {
-                debug!("finished");
-            }
-            Err(e) => {
-                error!("{:#}", e)
-            }
-        }
-    }
-    .instrument(span.or_current());
-    tokio::spawn(fut);
 }
 
 #[derive(Clone, Copy, Debug)]

crates/librqbit/src/torrent_manager.rs

@@ -1,380 +0,0 @@
-use std::{
-    collections::HashSet,
-    fs::{File, OpenOptions},
-    net::SocketAddr,
-    path::{Path, PathBuf},
-    sync::Arc,
-    time::{Duration, Instant},
-};
-
-use anyhow::Context;
-use bencode::from_bytes;
-use buffers::ByteString;
-use librqbit_core::{
-    id20::Id20, lengths::Lengths, peer_id::generate_peer_id, speed_estimator::SpeedEstimator,
-    torrent_metainfo::TorrentMetaV1Info,
-};
-use parking_lot::Mutex;
-use reqwest::Url;
-use sha1w::Sha1;
-use size_format::SizeFormatterBinary as SF;
-use tracing::{debug, info, span, warn, Level};
-
-use crate::{
-    chunk_tracker::ChunkTracker,
-    file_ops::FileOps,
-    spawn_utils::{spawn, BlockingSpawner},
-    torrent_state::{TorrentState, TorrentStateOptions},
-    tracker_comms::{TrackerError, TrackerRequest, TrackerRequestEvent, TrackerResponse},
-};
-
-#[derive(Default)]
-struct TorrentManagerOptions {
-    force_tracker_interval: Option<Duration>,
-    peer_connect_timeout: Option<Duration>,
-    peer_read_write_timeout: Option<Duration>,
-    only_files: Option<Vec<usize>>,
-    peer_id: Option<Id20>,
-    overwrite: bool,
-}
-
-pub struct TorrentManagerBuilder {
-    info: TorrentMetaV1Info<ByteString>,
-    info_hash: Id20,
-    output_folder: PathBuf,
-    options: TorrentManagerOptions,
-    spawner: Option<BlockingSpawner>,
-}
-
-impl TorrentManagerBuilder {
-    pub fn new<P: AsRef<Path>>(
-        info: TorrentMetaV1Info<ByteString>,
-        info_hash: Id20,
-        output_folder: P,
-    ) -> Self {
-        Self {
-            info,
-            info_hash,
-            output_folder: output_folder.as_ref().into(),
-            spawner: None,
-            options: TorrentManagerOptions::default(),
-        }
-    }
-
-    pub fn only_files(&mut self, only_files: Vec<usize>) -> &mut Self {
-        self.options.only_files = Some(only_files);
-        self
-    }
-
-    pub fn overwrite(&mut self, overwrite: bool) -> &mut Self {
-        self.options.overwrite = overwrite;
-        self
-    }
-
-    pub fn force_tracker_interval(&mut self, force_tracker_interval: Duration) -> &mut Self {
-        self.options.force_tracker_interval = Some(force_tracker_interval);
-        self
-    }
-
-    pub fn spawner(&mut self, spawner: BlockingSpawner) -> &mut Self {
-        self.spawner = Some(spawner);
-        self
-    }
-
-    pub fn peer_id(&mut self, peer_id: Id20) -> &mut Self {
-        self.options.peer_id = Some(peer_id);
-        self
-    }
-
-    pub fn peer_connect_timeout(&mut self, timeout: Duration) -> &mut Self {
-        self.options.peer_connect_timeout = Some(timeout);
-        self
-    }
-
-    pub fn peer_read_write_timeout(&mut self, timeout: Duration) -> &mut Self {
-        self.options.peer_read_write_timeout = Some(timeout);
-        self
-    }
-
-    pub fn start_manager(self) -> anyhow::Result<TorrentManagerHandle> {
-        TorrentManager::start(
-            self.info,
-            self.info_hash,
-            self.output_folder,
-            self.spawner.unwrap_or_else(|| BlockingSpawner::new(true)),
-            Some(self.options),
-        )
-    }
-}
-
-#[derive(Clone)]
-pub struct TorrentManagerHandle {
-    manager: Arc<TorrentManager>,
-}
-
-impl TorrentManagerHandle {
-    pub fn add_tracker(&self, url: Url) -> bool {
-        let mgr = self.manager.clone();
-        if mgr.trackers.lock().insert(url.clone()) {
-            spawn(
-                span!(Level::ERROR, "tracker_monitor", url = url.to_string()),
-                async move { mgr.single_tracker_monitor(url).await },
-            );
-            true
-        } else {
-            false
-        }
-    }
-    pub fn only_files(&self) -> Option<&[usize]> {
-        self.manager.options.only_files.as_deref()
-    }
-    pub fn add_peer(&self, addr: SocketAddr) -> bool {
-        self.manager.state.add_peer_if_not_seen(addr)
-    }
-    pub fn torrent_state(&self) -> &TorrentState {
-        &self.manager.state
-    }
-    pub fn speed_estimator(&self) -> &Arc<SpeedEstimator> {
-        &self.manager.speed_estimator
-    }
-    pub async fn cancel(&self) -> anyhow::Result<()> {
-        todo!()
-    }
-    pub async fn wait_until_completed(&self) -> anyhow::Result<()> {
-        self.manager.state.wait_until_completed().await;
-        Ok(())
-    }
-}
-
-struct TorrentManager {
-    state: Arc<TorrentState>,
-    #[allow(dead_code)]
-    speed_estimator: Arc<SpeedEstimator>,
-    trackers: Mutex<HashSet<Url>>,
-    options: TorrentManagerOptions,
-}
-
-fn make_lengths<ByteBuf: AsRef<[u8]>>(
-    torrent: &TorrentMetaV1Info<ByteBuf>,
-) -> anyhow::Result<Lengths> {
-    let total_length = torrent.iter_file_lengths()?.sum();
-    Lengths::new(total_length, torrent.piece_length, None)
-}
-
-fn ensure_file_length(file: &File, length: u64) -> anyhow::Result<()> {
-    Ok(file.set_len(length)?)
-}
-
-impl TorrentManager {
-    fn start<P: AsRef<Path>>(
-        info: TorrentMetaV1Info<ByteString>,
-        info_hash: Id20,
-        out: P,
-        spawner: BlockingSpawner,
-        options: Option<TorrentManagerOptions>,
-    ) -> anyhow::Result<TorrentManagerHandle> {
-        let options = options.unwrap_or_default();
-        let (files, filenames) = {
-            let mut files =
-                Vec::<Arc<Mutex<File>>>::with_capacity(info.iter_file_lengths()?.count());
-            let mut filenames = Vec::new();
-            for (path_bits, _) in info.iter_filenames_and_lengths()? {
-                let mut full_path = out.as_ref().to_owned();
-                let relative_path = path_bits
-                    .to_pathbuf()
-                    .context("error converting file to path")?;
-                full_path.push(relative_path);
-
-                std::fs::create_dir_all(full_path.parent().unwrap())?;
-                let file = if options.overwrite {
-                    OpenOptions::new()
-                        .create(true)
-                        .read(true)
-                        .write(true)
-                        .open(&full_path)?
-                } else {
-                    // TODO: create_new does not seem to work with read(true), so calling this twice.
-                    OpenOptions::new()
-                        .create_new(true)
-                        .write(true)
-                        .open(&full_path)
-                        .with_context(|| format!("error creating {:?}", &full_path))?;
-                    OpenOptions::new().read(true).write(true).open(&full_path)?
-                };
-                filenames.push(full_path);
-                files.push(Arc::new(Mutex::new(file)))
-            }
-            (files, filenames)
-        };
-
-        let peer_id = options.peer_id.unwrap_or_else(generate_peer_id);
-        let lengths = make_lengths(&info).context("unable to compute Lengths from torrent")?;
-        debug!("computed lengths: {:?}", &lengths);
-
-        info!("Doing initial checksum validation, this might take a while...");
-        let initial_check_results = spawner.spawn_block_in_place(|| {
-            FileOps::<Sha1>::new(&info, &files, &lengths)
-                .initial_check(options.only_files.as_deref())
-        })?;
-
-        info!(
-            "Initial check results: have {}, needed {}",
-            SF::new(initial_check_results.have_bytes),
-            SF::new(initial_check_results.needed_bytes)
-        );
-
-        spawner.spawn_block_in_place(|| {
-            for (idx, (file, (name, length))) in files
-                .iter()
-                .zip(info.iter_filenames_and_lengths().unwrap())
-                .enumerate()
-            {
-                if options
-                    .only_files
-                    .as_ref()
-                    .map(|v| !v.contains(&idx))
-                    .unwrap_or(false)
-                {
-                    continue;
-                }
-                let now = Instant::now();
-                if let Err(err) = ensure_file_length(&file.lock(), length) {
-                    warn!(
-                        "Error setting length for file {:?} to {}: {:#?}",
-                        name, length, err
-                    );
-                } else {
-                    debug!(
-                        "Set length for file {:?} to {} in {:?}",
-                        name,
-                        SF::new(length),
-                        now.elapsed()
-                    );
-                }
-            }
-        });
-
-        let chunk_tracker = ChunkTracker::new(
-            initial_check_results.needed_pieces,
-            initial_check_results.have_pieces,
-            lengths,
-        );
-
-        #[allow(clippy::needless_update)]
-        let state_options = TorrentStateOptions {
-            peer_connect_timeout: options.peer_connect_timeout,
-            peer_read_write_timeout: options.peer_read_write_timeout,
-            ..Default::default()
-        };
-
-        let state = TorrentState::new(
-            info,
-            info_hash,
-            peer_id,
-            files,
-            filenames,
-            chunk_tracker,
-            lengths,
-            initial_check_results.have_bytes,
-            initial_check_results.needed_bytes,
-            spawner,
-            Some(state_options),
-        );
-
-        let estimator = Arc::new(SpeedEstimator::new(5));
-
-        let mgr = Arc::new(Self {
-            state,
-            speed_estimator: estimator.clone(),
-            trackers: Mutex::new(HashSet::new()),
-            options,
-        });
-
-        spawn(span!(Level::ERROR, "speed_estimator_updater"), {
-            let state = mgr.state.clone();
-            async move {
-                loop {
-                    let stats = state.stats_snapshot();
-                    let fetched = stats.fetched_bytes;
-                    let needed = state.initially_needed();
-                    // fetched can be too high in theory, so for safety make sure that it doesn't wrap around u64.
-                    let remaining = needed
-                        .wrapping_sub(fetched)
-                        .min(needed - stats.downloaded_and_checked_bytes);
-                    estimator.add_snapshot(fetched, remaining, Instant::now());
-                    tokio::time::sleep(Duration::from_secs(1)).await;
-                }
-            }
-        });
-
-        Ok(mgr.into_handle())
-    }
-
-    fn into_handle(self: Arc<Self>) -> TorrentManagerHandle {
-        TorrentManagerHandle { manager: self }
-    }
-
-    async fn tracker_one_request(&self, tracker_url: Url) -> anyhow::Result<u64> {
-        let response: reqwest::Response = reqwest::get(tracker_url).await?;
-        if !response.status().is_success() {
-            anyhow::bail!("tracker responded with {:?}", response.status());
-        }
-        let bytes = response.bytes().await?;
-        if let Ok(error) = from_bytes::<TrackerError>(&bytes) {
-            anyhow::bail!(
-                "tracker returned failure. Failure reason: {}",
-                error.failure_reason
-            )
-        };
-        let response = from_bytes::<TrackerResponse>(&bytes)?;
-
-        for peer in response.peers.iter_sockaddrs() {
-            self.state.add_peer_if_not_seen(peer);
-        }
-        Ok(response.interval)
-    }
-
-    async fn single_tracker_monitor(&self, mut tracker_url: Url) -> anyhow::Result<()> {
-        let mut event = Some(TrackerRequestEvent::Started);
-        loop {
-            let request = TrackerRequest {
-                info_hash: self.state.info_hash(),
-                peer_id: self.state.peer_id(),
-                port: 6778,
-                uploaded: self.state.get_uploaded_bytes(),
-                downloaded: self.state.get_downloaded_bytes(),
-                left: self.state.get_left_to_download_bytes(),
-                compact: true,
-                no_peer_id: false,
-                event,
-                ip: None,
-                numwant: None,
-                key: None,
-                trackerid: None,
-            };
-
-            let request_query = request.as_querystring();
-            tracker_url.set_query(Some(&request_query));
-
-            match self.tracker_one_request(tracker_url.clone()).await {
-                Ok(interval) => {
-                    event = None;
-                    let interval = self
-                        .options
-                        .force_tracker_interval
-                        .unwrap_or_else(|| Duration::from_secs(interval));
-                    debug!(
-                        "sleeping for {:?} after calling tracker {}",
-                        interval,
-                        tracker_url.host().unwrap()
-                    );
-                    tokio::time::sleep(interval).await;
-                }
-                Err(e) => {
-                    debug!("error calling the tracker {}: {:#}", tracker_url, e);
-                    tokio::time::sleep(Duration::from_secs(60)).await;
-                }
-            };
-        }
-    }
-}

crates/librqbit/src/torrent_state/initializing.rs

@@ -0,0 +1,140 @@
+use std::{
+    fs::{File, OpenOptions},
+    sync::{atomic::AtomicU64, Arc},
+    time::Instant,
+};
+
+use anyhow::Context;
+
+use parking_lot::Mutex;
+
+use sha1w::Sha1;
+use size_format::SizeFormatterBinary as SF;
+use tracing::{debug, info, warn};
+
+use crate::{chunk_tracker::ChunkTracker, file_ops::FileOps};
+
+use super::{paused::TorrentStatePaused, ManagedTorrentInfo};
+
+fn ensure_file_length(file: &File, length: u64) -> anyhow::Result<()> {
+    Ok(file.set_len(length)?)
+}
+
+pub struct TorrentStateInitializing {
+    pub(crate) meta: Arc<ManagedTorrentInfo>,
+    pub(crate) only_files: Option<Vec<usize>>,
+    pub(crate) checked_bytes: AtomicU64,
+}
+
+impl TorrentStateInitializing {
+    pub fn new(meta: Arc<ManagedTorrentInfo>, only_files: Option<Vec<usize>>) -> Self {
+        Self {
+            meta,
+            only_files,
+            checked_bytes: AtomicU64::new(0),
+        }
+    }
+
+    pub fn get_checked_bytes(&self) -> u64 {
+        self.checked_bytes
+            .load(std::sync::atomic::Ordering::Relaxed)
+    }
+
+    pub async fn check(&self) -> anyhow::Result<TorrentStatePaused> {
+        let (files, filenames) = {
+            let mut files =
+                Vec::<Arc<Mutex<File>>>::with_capacity(self.meta.info.iter_file_lengths()?.count());
+            let mut filenames = Vec::new();
+            for (path_bits, _) in self.meta.info.iter_filenames_and_lengths()? {
+                let mut full_path = self.meta.out_dir.clone();
+                let relative_path = path_bits
+                    .to_pathbuf()
+                    .context("error converting file to path")?;
+                full_path.push(relative_path);
+
+                std::fs::create_dir_all(full_path.parent().unwrap())?;
+                let file = if self.meta.options.overwrite {
+                    OpenOptions::new()
+                        .create(true)
+                        .read(true)
+                        .write(true)
+                        .open(&full_path)
+                        .with_context(|| {
+                            format!("error opening {full_path:?} in read/write mode")
+                        })?
+                } else {
+                    // TODO: create_new does not seem to work with read(true), so calling this twice.
+                    OpenOptions::new()
+                        .create_new(true)
+                        .write(true)
+                        .open(&full_path)
+                        .with_context(|| format!("error creating {:?}", &full_path))?;
+                    OpenOptions::new().read(true).write(true).open(&full_path)?
+                };
+                filenames.push(full_path);
+                files.push(Arc::new(Mutex::new(file)))
+            }
+            (files, filenames)
+        };
+
+        debug!("computed lengths: {:?}", &self.meta.lengths);
+
+        info!("Doing initial checksum validation, this might take a while...");
+        let initial_check_results = self.meta.spawner.spawn_block_in_place(|| {
+            FileOps::<Sha1>::new(&self.meta.info, &files, &self.meta.lengths)
+                .initial_check(self.only_files.as_deref(), &self.checked_bytes)
+        })?;
+
+        info!(
+            "Initial check results: have {}, needed {}",
+            SF::new(initial_check_results.have_bytes),
+            SF::new(initial_check_results.needed_bytes)
+        );
+
+        self.meta.spawner.spawn_block_in_place(|| {
+            for (idx, (file, (name, length))) in files
+                .iter()
+                .zip(self.meta.info.iter_filenames_and_lengths().unwrap())
+                .enumerate()
+            {
+                if self
+                    .only_files
+                    .as_ref()
+                    .map(|v| !v.contains(&idx))
+                    .unwrap_or(false)
+                {
+                    continue;
+                }
+                let now = Instant::now();
+                if let Err(err) = ensure_file_length(&file.lock(), length) {
+                    warn!(
+                        "Error setting length for file {:?} to {}: {:#?}",
+                        name, length, err
+                    );
+                } else {
+                    debug!(
+                        "Set length for file {:?} to {} in {:?}",
+                        name,
+                        SF::new(length),
+                        now.elapsed()
+                    );
+                }
+            }
+        });
+
+        let chunk_tracker = ChunkTracker::new(
+            initial_check_results.needed_pieces,
+            initial_check_results.have_pieces,
+            self.meta.lengths,
+        );
+
+        let paused = TorrentStatePaused {
+            info: self.meta.clone(),
+            files,
+            filenames,
+            chunk_tracker,
+            have_bytes: initial_check_results.have_bytes,
+        };
+        Ok(paused)
+    }
+}

crates/librqbit/src/torrent_state/live/peer/mod.rs

@@ -0,0 +1,187 @@
+pub mod stats;
+
+use std::collections::HashSet;
+
+use anyhow::Context;
+
+use librqbit_core::id20::Id20;
+use librqbit_core::lengths::{ChunkInfo, ValidPieceIndex};
+
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+
+use crate::peer_connection::WriterRequest;
+use crate::type_aliases::BF;
+
+use super::peers::stats::atomic::AggregatePeerStatsAtomic;
+
+#[derive(Debug, Hash, PartialEq, Eq)]
+pub(crate) struct InflightRequest {
+    pub piece: ValidPieceIndex,
+    pub chunk: u32,
+}
+
+impl From<&ChunkInfo> for InflightRequest {
+    fn from(c: &ChunkInfo) -> Self {
+        Self {
+            piece: c.piece_index,
+            chunk: c.chunk_index,
+        }
+    }
+}
+
+// TODO: Arc can be removed probably, as UnboundedSender should be clone + it can be downgraded to weak.
+pub(crate) type PeerRx = UnboundedReceiver<WriterRequest>;
+pub(crate) type PeerTx = UnboundedSender<WriterRequest>;
+
+pub trait SendMany {
+    fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()>;
+}
+
+impl SendMany for PeerTx {
+    fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()> {
+        requests
+            .into_iter()
+            .try_for_each(|r| self.send(r))
+            .context("peer dropped")
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct Peer {
+    pub state: PeerStateNoMut,
+    pub stats: stats::atomic::PeerStats,
+}
+
+#[derive(Debug, Default)]
+pub(crate) enum PeerState {
+    #[default]
+    // Will be tried to be connected as soon as possible.
+    Queued,
+    Connecting(PeerTx),
+    Live(LivePeerState),
+    // There was an error, and it's waiting for exponential backoff.
+    Dead,
+    // We don't need to do anything with the peer any longer.
+    // The peer has the full torrent, and we have the full torrent, so no need
+    // to keep talking to it.
+    NotNeeded,
+}
+
+impl std::fmt::Display for PeerState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.name())
+    }
+}
+
+impl PeerState {
+    pub fn name(&self) -> &'static str {
+        match self {
+            PeerState::Queued => "queued",
+            PeerState::Connecting(_) => "connecting",
+            PeerState::Live(_) => "live",
+            PeerState::Dead => "dead",
+            PeerState::NotNeeded => "not needed",
+        }
+    }
+
+    pub fn take_live_no_counters(self) -> Option<LivePeerState> {
+        match self {
+            PeerState::Live(l) => Some(l),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct PeerStateNoMut(PeerState);
+
+impl PeerStateNoMut {
+    pub fn get(&self) -> &PeerState {
+        &self.0
+    }
+
+    pub fn take(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
+        self.set(Default::default(), counters)
+    }
+
+    pub fn set(&mut self, new: PeerState, counters: &AggregatePeerStatsAtomic) -> PeerState {
+        counters.incdec(&self.0, &new);
+        std::mem::replace(&mut self.0, new)
+    }
+
+    pub fn get_live_mut(&mut self) -> Option<&mut LivePeerState> {
+        match &mut self.0 {
+            PeerState::Live(l) => Some(l),
+            _ => None,
+        }
+    }
+
+    pub fn queued_to_connecting(
+        &mut self,
+        counters: &AggregatePeerStatsAtomic,
+    ) -> Option<(PeerRx, PeerTx)> {
+        if let PeerState::Queued = &self.0 {
+            let (tx, rx) = unbounded_channel();
+            let tx_2 = tx.clone();
+            self.set(PeerState::Connecting(tx), counters);
+            Some((rx, tx_2))
+        } else {
+            None
+        }
+    }
+    pub fn connecting_to_live(
+        &mut self,
+        peer_id: Id20,
+        counters: &AggregatePeerStatsAtomic,
+    ) -> Option<&mut LivePeerState> {
+        if let PeerState::Connecting(_) = &self.0 {
+            let tx = match self.take(counters) {
+                PeerState::Connecting(tx) => tx,
+                _ => unreachable!(),
+            };
+            self.set(PeerState::Live(LivePeerState::new(peer_id, tx)), counters);
+            self.get_live_mut()
+        } else {
+            None
+        }
+    }
+
+    pub fn set_not_needed(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
+        self.set(PeerState::NotNeeded, counters)
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct LivePeerState {
+    #[allow(dead_code)]
+    peer_id: Id20,
+
+    pub peer_interested: bool,
+
+    // This is used to track the pieces the peer has.
+    pub bitfield: BF,
+
+    // When the peer sends us data this is used to track if we asked for it.
+    pub inflight_requests: HashSet<InflightRequest>,
+
+    // The main channel to send requests to peer.
+    pub tx: PeerTx,
+}
+
+impl LivePeerState {
+    pub fn new(peer_id: Id20, tx: PeerTx) -> Self {
+        LivePeerState {
+            peer_id,
+            peer_interested: false,
+            bitfield: BF::new(),
+            inflight_requests: Default::default(),
+            tx,
+        }
+    }
+
+    pub fn has_full_torrent(&self, total_pieces: usize) -> bool {
+        self.bitfield
+            .get(0..total_pieces)
+            .map_or(false, |s| s.all())
+    }
+}

crates/librqbit/src/torrent_state/live/peer/stats/atomic.rs

@@ -0,0 +1,41 @@
+use std::{
+    sync::{
+        atomic::{AtomicU32, AtomicU64},
+        Arc,
+    },
+    time::Duration,
+};
+
+use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
+
+#[derive(Default, Debug)]
+pub(crate) struct PeerCountersAtomic {
+    pub fetched_bytes: AtomicU64,
+    pub total_time_connecting_ms: AtomicU64,
+    pub connection_attempts: AtomicU32,
+    pub connections: AtomicU32,
+    pub errors: AtomicU32,
+    pub fetched_chunks: AtomicU32,
+    pub downloaded_and_checked_pieces: AtomicU32,
+    pub downloaded_and_checked_bytes: AtomicU64,
+}
+
+#[derive(Debug)]
+pub(crate) struct PeerStats {
+    pub counters: Arc<PeerCountersAtomic>,
+    pub backoff: ExponentialBackoff,
+}
+
+impl Default for PeerStats {
+    fn default() -> Self {
+        Self {
+            counters: Arc::new(Default::default()),
+            backoff: ExponentialBackoffBuilder::new()
+                .with_initial_interval(Duration::from_secs(10))
+                .with_multiplier(6.)
+                .with_max_interval(Duration::from_secs(3600))
+                .with_max_elapsed_time(Some(Duration::from_secs(86400)))
+                .build(),
+        }
+    }
+}

crates/librqbit/src/torrent_state/live/peer/stats/mod.rs

@@ -0,0 +1,2 @@
+pub mod atomic;
+pub mod snapshot;

crates/librqbit/src/torrent_state/live/peer/stats/snapshot.rs

@@ -0,0 +1,70 @@
+use std::{collections::HashMap, sync::atomic::Ordering};
+
+use serde::{Deserialize, Serialize};
+
+use crate::torrent_state::live::peer::{Peer, PeerState};
+
+#[derive(Serialize, Deserialize)]
+pub struct PeerCounters {
+    pub fetched_bytes: u64,
+    pub total_time_connecting_ms: u64,
+    pub connection_attempts: u32,
+    pub connections: u32,
+    pub errors: u32,
+    pub fetched_chunks: u32,
+    pub downloaded_and_checked_pieces: u32,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct PeerStats {
+    pub counters: PeerCounters,
+    pub state: &'static str,
+}
+
+impl From<&super::atomic::PeerCountersAtomic> for PeerCounters {
+    fn from(counters: &super::atomic::PeerCountersAtomic) -> Self {
+        Self {
+            fetched_bytes: counters.fetched_bytes.load(Ordering::Relaxed),
+            total_time_connecting_ms: counters.total_time_connecting_ms.load(Ordering::Relaxed),
+            connection_attempts: counters.connection_attempts.load(Ordering::Relaxed),
+            connections: counters.connections.load(Ordering::Relaxed),
+            errors: counters.errors.load(Ordering::Relaxed),
+            fetched_chunks: counters.fetched_chunks.load(Ordering::Relaxed),
+            downloaded_and_checked_pieces: counters
+                .downloaded_and_checked_pieces
+                .load(Ordering::Relaxed),
+        }
+    }
+}
+
+impl From<&Peer> for PeerStats {
+    fn from(peer: &Peer) -> Self {
+        Self {
+            counters: peer.stats.counters.as_ref().into(),
+            state: peer.state.get().name(),
+        }
+    }
+}
+
+#[derive(Serialize)]
+pub struct PeerStatsSnapshot {
+    pub peers: HashMap<String, PeerStats>,
+}
+
+#[derive(Clone, Copy, Default, Deserialize)]
+pub enum PeerStatsFilterState {
+    All,
+    #[default]
+    Live,
+}
+
+impl PeerStatsFilterState {
+    pub(crate) fn matches(&self, s: &PeerState) -> bool {
+        matches!((self, s), (Self::All, _) | (Self::Live, PeerState::Live(_)))
+    }
+}
+
+#[derive(Default, Deserialize)]
+pub struct PeerStatsFilter {
+    pub state: PeerStatsFilterState,
+}

crates/librqbit/src/torrent_state/live/peers/mod.rs

@@ -0,0 +1,107 @@
+use std::net::SocketAddr;
+
+use anyhow::Context;
+use backoff::backoff::Backoff;
+use dashmap::DashMap;
+
+use crate::{
+    torrent_state::utils::{atomic_inc, TimedExistence},
+    type_aliases::{PeerHandle, BF},
+};
+
+use self::stats::{atomic::AggregatePeerStatsAtomic, snapshot::AggregatePeerStats};
+
+use super::peer::{LivePeerState, Peer, PeerRx, PeerState, PeerTx};
+
+pub mod stats;
+
+#[derive(Default)]
+pub(crate) struct PeerStates {
+    pub stats: AggregatePeerStatsAtomic,
+    pub states: DashMap<PeerHandle, Peer>,
+}
+
+impl PeerStates {
+    pub fn stats(&self) -> AggregatePeerStats {
+        AggregatePeerStats::from(&self.stats)
+    }
+
+    pub fn add_if_not_seen(&self, addr: SocketAddr) -> Option<PeerHandle> {
+        use dashmap::mapref::entry::Entry;
+        match self.states.entry(addr) {
+            Entry::Occupied(_) => None,
+            Entry::Vacant(vac) => {
+                vac.insert(Default::default());
+                atomic_inc(&self.stats.queued);
+                atomic_inc(&self.stats.seen);
+                Some(addr)
+            }
+        }
+    }
+    pub fn with_peer<R>(&self, addr: PeerHandle, f: impl FnOnce(&Peer) -> R) -> Option<R> {
+        self.states.get(&addr).map(|e| f(e.value()))
+    }
+
+    pub fn with_peer_mut<R>(
+        &self,
+        addr: PeerHandle,
+        reason: &'static str,
+        f: impl FnOnce(&mut Peer) -> R,
+    ) -> Option<R> {
+        use crate::torrent_state::utils::timeit;
+        timeit(reason, || self.states.get_mut(&addr))
+            .map(|e| f(TimedExistence::new(e, reason).value_mut()))
+    }
+
+    pub fn with_live_mut<R>(
+        &self,
+        addr: PeerHandle,
+        reason: &'static str,
+        f: impl FnOnce(&mut LivePeerState) -> R,
+    ) -> Option<R> {
+        self.with_peer_mut(addr, reason, |peer| peer.state.get_live_mut().map(f))
+            .flatten()
+    }
+
+    pub fn drop_peer(&self, handle: PeerHandle) -> Option<Peer> {
+        let p = self.states.remove(&handle).map(|r| r.1)?;
+        self.stats.dec(p.state.get());
+        Some(p)
+    }
+
+    pub fn mark_peer_interested(&self, handle: PeerHandle, is_interested: bool) -> Option<bool> {
+        self.with_live_mut(handle, "mark_peer_interested", |live| {
+            let prev = live.peer_interested;
+            live.peer_interested = is_interested;
+            prev
+        })
+    }
+    pub fn update_bitfield_from_vec(&self, handle: PeerHandle, bitfield: Vec<u8>) -> Option<()> {
+        self.with_live_mut(handle, "update_bitfield_from_vec", |live| {
+            live.bitfield = BF::from_vec(bitfield);
+        })
+    }
+    pub fn mark_peer_connecting(&self, h: PeerHandle) -> anyhow::Result<(PeerRx, PeerTx)> {
+        let rx = self
+            .with_peer_mut(h, "mark_peer_connecting", |peer| {
+                peer.state
+                    .queued_to_connecting(&self.stats)
+                    .context("invalid peer state")
+            })
+            .context("peer not found in states")??;
+        Ok(rx)
+    }
+
+    pub fn reset_peer_backoff(&self, handle: PeerHandle) {
+        self.with_peer_mut(handle, "reset_peer_backoff", |p| {
+            p.stats.backoff.reset();
+        });
+    }
+
+    pub fn mark_peer_not_needed(&self, handle: PeerHandle) -> Option<PeerState> {
+        let prev = self.with_peer_mut(handle, "mark_peer_not_needed", |peer| {
+            peer.state.set_not_needed(&self.stats)
+        })?;
+        Some(prev)
+    }
+}

crates/librqbit/src/torrent_state/live/peers/stats/atomic.rs

@@ -0,0 +1,43 @@
+use std::sync::atomic::AtomicU32;
+
+use serde::Serialize;
+
+use crate::torrent_state::{
+    live::peer::PeerState,
+    utils::{atomic_dec, atomic_inc},
+};
+
+#[derive(Debug, Default, Serialize)]
+pub(crate) struct AggregatePeerStatsAtomic {
+    pub queued: AtomicU32,
+    pub connecting: AtomicU32,
+    pub live: AtomicU32,
+    pub seen: AtomicU32,
+    pub dead: AtomicU32,
+    pub not_needed: AtomicU32,
+}
+
+impl AggregatePeerStatsAtomic {
+    pub fn counter(&self, state: &PeerState) -> &AtomicU32 {
+        match state {
+            PeerState::Connecting(_) => &self.connecting,
+            PeerState::Live(_) => &self.live,
+            PeerState::Queued => &self.queued,
+            PeerState::Dead => &self.dead,
+            PeerState::NotNeeded => &self.not_needed,
+        }
+    }
+
+    pub fn inc(&self, state: &PeerState) {
+        atomic_inc(self.counter(state));
+    }
+
+    pub fn dec(&self, state: &PeerState) {
+        atomic_dec(self.counter(state));
+    }
+
+    pub fn incdec(&self, old: &PeerState, new: &PeerState) {
+        self.dec(old);
+        self.inc(new);
+    }
+}

crates/librqbit/src/torrent_state/live/peers/stats/mod.rs

@@ -0,0 +1,2 @@
+pub mod atomic;
+pub mod snapshot;

crates/librqbit/src/torrent_state/live/peers/stats/snapshot.rs

@@ -0,0 +1,29 @@
+use std::sync::atomic::Ordering;
+
+use serde::Serialize;
+
+use super::atomic::AggregatePeerStatsAtomic;
+
+#[derive(Debug, Default, Serialize, PartialEq, Eq)]
+pub struct AggregatePeerStats {
+    pub queued: usize,
+    pub connecting: usize,
+    pub live: usize,
+    pub seen: usize,
+    pub dead: usize,
+    pub not_needed: usize,
+}
+
+impl<'a> From<&'a AggregatePeerStatsAtomic> for AggregatePeerStats {
+    fn from(s: &'a AggregatePeerStatsAtomic) -> Self {
+        let ordering = Ordering::Relaxed;
+        Self {
+            queued: s.queued.load(ordering) as usize,
+            connecting: s.connecting.load(ordering) as usize,
+            live: s.live.load(ordering) as usize,
+            seen: s.seen.load(ordering) as usize,
+            dead: s.dead.load(ordering) as usize,
+            not_needed: s.not_needed.load(ordering) as usize,
+        }
+    }
+}

crates/librqbit/src/torrent_state/live/stats/atomic.rs

@@ -0,0 +1,25 @@
+use std::{
+    sync::atomic::{AtomicU64, Ordering},
+    time::Duration,
+};
+
+#[derive(Default, Debug)]
+pub struct AtomicStats {
+    pub have_bytes: AtomicU64,
+    pub downloaded_and_checked_bytes: AtomicU64,
+    pub downloaded_and_checked_pieces: AtomicU64,
+    pub uploaded_bytes: AtomicU64,
+    pub fetched_bytes: AtomicU64,
+    pub total_piece_download_ms: AtomicU64,
+}
+
+impl AtomicStats {
+    pub fn average_piece_download_time(&self) -> Option<Duration> {
+        let d = self.downloaded_and_checked_pieces.load(Ordering::Acquire);
+        let t = self.total_piece_download_ms.load(Ordering::Acquire);
+        if d == 0 {
+            return None;
+        }
+        Some(Duration::from_secs_f64(t as f64 / d as f64 / 1000f64))
+    }
+}

crates/librqbit/src/torrent_state/live/stats/mod.rs

@@ -0,0 +1,2 @@
+pub mod atomic;
+pub mod snapshot;

crates/librqbit/src/torrent_state/live/stats/snapshot.rs

@@ -0,0 +1,30 @@
+use std::time::Duration;
+
+use serde::Serialize;
+
+use crate::torrent_state::live::peers::stats::snapshot::AggregatePeerStats;
+
+#[derive(Debug, Serialize, Default)]
+pub struct StatsSnapshot {
+    pub have_bytes: u64,
+    pub downloaded_and_checked_bytes: u64,
+    pub downloaded_and_checked_pieces: u64,
+    pub fetched_bytes: u64,
+    pub uploaded_bytes: u64,
+    pub initially_needed_bytes: u64,
+    pub remaining_bytes: u64,
+    pub total_bytes: u64,
+    pub total_piece_download_ms: u64,
+    pub peer_stats: AggregatePeerStats,
+}
+
+impl StatsSnapshot {
+    pub fn average_piece_download_time(&self) -> Option<Duration> {
+        let d = self.downloaded_and_checked_pieces;
+        let t = self.total_piece_download_ms;
+        if d == 0 {
+            return None;
+        }
+        Some(Duration::from_secs_f64(t as f64 / d as f64 / 1000f64))
+    }
+}

crates/librqbit/src/torrent_state/mod.rs

@@ -0,0 +1,490 @@
+pub mod initializing;
+pub mod live;
+pub mod paused;
+pub mod stats;
+pub mod utils;
+
+use std::collections::HashSet;
+use std::net::SocketAddr;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::time::Duration;
+
+use anyhow::bail;
+use anyhow::Context;
+use buffers::ByteString;
+use librqbit_core::id20::Id20;
+use librqbit_core::lengths::Lengths;
+use librqbit_core::peer_id::generate_peer_id;
+
+use librqbit_core::torrent_metainfo::TorrentMetaV1Info;
+pub use live::*;
+use parking_lot::RwLock;
+
+use tokio_stream::StreamExt;
+use tracing::debug;
+use tracing::error;
+use tracing::error_span;
+use tracing::warn;
+use url::Url;
+
+use crate::chunk_tracker::ChunkTracker;
+use crate::spawn_utils::spawn;
+use crate::spawn_utils::BlockingSpawner;
+use crate::torrent_state::stats::LiveStats;
+
+use initializing::TorrentStateInitializing;
+
+use self::paused::TorrentStatePaused;
+use self::stats::TorrentStats;
+
+pub enum ManagedTorrentState {
+    Initializing(Arc<TorrentStateInitializing>),
+    Paused(TorrentStatePaused),
+    Live(Arc<TorrentStateLive>),
+    Error(anyhow::Error),
+
+    // This is used when swapping between states, outside world should never see it.
+    None,
+}
+
+impl ManagedTorrentState {
+    fn assert_paused(self) -> TorrentStatePaused {
+        match self {
+            Self::Paused(paused) => paused,
+            _ => panic!("Expected paused state"),
+        }
+    }
+
+    pub(crate) fn take(&mut self) -> Self {
+        std::mem::replace(self, Self::None)
+    }
+}
+
+pub(crate) struct ManagedTorrentLocked {
+    pub state: ManagedTorrentState,
+}
+
+#[derive(Default)]
+pub(crate) struct ManagedTorrentOptions {
+    pub force_tracker_interval: Option<Duration>,
+    pub peer_connect_timeout: Option<Duration>,
+    pub peer_read_write_timeout: Option<Duration>,
+    pub overwrite: bool,
+}
+
+pub struct ManagedTorrentInfo {
+    pub info: TorrentMetaV1Info<ByteString>,
+    pub info_hash: Id20,
+    pub out_dir: PathBuf,
+    pub spawner: BlockingSpawner,
+    pub trackers: HashSet<Url>,
+    pub peer_id: Id20,
+    pub lengths: Lengths,
+    pub span: tracing::Span,
+    pub(crate) options: ManagedTorrentOptions,
+}
+
+pub struct ManagedTorrent {
+    pub info: Arc<ManagedTorrentInfo>,
+    pub(crate) only_files: Option<Vec<usize>>,
+    locked: RwLock<ManagedTorrentLocked>,
+}
+
+impl ManagedTorrent {
+    pub fn info(&self) -> &ManagedTorrentInfo {
+        &self.info
+    }
+
+    pub fn get_total_bytes(&self) -> u64 {
+        self.info.lengths.total_length()
+    }
+
+    pub fn info_hash(&self) -> Id20 {
+        self.info.info_hash
+    }
+
+    pub fn only_files(&self) -> Option<Vec<usize>> {
+        self.only_files.clone()
+    }
+
+    pub fn with_state<R>(&self, f: impl FnOnce(&ManagedTorrentState) -> R) -> R {
+        f(&self.locked.read().state)
+    }
+
+    pub(crate) fn with_state_mut<R>(&self, f: impl FnOnce(&mut ManagedTorrentState) -> R) -> R {
+        f(&mut self.locked.write().state)
+    }
+
+    pub fn with_chunk_tracker<R>(&self, f: impl FnOnce(&ChunkTracker) -> R) -> anyhow::Result<R> {
+        let g = self.locked.read();
+        match &g.state {
+            ManagedTorrentState::Paused(p) => Ok(f(&p.chunk_tracker)),
+            ManagedTorrentState::Live(l) => Ok(f(l
+                .lock_read("chunk_tracker")
+                .get_chunks()
+                .context("error getting chunks")?)),
+            _ => bail!("no chunk tracker, torrent neither paused nor live"),
+        }
+    }
+
+    pub fn live(&self) -> Option<Arc<TorrentStateLive>> {
+        let g = self.locked.read();
+        match &g.state {
+            ManagedTorrentState::Live(live) => Some(live.clone()),
+            _ => None,
+        }
+    }
+
+    fn stop_with_error(&self, error: anyhow::Error) {
+        let mut g = self.locked.write();
+
+        match g.state.take() {
+            ManagedTorrentState::Live(live) => {
+                if let Err(err) = live.pause() {
+                    warn!(
+                        "error pausing live torrent during fatal error handling: {:?}",
+                        err
+                    );
+                }
+            }
+            ManagedTorrentState::Error(e) => {
+                warn!("bug: torrent already was in error state when trying to stop it. Previous error was: {:?}", e);
+            }
+            ManagedTorrentState::None => {
+                warn!("bug: torrent encountered in None state during fatal error handling")
+            }
+            _ => {}
+        };
+
+        g.state = ManagedTorrentState::Error(error)
+    }
+
+    pub fn start(
+        self: &Arc<Self>,
+        initial_peers: Vec<SocketAddr>,
+        peer_rx: Option<impl StreamExt<Item = SocketAddr> + Unpin + Send + Sync + 'static>,
+        start_paused: bool,
+    ) -> anyhow::Result<()> {
+        let mut g = self.locked.write();
+
+        let spawn_fatal_errors_receiver =
+            |state: &Arc<Self>, rx: tokio::sync::oneshot::Receiver<anyhow::Error>| {
+                let span = state.info.span.clone();
+                let state = Arc::downgrade(state);
+                spawn(
+                    "fatal_errors_receiver",
+                    error_span!(parent: span, "fatal_errors_receiver"),
+                    async move {
+                        let e = match rx.await {
+                            Ok(e) => e,
+                            Err(_) => return Ok(()),
+                        };
+                        if let Some(state) = state.upgrade() {
+                            state.stop_with_error(e);
+                        } else {
+                            warn!("tried to stop the torrent with error, but it's couldn't upgrade the arc");
+                        }
+                        Ok(())
+                    },
+                );
+            };
+
+        fn spawn_peer_adder(
+            live: &Arc<TorrentStateLive>,
+            initial_peers: Vec<SocketAddr>,
+            peer_rx: Option<impl StreamExt<Item = SocketAddr> + Unpin + Send + Sync + 'static>,
+        ) {
+            let span = live.meta().span.clone();
+            let live = Arc::downgrade(live);
+            spawn(
+                "external_peer_adder",
+                error_span!(parent: span, "external_peer_adder"),
+                async move {
+                    {
+                        let live: Arc<TorrentStateLive> =
+                            live.upgrade().context("no longer live")?;
+                        for peer in initial_peers {
+                            live.add_peer_if_not_seen(peer).context("torrent closed")?;
+                        }
+                    }
+
+                    if let Some(mut peer_rx) = peer_rx {
+                        while let Some(peer) = peer_rx.next().await {
+                            let live = match live.upgrade() {
+                                Some(live) => live,
+                                None => return Ok(()),
+                            };
+                            live.add_peer_if_not_seen(peer).context("torrent closed")?;
+                        }
+                    } else {
+                        error!("peer rx is not set");
+                    }
+
+                    Ok(())
+                },
+            );
+        }
+
+        match &g.state {
+            ManagedTorrentState::Live(_) => {
+                bail!("torrent is already live");
+            }
+            ManagedTorrentState::Initializing(init) => {
+                let init = init.clone();
+                drop(g);
+                let t = self.clone();
+                let span = self.info().span.clone();
+                spawn(
+                    "initialize_and_start",
+                    error_span!(parent: span.clone(), "initialize_and_start"),
+                    async move {
+                        match init.check().await {
+                            Ok(paused) => {
+                                let mut g = t.locked.write();
+                                if let ManagedTorrentState::Initializing(_) = &g.state {
+                                } else {
+                                    debug!("no need to start torrent anymore, as it switched state from initilizing");
+                                    return Ok(());
+                                }
+
+                                if start_paused {
+                                    g.state = ManagedTorrentState::Paused(paused);
+                                    return Ok(());
+                                }
+
+                                let (tx, rx) = tokio::sync::oneshot::channel();
+                                let live = TorrentStateLive::new(paused, tx);
+                                g.state = ManagedTorrentState::Live(live.clone());
+
+                                spawn_fatal_errors_receiver(&t, rx);
+                                spawn_peer_adder(&live, initial_peers, peer_rx);
+
+                                Ok(())
+                            }
+                            Err(err) => {
+                                let result = anyhow::anyhow!("{:?}", err);
+                                t.locked.write().state = ManagedTorrentState::Error(err);
+                                Err(result)
+                            }
+                        }
+                    },
+                );
+                Ok(())
+            }
+            ManagedTorrentState::Paused(_) => {
+                let paused = g.state.take().assert_paused();
+                let (tx, rx) = tokio::sync::oneshot::channel();
+                let live = TorrentStateLive::new(paused, tx);
+                g.state = ManagedTorrentState::Live(live.clone());
+                spawn_fatal_errors_receiver(self, rx);
+                spawn_peer_adder(&live, initial_peers, peer_rx);
+                Ok(())
+            }
+            ManagedTorrentState::Error(_) => {
+                let initializing = Arc::new(TorrentStateInitializing::new(
+                    self.info.clone(),
+                    self.only_files.clone(),
+                ));
+                g.state = ManagedTorrentState::Initializing(initializing.clone());
+                drop(g);
+
+                // Recurse.
+                self.start(initial_peers, peer_rx, start_paused)
+            }
+            ManagedTorrentState::None => bail!("bug: torrent is in empty state"),
+        }
+    }
+
+    pub fn pause(&self) -> anyhow::Result<()> {
+        let mut g = self.locked.write();
+        match &g.state {
+            ManagedTorrentState::Live(live) => {
+                let paused = live.pause()?;
+                g.state = ManagedTorrentState::Paused(paused);
+                Ok(())
+            }
+            ManagedTorrentState::Initializing(_) => {
+                bail!("torrent is initializing, can't pause");
+            }
+            ManagedTorrentState::Paused(_) => {
+                bail!("torrent is already paused");
+            }
+            ManagedTorrentState::Error(_) => {
+                bail!("can't pause torrent in error state")
+            }
+            ManagedTorrentState::None => bail!("bug: torrent is in empty state"),
+        }
+    }
+
+    pub fn stats(&self) -> TorrentStats {
+        let mut resp = TorrentStats {
+            total_bytes: self.info().lengths.total_length(),
+            state: "",
+            error: None,
+            progress_bytes: 0,
+            finished: false,
+            live: None,
+        };
+
+        self.with_state(|s| {
+            match s {
+                ManagedTorrentState::Initializing(i) => {
+                    resp.state = "initializing";
+                    resp.progress_bytes = i.checked_bytes.load(Ordering::Relaxed);
+                }
+                ManagedTorrentState::Paused(p) => {
+                    resp.state = "paused";
+                    resp.progress_bytes = p.have_bytes;
+                    resp.finished = p.have_bytes == resp.total_bytes;
+                }
+                ManagedTorrentState::Live(l) => {
+                    resp.state = "live";
+                    let live_stats = LiveStats::from(l.as_ref());
+                    resp.progress_bytes = live_stats.snapshot.have_bytes;
+                    resp.finished = resp.progress_bytes == resp.total_bytes;
+                    resp.live = Some(live_stats);
+                }
+                ManagedTorrentState::Error(e) => {
+                    resp.state = "error";
+                    resp.error = Some(format!("{:?}", e))
+                }
+                ManagedTorrentState::None => {
+                    resp.state = "error";
+                    resp.error = Some("bug: torrent in broken \"None\" state".to_string());
+                }
+            }
+            resp
+        })
+    }
+
+    pub async fn wait_until_completed(&self) -> anyhow::Result<()> {
+        // TODO: rewrite, this polling is horrible
+        let live = loop {
+            let live = self.with_state(|s| match s {
+                ManagedTorrentState::Initializing(_) | ManagedTorrentState::Paused(_) => Ok(None),
+                ManagedTorrentState::Live(l) => Ok(Some(l.clone())),
+                ManagedTorrentState::Error(e) => bail!("{:?}", e),
+                ManagedTorrentState::None => bail!("bug: torrent state is None"),
+            })?;
+            if let Some(live) = live {
+                break live;
+            }
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        };
+
+        live.wait_until_completed().await;
+        Ok(())
+    }
+}
+
+pub struct ManagedTorrentBuilder {
+    info: TorrentMetaV1Info<ByteString>,
+    info_hash: Id20,
+    output_folder: PathBuf,
+    force_tracker_interval: Option<Duration>,
+    peer_connect_timeout: Option<Duration>,
+    peer_read_write_timeout: Option<Duration>,
+    only_files: Option<Vec<usize>>,
+    trackers: Vec<Url>,
+    peer_id: Option<Id20>,
+    overwrite: bool,
+    spawner: Option<BlockingSpawner>,
+}
+
+impl ManagedTorrentBuilder {
+    pub fn new<P: AsRef<Path>>(
+        info: TorrentMetaV1Info<ByteString>,
+        info_hash: Id20,
+        output_folder: P,
+    ) -> Self {
+        Self {
+            info,
+            info_hash,
+            output_folder: output_folder.as_ref().into(),
+            spawner: None,
+            force_tracker_interval: None,
+            peer_connect_timeout: None,
+            peer_read_write_timeout: None,
+            only_files: None,
+            trackers: Default::default(),
+            peer_id: None,
+            overwrite: false,
+        }
+    }
+
+    pub fn only_files(&mut self, only_files: Vec<usize>) -> &mut Self {
+        self.only_files = Some(only_files);
+        self
+    }
+
+    pub fn trackers(&mut self, trackers: Vec<Url>) -> &mut Self {
+        self.trackers = trackers;
+        self
+    }
+
+    pub fn overwrite(&mut self, overwrite: bool) -> &mut Self {
+        self.overwrite = overwrite;
+        self
+    }
+
+    pub fn force_tracker_interval(&mut self, force_tracker_interval: Duration) -> &mut Self {
+        self.force_tracker_interval = Some(force_tracker_interval);
+        self
+    }
+
+    pub fn spawner(&mut self, spawner: BlockingSpawner) -> &mut Self {
+        self.spawner = Some(spawner);
+        self
+    }
+
+    pub fn peer_id(&mut self, peer_id: Id20) -> &mut Self {
+        self.peer_id = Some(peer_id);
+        self
+    }
+
+    pub fn peer_connect_timeout(&mut self, timeout: Duration) -> &mut Self {
+        self.peer_connect_timeout = Some(timeout);
+        self
+    }
+
+    pub fn peer_read_write_timeout(&mut self, timeout: Duration) -> &mut Self {
+        self.peer_read_write_timeout = Some(timeout);
+        self
+    }
+
+    pub(crate) fn build(self, span: tracing::Span) -> anyhow::Result<ManagedTorrentHandle> {
+        let lengths = Lengths::from_torrent(&self.info)?;
+        let info = Arc::new(ManagedTorrentInfo {
+            span,
+            info: self.info,
+            info_hash: self.info_hash,
+            out_dir: self.output_folder,
+            trackers: self.trackers.into_iter().collect(),
+            spawner: self.spawner.unwrap_or_default(),
+            peer_id: self.peer_id.unwrap_or_else(generate_peer_id),
+            lengths,
+            options: ManagedTorrentOptions {
+                force_tracker_interval: self.force_tracker_interval,
+                peer_connect_timeout: self.peer_connect_timeout,
+                peer_read_write_timeout: self.peer_read_write_timeout,
+                overwrite: self.overwrite,
+            },
+        });
+        let initializing = Arc::new(TorrentStateInitializing::new(
+            info.clone(),
+            self.only_files.clone(),
+        ));
+        Ok(Arc::new(ManagedTorrent {
+            only_files: self.only_files,
+            locked: RwLock::new(ManagedTorrentLocked {
+                state: ManagedTorrentState::Initializing(initializing),
+            }),
+            info,
+        }))
+    }
+}
+
+pub type ManagedTorrentHandle = Arc<ManagedTorrent>;

crates/librqbit/src/torrent_state/paused.rs

@@ -0,0 +1,24 @@
+use std::{fs::File, path::PathBuf, sync::Arc};
+
+use parking_lot::Mutex;
+
+use crate::chunk_tracker::ChunkTracker;
+
+use super::ManagedTorrentInfo;
+
+pub struct TorrentStatePaused {
+    pub(crate) info: Arc<ManagedTorrentInfo>,
+    pub(crate) files: Vec<Arc<Mutex<File>>>,
+    pub(crate) filenames: Vec<PathBuf>,
+    pub(crate) chunk_tracker: ChunkTracker,
+    pub(crate) have_bytes: u64,
+}
+
+// impl TorrentStatePaused {
+//     pub fn get_have_bytes(&self) -> u64 {
+//         self.have_bytes
+//     }
+//     pub fn get_needed_bytes(&self) -> u64 {
+//         self.needed_bytes
+//     }
+// }

crates/librqbit/src/torrent_state/stats.rs

@@ -0,0 +1,198 @@
+use std::time::Duration;
+
+use serde::Serialize;
+
+use super::{live::stats::snapshot::StatsSnapshot, TorrentStateLive};
+use size_format::SizeFormatterBinary as SF;
+
+#[derive(Serialize, Default, Debug)]
+pub struct LiveStats {
+    pub snapshot: StatsSnapshot,
+    pub average_piece_download_time: Option<Duration>,
+    pub download_speed: Speed,
+    pub time_remaining: Option<DurationWithHumanReadable>,
+}
+
+impl std::fmt::Display for LiveStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "down speed: {}", self.download_speed)?;
+        if let Some(time_remaining) = &self.time_remaining {
+            write!(f, " eta: {time_remaining}")?;
+        }
+        Ok(())
+    }
+}
+
+impl From<&TorrentStateLive> for LiveStats {
+    fn from(live: &TorrentStateLive) -> Self {
+        let snapshot = live.stats_snapshot();
+        let estimator = live.speed_estimator();
+
+        Self {
+            average_piece_download_time: snapshot.average_piece_download_time(),
+            snapshot,
+            download_speed: estimator.download_mbps().into(),
+            time_remaining: estimator.time_remaining().map(DurationWithHumanReadable),
+        }
+    }
+}
+
+#[derive(Serialize, Debug)]
+pub struct TorrentStats {
+    pub state: &'static str,
+    pub error: Option<String>,
+    pub progress_bytes: u64,
+    pub total_bytes: u64,
+    pub finished: bool,
+    pub live: Option<LiveStats>,
+}
+
+impl std::fmt::Display for TorrentStats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}: ", self.state)?;
+        if let Some(error) = &self.error {
+            return write!(f, "{error}");
+        }
+        write!(
+            f,
+            "{} ({})",
+            self.progress_percent_human_readable(),
+            self.progress_bytes_human_readable()
+        )?;
+        if let Some(live) = &self.live {
+            write!(f, " [{live}]")?;
+        }
+        Ok(())
+    }
+}
+
+impl TorrentStats {
+    pub fn progress_percent_human_readable(&self) -> impl std::fmt::Display {
+        struct Percents {
+            progress: u64,
+            total: u64,
+        }
+        impl std::fmt::Display for Percents {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                if self.total == 0 {
+                    return write!(f, "N/A");
+                }
+                let pct = self.progress as f64 / self.total as f64 * 100f64;
+                write!(f, "{pct:.2}%")
+            }
+        }
+        Percents {
+            progress: self.progress_bytes,
+            total: self.total_bytes,
+        }
+    }
+
+    pub fn progress_bytes_human_readable(&self) -> impl std::fmt::Display {
+        struct Progress {
+            progress: u64,
+            total: u64,
+        }
+        impl std::fmt::Display for Progress {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(f, "{} / {}", SF::new(self.progress), SF::new(self.total))
+            }
+        }
+        Progress {
+            progress: self.progress_bytes,
+            total: self.total_bytes,
+        }
+    }
+}
+
+fn format_seconds_to_time(seconds: u64, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let hours = seconds / 3600;
+    let minutes = (seconds % 3600) / 60;
+    let seconds = seconds % 60;
+
+    if hours > 0 {
+        write!(f, "{}h {}m", hours, minutes)
+    } else if minutes > 0 {
+        write!(f, "{}m {}s", minutes, seconds)
+    } else {
+        write!(f, "{}s", seconds)
+    }
+}
+
+pub struct DurationWithHumanReadable(Duration);
+
+impl core::fmt::Display for DurationWithHumanReadable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> core::fmt::Result {
+        format_seconds_to_time(self.0.as_secs(), f)
+    }
+}
+
+impl core::fmt::Debug for DurationWithHumanReadable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
+impl Serialize for DurationWithHumanReadable {
+    fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        #[derive(Serialize)]
+        struct Tmp {
+            duration: Duration,
+            human_readable: String,
+        }
+        Tmp {
+            duration: self.0,
+            human_readable: format!("{}", self),
+        }
+        .serialize(serializer)
+    }
+}
+
+#[derive(Default)]
+pub struct Speed {
+    pub mbps: f64,
+}
+
+impl Speed {
+    fn new(mbps: f64) -> Self {
+        Self { mbps }
+    }
+}
+
+impl From<f64> for Speed {
+    fn from(mbps: f64) -> Self {
+        Self::new(mbps)
+    }
+}
+
+impl core::fmt::Display for Speed {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:.2} MiB/s", self.mbps)
+    }
+}
+
+impl core::fmt::Debug for Speed {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self)
+    }
+}
+
+impl Serialize for Speed {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        #[derive(Serialize)]
+        struct Tmp {
+            mbps: f64,
+            human_readable: String,
+        }
+        Tmp {
+            mbps: self.mbps,
+            human_readable: format!("{}", self),
+        }
+        .serialize(serializer)
+    }
+}

crates/librqbit/src/torrent_state/utils.rs

@@ -0,0 +1,108 @@
+use std::sync::atomic::{AtomicU32, Ordering};
+
+pub fn atomic_inc(c: &AtomicU32) -> u32 {
+    c.fetch_add(1, Ordering::Relaxed)
+}
+
+pub fn atomic_dec(c: &AtomicU32) -> u32 {
+    c.fetch_sub(1, Ordering::Relaxed)
+}
+
+// Used during debugging to see if some locks take too long.
+#[cfg(not(feature = "timed_existence"))]
+mod timed_existence {
+    use std::ops::{Deref, DerefMut};
+
+    pub struct TimedExistence<T>(T);
+
+    impl<T> TimedExistence<T> {
+        #[inline(always)]
+        pub fn new(object: T, _reason: &'static str) -> Self {
+            Self(object)
+        }
+    }
+
+    impl<T> Deref for TimedExistence<T> {
+        type Target = T;
+
+        #[inline(always)]
+        fn deref(&self) -> &Self::Target {
+            &self.0
+        }
+    }
+
+    impl<T> DerefMut for TimedExistence<T> {
+        #[inline(always)]
+        fn deref_mut(&mut self) -> &mut Self::Target {
+            &mut self.0
+        }
+    }
+
+    #[inline(always)]
+    pub fn timeit<R>(_n: impl std::fmt::Display, f: impl FnOnce() -> R) -> R {
+        f()
+    }
+}
+
+#[cfg(feature = "timed_existence")]
+mod timed_existence {
+    use std::ops::{Deref, DerefMut};
+    use std::time::{Duration, Instant};
+    use tracing::warn;
+
+    const MAX: Duration = Duration::from_millis(1);
+
+    // Prints if the object exists for too long.
+    // This is used to track long-lived locks for debugging.
+    pub struct TimedExistence<T> {
+        object: T,
+        reason: &'static str,
+        started: Instant,
+    }
+
+    impl<T> TimedExistence<T> {
+        pub fn new(object: T, reason: &'static str) -> Self {
+            Self {
+                object,
+                reason,
+                started: Instant::now(),
+            }
+        }
+    }
+
+    impl<T> Drop for TimedExistence<T> {
+        fn drop(&mut self) {
+            let elapsed = self.started.elapsed();
+            let reason = self.reason;
+            if elapsed > MAX {
+                warn!("elapsed on lock {reason:?}: {elapsed:?}")
+            }
+        }
+    }
+
+    impl<T> Deref for TimedExistence<T> {
+        type Target = T;
+
+        fn deref(&self) -> &Self::Target {
+            &self.object
+        }
+    }
+
+    impl<T> DerefMut for TimedExistence<T> {
+        fn deref_mut(&mut self) -> &mut Self::Target {
+            &mut self.object
+        }
+    }
+
+    pub fn timeit<R>(name: impl std::fmt::Display, f: impl FnOnce() -> R) -> R {
+        let now = Instant::now();
+        let r = f();
+        let elapsed = now.elapsed();
+        if elapsed > MAX {
+            warn!("elapsed on \"{name:}\": {elapsed:?}")
+        }
+        r
+    }
+}
+
+pub use timed_existence::{timeit, TimedExistence};

crates/librqbit/webui/dist/index.html

@@ -8,6 +8,8 @@
 	<!-- Include Bootstrap CSS -->
 	<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"
 		integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous" />
+	<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css"
+		integrity="sha384-4LISF5TTJX/fLmGSxO53rV4miRxdg84mZsxmO8Rx5jGtp/LbrixFETvWa5a6sESd" crossorigin="anonymous">
   <script type="module" crossorigin src="app.js"></script>
 </head>
 

crates/librqbit/webui/index.html

@@ -8,6 +8,8 @@
 	<!-- Include Bootstrap CSS -->
 	<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"
 		integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous" />
+	<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css"
+		integrity="sha384-4LISF5TTJX/fLmGSxO53rV4miRxdg84mZsxmO8Rx5jGtp/LbrixFETvWa5a6sESd" crossorigin="anonymous">
 </head>
 
 <body>

crates/librqbit/webui/src/api.ts

@@ -29,7 +29,7 @@ export interface ListTorrentsResponse {
 }
 
 // Interface for the Torrent Stats API response
-export interface TorrentStats {
+export interface LiveTorrentStats {
     snapshot: {
         have_bytes: number;
         downloaded_and_checked_bytes: number;
@@ -69,6 +69,20 @@ export interface TorrentStats {
     } | null;
 }
 
+export const STATE_INITIALIZING = 'initializing';
+export const STATE_PAUSED = 'paused';
+export const STATE_LIVE = 'live';
+export const STATE_ERROR = 'error';
+
+export interface TorrentStats {
+    state: 'initializing' | 'paused' | 'live' | 'error',
+    error: string | null,
+    progress_bytes: number,
+    finished: boolean,
+    total_bytes: number,
+    live: LiveTorrentStats | null;
+}
+
 
 export interface ErrorDetails {
     id?: number,
@@ -129,7 +143,7 @@ export const API = {
         return makeRequest('GET', `/torrents/${index}`);
     },
     getTorrentStats: (index: number): Promise<TorrentStats> => {
-        return makeRequest('GET', `/torrents/${index}/stats`);
+        return makeRequest('GET', `/torrents/${index}/stats/v1`);
     },
 
     uploadTorrent: (data: string | File, opts?: {
@@ -144,5 +158,21 @@ export const API = {
             url += `&only_files=${opts.selectedFiles.join(',')}`;
         }
         return makeRequest('POST', url, data)
+    },
+
+    pause: (index: number): Promise<void> => {
+        return makeRequest('POST', `/torrents/${index}/pause`);
+    },
+
+    start: (index: number): Promise<void> => {
+        return makeRequest('POST', `/torrents/${index}/start`);
+    },
+
+    forget: (index: number): Promise<void> => {
+        return makeRequest('POST', `/torrents/${index}/forget`);
+    },
+
+    delete: (index: number): Promise<void> => {
+        return makeRequest('POST', `/torrents/${index}/delete`);
     }
 }

crates/librqbit/webui/src/index.tsx

@@ -1,7 +1,7 @@
-import { StrictMode, createContext, useContext, useEffect, useRef, useState } from 'react';
+import { MouseEventHandler, StrictMode, createContext, useContext, useEffect, useRef, useState } from 'react';
 import ReactDOM from 'react-dom/client';
 import { ProgressBar, Button, Container, Row, Col, Alert, Modal, Form, Spinner, Table } from 'react-bootstrap';
-import { AddTorrentResponse, TorrentDetails, TorrentFile, TorrentId, TorrentStats, ErrorDetails, API } from './api';
+import { AddTorrentResponse, TorrentDetails, TorrentFile, TorrentId, TorrentStats, ErrorDetails, API, STATE_INITIALIZING, STATE_LIVE, STATE_PAUSED, STATE_ERROR } from './api';
 
 interface Error {
     text: string,
@@ -14,33 +14,214 @@ interface ContextType {
 }
 
 const AppContext = createContext<ContextType>(null);
+const RefreshTorrentStatsContext = createContext<{ refresh: () => void }>(null);
+
+const IconButton: React.FC<{
+    className: string,
+    onClick: () => void,
+    disabled?: boolean,
+    color?: string,
+}> = ({ className, onClick, disabled, color }) => {
+    const onClickStopPropagation = (e) => {
+        e.stopPropagation();
+        if (disabled) {
+            return;
+        }
+        onClick();
+    }
+    return <a className={`bi ${className} p-1`} onClick={onClickStopPropagation} href='#'></a>
+}
+
+const DeleteTorrentModal = ({ id, show, onHide }) => {
+    if (!show) {
+        return null;
+    }
+    const [deleteFiles, setDeleteFiles] = useState(false);
+    const [error, setError] = useState<Error>(null);
+    const [deleting, setDeleting] = useState(false);
+
+    const ctx = useContext(AppContext);
+
+    const close = () => {
+        setDeleteFiles(false);
+        setError(null);
+        setDeleting(false);
+        onHide();
+    }
+
+    const deleteTorrent = () => {
+        setDeleting(true);
+
+        const call = deleteFiles ? API.delete : API.forget;
+
+        call(id).then(() => {
+            ctx.refreshTorrents();
+            close();
+        }).catch((e) => {
+            setError({
+                text: `Error deleting torrent id=${id}`,
+                details: e,
+            });
+            setDeleting(false);
+        })
+    }
+
+    return <Modal show={show} onHide={close}>
+        <Modal.Header closeButton>
+            Delete torrent
+        </Modal.Header>
+        <Modal.Body>
+            <Form>
+                <Form.Group controlId='delete-torrent'>
+                    <Form.Check
+                        type="checkbox"
+                        label='Also delete files'
+                        checked={deleteFiles}
+                        onChange={() => setDeleteFiles(!deleteFiles)}>
+                    </Form.Check>
+                </Form.Group>
+            </Form>
+            {error && <ErrorComponent error={error} />}
+        </Modal.Body>
+        <Modal.Footer>
+            {deleting && <Spinner />}
+            <Button variant="primary" onClick={deleteTorrent} disabled={deleting}>
+                OK
+            </Button>
+            <Button variant="secondary" onClick={close}>
+                Cancel
+            </Button>
+        </Modal.Footer>
+    </Modal>
+}
+
+const TorrentActions: React.FC<{
+    id: number, statsResponse: TorrentStats
+}> = ({ id, statsResponse }) => {
+    let state = statsResponse.state;
+
+    let [disabled, setDisabled] = useState<boolean>(false);
+    let [deleting, setDeleting] = useState<boolean>(false);
+
+    let refreshCtx = useContext(RefreshTorrentStatsContext);
+
+    const canPause = state == 'live';
+    const canUnpause = state == 'paused' || state == 'error';
+
+    const ctx = useContext(AppContext);
+
+    const unpause = () => {
+        setDisabled(true);
+        API.start(id).then(() => { refreshCtx.refresh() }, (e) => {
+            ctx.setCloseableError({
+                text: `Error starting torrent id=${id}`,
+                details: e,
+            });
+        }).finally(() => setDisabled(false))
+    };
+
+    const pause = () => {
+        setDisabled(true);
+        API.pause(id).then(() => { refreshCtx.refresh() }, (e) => {
+            ctx.setCloseableError({
+                text: `Error pausing torrent id=${id}`,
+                details: e,
+            });
+        }).finally(() => setDisabled(false))
+    };
+
+    const startDeleting = () => {
+        setDisabled(true);
+        setDeleting(true);
+    }
+
+    const cancelDeleting = () => {
+        setDisabled(false);
+        setDeleting(false);
+    }
+
+    return <Row>
+        <Col>
+            {canUnpause && <IconButton className="bi-play-circle" onClick={unpause} disabled={disabled} color='success' />}
+            {canPause && <IconButton className="bi-pause-circle" onClick={pause} disabled={disabled} />}
+            <IconButton className="bi-x-circle" onClick={startDeleting} disabled={disabled} color='danger' />
+            <DeleteTorrentModal id={id} show={deleting} onHide={cancelDeleting} />
+        </Col>
+    </Row>
+}
 
 const TorrentRow: React.FC<{
     id: number, detailsResponse: TorrentDetails, statsResponse: TorrentStats
 }> = ({ id, detailsResponse, statsResponse }) => {
-    const totalBytes = statsResponse?.snapshot?.total_bytes ?? 1;
+    const state = statsResponse?.state ?? "";
-    const downloadedBytes = statsResponse?.snapshot?.have_bytes ?? 0;
+    const error = statsResponse?.error;
-    const finished = totalBytes == downloadedBytes;
+    const totalBytes = statsResponse?.total_bytes ?? 1;
-    const downloadPercentage = (downloadedBytes / totalBytes) * 100;
+    const progressBytes = statsResponse?.progress_bytes ?? 0;
+    const finished = statsResponse?.finished || false;
+    const progressPercentage = error ? 100 : (progressBytes / totalBytes) * 100;
+    const isAnimated = (state == STATE_INITIALIZING || state == STATE_LIVE) && !finished;
+    const progressLabel = error ? 'Error' : `${progressPercentage.toFixed(2)}%`;
+    const progressBarVariant = error ? 'danger' : finished ? 'success' : state == STATE_INITIALIZING ? 'warning' : 'primary';
+
+    const formatPeersString = () => {
+        let peer_stats = statsResponse?.live?.snapshot.peer_stats;
+        if (!peer_stats) {
+            return '';
+        }
+        return `${peer_stats.live} / ${peer_stats.seen}`;
+    }
+
+    const formatDownloadSpeed = () => {
+        if (finished) {
+            return 'Completed';
+        }
+        switch (state) {
+            case STATE_PAUSED: return 'Paused';
+            case STATE_INITIALIZING: return 'Checking files';
+            case STATE_ERROR: return 'Error';
+        }
+
+        return statsResponse.live?.download_speed.human_readable ?? "N/A";
+    }
+
+    let classNames = [];
+
+    if (error) {
+        classNames.push('bg-warning');
+    } else {
+        if (id % 2 == 0) {
+            classNames.push('bg-light');
+        }
+    }
 
     return (
-        <Row className={`${id % 2 == 0 ? 'bg-light' : ''}`}>
+        <Row className={classNames.join(' ')}>
-            <Column size={4} label="Name">
+            <Column size={3} label="Name">
                 {detailsResponse ?
-                    <div className='text-truncate'>
+                    <>
-                        {getLargestFileName(detailsResponse)}
+                        <div className='text-truncate'>
-                    </div>
+                            {getLargestFileName(detailsResponse)}
+                        </div>
+                        {error && <p className='text-danger'><strong>Error:</strong> {error}</p>}
+                    </>
                     : <Spinner />}
             </Column>
             {statsResponse ?
                 <>
                     <Column label="Size">{`${formatBytes(totalBytes)} `}</Column>
-                    <Column size={2} label="Progress">
+                    <Column size={2} label={state == STATE_PAUSED ? 'Progress' : 'Progress'}>
-                        <ProgressBar now={downloadPercentage} label={`${downloadPercentage.toFixed(2)}% `} animated={!finished} />
+                        <ProgressBar
+                            now={progressPercentage}
+                            label={progressLabel}
+                            animated={isAnimated}
+                            variant={progressBarVariant} />
                     </Column>
-                    <Column size={2} label="Down Speed">{statsResponse.download_speed.human_readable}</Column>
+                    <Column size={2} label="Down Speed">{formatDownloadSpeed()}</Column>
                     <Column label="ETA">{getCompletionETA(statsResponse)}</Column>
-                    <Column size={2} label="Peers">{`${statsResponse.snapshot.peer_stats.live} / ${statsResponse.snapshot.peer_stats.seen}`}</Column >
+                    <Column size={2} label="Peers">{formatPeersString()}</Column >
+                    <Column label="Actions">
+                        <TorrentActions id={id} statsResponse={statsResponse} />
+                    </Column>
                 </>
                 : <Column label="Loading stats" size={8}><Spinner /></Column>
             }
@@ -63,6 +244,11 @@ const Column: React.FC<{
 const Torrent = ({ id, torrent }) => {
     const [detailsResponse, updateDetailsResponse] = useState<TorrentDetails>(null);
     const [statsResponse, updateStatsResponse] = useState<TorrentStats>(null);
+    const [forceStatsRefresh, setForceStatsRefresh] = useState(0);
+
+    const forceStatsRefreshCallback = () => {
+        setForceStatsRefresh(forceStatsRefresh + 1);
+    }
 
     // Update details once.
     useEffect(() => {
@@ -76,18 +262,29 @@ const Torrent = ({ id, torrent }) => {
     // Update stats once then forever.
     useEffect(() => customSetInterval((async () => {
         const errorInterval = 10000;
-        const liveInterval = 500;
+        const liveInterval = 1000;
-        const finishedInterval = 5000;
+        const finishedInterval = 10000;
+        const nonLiveInterval = 10000;
 
         return API.getTorrentStats(torrent.id).then((stats) => {
             updateStatsResponse(stats);
-            return torrentIsDone(stats) ? finishedInterval : liveInterval;
+            return stats;
+        }).then((stats) => {
+            if (stats.finished) {
+                return finishedInterval;
+            }
+            if (stats.state == STATE_INITIALIZING || stats.state == STATE_LIVE) {
+                return liveInterval;
+            }
+            return nonLiveInterval;
         }, (e) => {
             return errorInterval;
         });
-    }), 0), []);
+    }), 0), [forceStatsRefresh]);
 
-    return <TorrentRow id={id} detailsResponse={detailsResponse} statsResponse={statsResponse} />
+    return <RefreshTorrentStatsContext.Provider value={{ refresh: forceStatsRefreshCallback }}>
+        <TorrentRow id={id} detailsResponse={detailsResponse} statsResponse={statsResponse} />
+    </RefreshTorrentStatsContext.Provider >
 }
 
 const TorrentsList = (props: { torrents: Array<TorrentId>, loading: boolean }) => {
@@ -143,7 +340,7 @@ const Root = () => {
 
     return <AppContext.Provider value={context}>
         <div className='text-center'>
-            <h1 className="mt-3 mb-4">rqbit web 0.0.1-alpha</h1>
+            <h1 className="mt-3 mb-4">rqbit web 4.0.0-beta.0</h1>
             <RootContent
                 closeableError={closeableError}
                 otherError={otherError}
@@ -199,7 +396,7 @@ const UploadButton = ({ buttonText, onClick, data, resetData, variant }) => {
                 const response = await API.uploadTorrent(data, { listOnly: true });
                 setFileList(response.details.files);
             } catch (e) {
-                setFileListError({ text: 'Error listing torrent', details: e });
+                setFileListError({ text: 'Error uploading torrent', details: e });
             } finally {
                 setLoading(false);
             }
@@ -373,10 +570,6 @@ const RootContent = (props: { closeableError: ErrorDetails, otherError: ErrorDet
     </Container>
 };
 
-function torrentIsDone(stats: TorrentStats): boolean {
-    return stats.snapshot.have_bytes == stats.snapshot.total_bytes;
-}
-
 function formatBytes(bytes: number): string {
     if (bytes === 0) return '0 Bytes';
 
@@ -398,11 +591,11 @@ function getLargestFileName(torrentDetails: TorrentDetails): string {
 }
 
 function getCompletionETA(stats: TorrentStats): string {
-    if (stats.time_remaining && stats.time_remaining.duration) {
+    let duration = stats?.live?.time_remaining?.duration?.secs;
-        return formatSecondsToTime(stats.time_remaining.duration.secs);
+    if (duration == null) {
-    } else {
         return 'N/A';
     }
+    return formatSecondsToTime(duration);
 }
 
 function formatSecondsToTime(seconds: number): string {

crates/librqbit/webui/vite.config.ts

@@ -1,5 +1,4 @@
 import { defineConfig } from 'vite';
-import react from '@vitejs/plugin-react'
 
 // https://vitejs.dev/config/
 export default defineConfig({

crates/librqbit_core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "librqbit-core"
-version = "3.0.0"
+version = "3.1.0"
 edition = "2021"
 description = "Important utilities used throughout librqbit useful for working with torrents."
 license = "Apache-2.0"
@@ -17,6 +17,8 @@ sha1-openssl = ["bencode/sha1-openssl"]
 sha1-rust = ["bencode/sha1-rust"]
 
 [dependencies]
+tracing = "0.1.40"
+tokio = "1"
 hex = "0.4"
 anyhow = "1"
 url = "2"

crates/librqbit_core/src/lengths.rs

@@ -1,4 +1,4 @@
-use crate::constants::CHUNK_SIZE;
+use crate::{constants::CHUNK_SIZE, torrent_metainfo::TorrentMetaV1Info};
 
 const fn is_power_of_two(x: u64) -> bool {
     (x != 0) && ((x & (x - 1)) == 0)
@@ -61,6 +61,13 @@ impl ValidPieceIndex {
 }
 
 impl Lengths {
+    pub fn from_torrent<ByteBuf: AsRef<[u8]>>(
+        torrent: &TorrentMetaV1Info<ByteBuf>,
+    ) -> anyhow::Result<Lengths> {
+        let total_length = torrent.iter_file_lengths()?.sum();
+        Lengths::new(total_length, torrent.piece_length, None)
+    }
+
     pub fn new(
         total_length: u64,
         piece_length: u32,

crates/librqbit_core/src/lib.rs

@@ -3,5 +3,6 @@ pub mod id20;
 pub mod lengths;
 pub mod magnet;
 pub mod peer_id;
+pub mod spawn_utils;
 pub mod speed_estimator;
 pub mod torrent_metainfo;

crates/librqbit_core/src/magnet.rs

@@ -41,6 +41,17 @@ impl Magnet {
     }
 }
 
+impl std::fmt::Display for Magnet {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "magnet:?xt=urn:btih:{}&tr={}",
+            self.info_hash.as_string(),
+            self.trackers.join("&tr=")
+        )
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #[test]

crates/librqbit_core/src/spawn_utils.rs

@@ -0,0 +1,20 @@
+use tracing::{debug, error, trace, Instrument};
+
+pub fn spawn(
+    span: tracing::Span,
+    fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
+) -> tokio::task::JoinHandle<()> {
+    let fut = async move {
+        trace!("started");
+        match fut.await {
+            Ok(_) => {
+                debug!("finished");
+            }
+            Err(e) => {
+                error!("finished with error: {:#}", e)
+            }
+        }
+    }
+    .instrument(span);
+    tokio::task::spawn(fut)
+}

crates/peer_binary_protocol/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "librqbit-peer-protocol"
-version = "3.0.0"
+version = "3.1.0"
 edition = "2021"
 description = "Protocol for working with torrent peers. Used in rqbit torrent client."
 license = "Apache-2.0"
@@ -23,6 +23,6 @@ byteorder = "1"
 buffers = {path="../buffers", package="librqbit-buffers", version = "2.2.1"}
 bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
 clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
-librqbit-core = {path="../librqbit_core", version = "3.0.0"}
+librqbit-core = {path="../librqbit_core", version = "3.1.0"}
 bitvec = "1"
 anyhow = "1"

crates/rqbit/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rqbit"
-version = "3.3.0"
+version = "4.0.0-beta.0"
 authors = ["Igor Katson <igor.katson@gmail.com>"]
 edition = "2021"
 description = "A bittorrent command line client and server."
@@ -13,6 +13,7 @@ readme = "README.md"
 
 [features]
 default = ["sha1-system", "default-tls", "webui"]
+tokio-console = ["console-subscriber", "tokio/tracing"]
 webui = ["librqbit/webui"]
 timed_existence = ["librqbit/timed_existence"]
 sha1-system = ["librqbit/sha1-system"]
@@ -22,9 +23,10 @@ default-tls = ["librqbit/default-tls"]
 rust-tls = ["librqbit/rust-tls"]
 
 [dependencies]
-librqbit = {path="../librqbit", default-features=false, version = "3.3.0"}
+librqbit = {path="../librqbit", default-features=false, version = "4.0.0-beta.0"}
-dht = {path="../dht", package="librqbit-dht", version="3.1.0"}
+dht = {path="../dht", package="librqbit-dht", version="3.2.0"}
 tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
+console-subscriber = {version = "0.2", optional = true}
 anyhow = "1"
 clap = {version = "4", features = ["derive", "deprecated"]}
 tracing = "0.1"

crates/rqbit/src/main.rs

@@ -7,14 +7,14 @@ use librqbit::{
     http_api_client,
     peer_connection::PeerConnectionOptions,
     session::{
-        AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, ManagedTorrentState,
+        AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session,
-        Session, SessionOptions,
+        SessionOptions,
     },
     spawn_utils::{spawn, BlockingSpawner},
-    torrent_state::timeit,
+    torrent_state::ManagedTorrentState,
 };
 use size_format::SizeFormatterBinary as SF;
-use tracing::{error, info, span, warn, Level};
+use tracing::{error, error_span, info, trace_span, warn};
 
 #[derive(Debug, Clone, Copy, ValueEnum)]
 enum LogLevel {
@@ -77,6 +77,13 @@ struct Opts {
 struct ServerStartOptions {
     /// The output folder to write to. If not exists, it will be created.
     output_folder: String,
+    #[arg(
+        long = "disable-persistence",
+        help = "Disable server persistence. It will not read or write its state to disk."
+    )]
+    disable_persistence: bool,
+    #[arg(long = "persistence-filename")]
+    persistence_filename: Option<String>,
 }
 
 #[derive(Parser)]
@@ -134,31 +141,84 @@ enum SubCommand {
     Download(DownloadOpts),
 }
 
-fn init_logging(opts: &Opts) {
+// Iint logging and make a channel to send new RUST_LOG values to.
-    if std::env::var_os("RUST_LOG").is_none() {
+fn init_logging(opts: &Opts) -> tokio::sync::mpsc::UnboundedSender<String> {
-        match opts.log_level.as_ref() {
+    let default_rust_log = match opts.log_level.as_ref() {
-            Some(level) => {
+        Some(level) => match level {
-                let level_str = match level {
+            LogLevel::Trace => "trace",
-                    LogLevel::Trace => "trace",
+            LogLevel::Debug => "debug",
-                    LogLevel::Debug => "debug",
+            LogLevel::Info => "info",
-                    LogLevel::Info => "info",
+            LogLevel::Warn => "warn",
-                    LogLevel::Warn => "warn",
+            LogLevel::Error => "error",
-                    LogLevel::Error => "error",
+        },
-                };
+        None => "info",
-                std::env::set_var("RUST_LOG", level_str);
+    };
-            }
+    let stderr_filter = match std::env::var("RUST_LOG").ok() {
-            None => {
+        Some(rust_log) => EnvFilter::builder()
-                std::env::set_var("RUST_LOG", "info");
+            .parse(rust_log)
-            }
+            .expect("can't parse RUST_LOG"),
-        };
+        None => EnvFilter::builder()
-    }
+            .parse(default_rust_log)
+            .expect("can't parse default_rust_log"),
+    };
+
+    let (stderr_filter, reload_stderr_filter) =
+        tracing_subscriber::reload::Layer::new(stderr_filter);
 
     use tracing_subscriber::{fmt, prelude::*, EnvFilter};
 
-    tracing_subscriber::registry()
+    #[cfg(feature = "tokio-console")]
-        .with(fmt::layer())
+    {
-        .with(EnvFilter::from_default_env())
+        let (console_layer, server) = console_subscriber::Builder::default()
-        .init();
+            .with_default_env()
+            .build();
+
+        tracing_subscriber::registry()
+            .with(fmt::layer().with_filter(stderr_filter))
+            .with(console_layer)
+            .init();
+
+        spawn(
+            "console_subscriber server",
+            error_span!("console_subscriber server"),
+            async move {
+                server
+                    .serve()
+                    .await
+                    .map_err(|e| anyhow::anyhow!("{:#?}", e))
+                    .context("error running console subscriber server")
+            },
+        );
+    }
+
+    #[cfg(not(feature = "tokio-console"))]
+    {
+        tracing_subscriber::registry()
+            .with(fmt::layer())
+            .with(stderr_filter)
+            .init();
+    }
+
+    let (reload_tx, mut reload_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
+    spawn(
+        "fmt_filter_reloader",
+        error_span!("fmt_filter_reloader"),
+        async move {
+            while let Some(rust_log) = reload_rx.recv().await {
+                let stderr_env_filter = match EnvFilter::builder().parse(&rust_log) {
+                    Ok(f) => f,
+                    Err(e) => {
+                        eprintln!("can't parse env filter {:?}: {:#?}", rust_log, e);
+                        continue;
+                    }
+                };
+                eprintln!("setting RUST_LOG to {:?}", rust_log);
+                let _ = reload_stderr_filter.reload(stderr_env_filter);
+            }
+            Ok(())
+        },
+    );
+    reload_tx
 }
 
 fn _start_deadlock_detector_thread() {
@@ -188,9 +248,6 @@ fn _start_deadlock_detector_thread() {
 fn main() -> anyhow::Result<()> {
     let opts = Opts::parse();
 
-    init_logging(&opts);
-    // start_deadlock_detector_thread();
-
     let (mut rt_builder, spawner) = match opts.single_thread_runtime {
         true => (
             tokio::runtime::Builder::new_current_thread(),
@@ -223,10 +280,14 @@ fn main() -> anyhow::Result<()> {
 }
 
 async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()> {
-    let sopts = SessionOptions {
+    let logging_reload_tx = init_logging(&opts);
+
+    let mut sopts = SessionOptions {
         disable_dht: opts.disable_dht,
         disable_dht_persistence: opts.disable_dht_persistence,
         dht_config: None,
+        persistence: false,
+        persistence_filename: None,
         peer_id: None,
         peer_opts: Some(PeerConnectionOptions {
             connect_timeout: Some(opts.peer_connect_timeout),
@@ -238,39 +299,49 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
     let stats_printer = |session: Arc<Session>| async move {
         loop {
             session.with_torrents(|torrents| {
-                    for (idx, torrent) in torrents.iter().enumerate() {
+                    for (idx, torrent) in torrents {
-                        match &torrent.state {
+                        let live = torrent.with_state(|s| {
-                            ManagedTorrentState::Initializing => {
+                            match s {
-                                info!("[{}] initializing", idx);
+                                ManagedTorrentState::Initializing(i) => {
-                            },
+                                    let total = torrent.get_total_bytes();
-                            ManagedTorrentState::Running(handle) => {
+                                    let progress = i.get_checked_bytes();
-                                let stats = timeit("stats_snapshot", || handle.torrent_state().stats_snapshot());
+                                    let pct =  (progress as f64 / total as f64) * 100f64;
-                                let speed = handle.speed_estimator();
+                                    info!("[{}] initializing {:.2}%", idx, pct)
-                                let total = stats.total_bytes;
+                                },
-                                let progress = stats.total_bytes - stats.remaining_bytes;
+                                ManagedTorrentState::Live(h) => return Some(h.clone()),
-                                let downloaded_pct = if stats.remaining_bytes == 0 {
+                                _ => {},
-                                    100f64
+                            };
-                                } else {
+                            None
-                                    (progress as f64 / total as f64) * 100f64
+                        });
-                                };
+                        let handle = match live {
-                                info!(
+                            Some(live) => live,
-                                    "[{}]: {:.2}% ({:.2}), down speed {:.2} MiB/s, fetched {}, remaining {:.2} of {:.2}, uploaded {:.2}, peers: {{live: {}, connecting: {}, queued: {}, seen: {}, dead: {}}}",
+                            None => continue
-                                    idx,
+                        };
-                                    downloaded_pct,
+                        let stats = handle.stats_snapshot();
-                                    SF::new(progress),
+                        let speed = handle.speed_estimator();
-                                    speed.download_mbps(),
+                        let total = stats.total_bytes;
-                                    SF::new(stats.fetched_bytes),
+                        let progress = stats.total_bytes - stats.remaining_bytes;
-                                    SF::new(stats.remaining_bytes),
+                        let downloaded_pct = if stats.remaining_bytes == 0 {
-                                    SF::new(total),
+                            100f64
-                                    SF::new(stats.uploaded_bytes),
+                        } else {
-                                    stats.peer_stats.live,
+                            (progress as f64 / total as f64) * 100f64
-                                    stats.peer_stats.connecting,
+                        };
-                                    stats.peer_stats.queued,
+                        info!(
-                                    stats.peer_stats.seen,
+                            "[{}]: {:.2}% ({:.2}), down speed {:.2} MiB/s, fetched {}, remaining {:.2} of {:.2}, uploaded {:.2}, peers: {{live: {}, connecting: {}, queued: {}, seen: {}, dead: {}}}",
-                                    stats.peer_stats.dead,
+                            idx,
-                                );
+                            downloaded_pct,
-                            },
+                            SF::new(progress),
-                        }
+                            speed.download_mbps(),
+                            SF::new(stats.fetched_bytes),
+                            SF::new(stats.remaining_bytes),
+                            SF::new(total),
+                            SF::new(stats.uploaded_bytes),
+                            stats.peer_stats.live,
+                            stats.peer_stats.connecting,
+                            stats.peer_stats.queued,
+                            stats.peer_stats.seen,
+                            stats.peer_stats.dead,
+                        );
                     }
                 });
             tokio::time::sleep(Duration::from_secs(1)).await;
@@ -280,23 +351,25 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
     match &opts.subcommand {
         SubCommand::Server(server_opts) => match &server_opts.subcommand {
             ServerSubcommand::Start(start_opts) => {
-                let session = Arc::new(
+                sopts.persistence = !start_opts.disable_persistence;
-                    Session::new_with_opts(
+                sopts.persistence_filename =
-                        PathBuf::from(&start_opts.output_folder),
+                    start_opts.persistence_filename.clone().map(PathBuf::from);
-                        spawner,
+                let session = Session::new_with_opts(
-                        sopts,
+                    PathBuf::from(&start_opts.output_folder),
-                    )
+                    spawner,
-                    .await
+                    sopts,
-                    .context("error initializing rqbit session")?,
+                )
-                );
+                .await
+                .context("error initializing rqbit session")?;
                 spawn(
-                    span!(Level::TRACE, "stats_printer"),
+                    "stats_printer",
+                    trace_span!("stats_printer"),
                     stats_printer(session.clone()),
                 );
-                let http_api = HttpApi::new(session);
+                let http_api = HttpApi::new(session, Some(logging_reload_tx));
                 let http_api_listen_addr = opts.http_api_listen_addr;
                 http_api
-                    .make_http_api_and_run(http_api_listen_addr)
+                    .make_http_api_and_run(http_api_listen_addr, false)
                     .await
                     .context("error starting HTTP API")
             }
@@ -353,30 +426,32 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
                 }
                 Ok(())
             } else {
-                let session = Arc::new(
+                let session = Session::new_with_opts(
-                    Session::new_with_opts(
+                    download_opts
-                        download_opts
+                        .output_folder
-                            .output_folder
+                        .as_ref()
-                            .as_ref()
+                        .map(PathBuf::from)
-                            .map(PathBuf::from)
+                        .context(
-                            .context(
+                            "output_folder is required if can't connect to an existing server",
-                                "output_folder is required if can't connect to an existing server",
+                        )?,
-                            )?,
+                    spawner,
-                        spawner,
+                    sopts,
-                        sopts,
+                )
-                    )
+                .await
-                    .await
+                .context("error initializing rqbit session")?;
-                    .context("error initializing rqbit session")?,
-                );
                 spawn(
-                    span!(Level::TRACE, "stats_printer"),
+                    "stats_printer",
+                    trace_span!("stats_printer"),
                     stats_printer(session.clone()),
                 );
-                let http_api = HttpApi::new(session.clone());
+                let http_api = HttpApi::new(session.clone(), Some(logging_reload_tx));
                 let http_api_listen_addr = opts.http_api_listen_addr;
                 spawn(
-                    span!(Level::ERROR, "http_api"),
+                    "http_api",
-                    http_api.clone().make_http_api_and_run(http_api_listen_addr),
+                    error_span!("http_api"),
+                    http_api
+                        .clone()
+                        .make_http_api_and_run(http_api_listen_addr, true),
                 );
 
                 let mut added = false;
@@ -392,10 +467,12 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
                         .await
                     {
                         Ok(v) => match v {
-                            AddTorrentResponse::AlreadyManaged(handle) => {
+                            AddTorrentResponse::AlreadyManaged(id, handle) => {
                                 info!(
-                                    "torrent {:?} is already managed, downloaded to {:?}",
+                                    "torrent {:?} is already managed, id={}, downloaded to {:?}",
-                                    handle.info_hash, handle.output_folder
+                                    handle.info_hash(),
+                                    id,
+                                    handle.info().out_dir
                                 );
                                 continue;
                             }
@@ -420,7 +497,7 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
                                 }
                                 continue;
                             }
-                            AddTorrentResponse::Added(handle) => {
+                            AddTorrentResponse::Added(_, handle) => {
                                 added = true;
                                 handle
                             }
@@ -431,7 +508,6 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
                         }
                     };
 
-                    http_api.add_torrent_handle(handle.clone());
                     handles.push(handle);
                 }
 
@@ -441,8 +517,11 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
                     if download_opts.exit_on_finish {
                         let results = futures::future::join_all(
                             handles.iter().map(|h| h.wait_until_completed()),
-                        );
+                        )
-                        results.await;
+                        .await;
+                        if results.iter().any(|r| r.is_err()) {
+                            anyhow::bail!("some downloads failed")
+                        }
                         info!("All downloads completed, exiting");
                         Ok(())
                     } else {