leyoda/rqbit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #37 from ikatson/refactor-split-up-torrent-state

Browse source 
A ton of new features + huge refactor
This commit is contained in:
Igor Katson 2023-11-27 09:31:51 +00:00 • committed by GitHub
commit f1e91d41cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
54 changed files with 3216 additions and 1663 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.cargo/config
View file

@ -1,2 +1,6 @@
[target.arm-unknown-linux-gnueabihf]
rustflags = ["-l", "atomic"]
rustflags = ["-l", "atomic"]
[target.armv7-unknown-linux-gnueabihf]
# Workaround for: https://github.com/rust-lang/compiler-builtins/issues/420
rustflags = ["-C", "link-arg=-Wl,--allow-multiple-definition"]

10
.github/workflows/release.yml vendored
View file

@ -29,7 +29,7 @@ jobs:
- name: install linux cross compiler
run:
brew tap messense/macos-cross-toolchains &&
brew install x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu arm-unknown-linux-gnueabihf
brew install x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu arm-unknown-linux-gnueabihf armv7-unknown-linux-gnueabihf
- name: Make a directory for output artifacts
run:
@ -41,12 +41,18 @@ jobs:
make release-linux-x86_64 &&
mv target/x86_64-unknown-linux-gnu/release-github/rqbit target/artifacts/rqbit-linux-static-x86_64
- name: Build release linux arm32bit binary
- name: Build release linux armv6 binary
run:
rustup target install arm-unknown-linux-gnueabihf &&
make release-linux-armv6 &&
mv target/arm-unknown-linux-gnueabihf/release-github/rqbit target/artifacts/rqbit-linux-static-arm32
- name: Build release linux armv7 binary
run:
rustup target install armv7-unknown-linux-gnueabihf &&
make release-linux-armv7 &&
mv target/armv7-unknown-linux-gnueabihf/release-github/rqbit target/artifacts/rqbit-linux-static-arm32
- name: Build release linux aarch64 binary
run:
rustup target install aarch64-unknown-linux-gnu &&

270
Cargo.lock generated
View file

@ -80,6 +80,28 @@ version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "async-trait"
version = "0.1.74"
@ -317,6 +339,43 @@ dependencies = [
"libc",
]
[[package]]
name = "console-api"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787"
dependencies = [
"futures-core",
"prost",
"prost-types",
"tonic",
"tracing-core",
]
[[package]]
name = "console-subscriber"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e"
dependencies = [
"console-api",
"crossbeam-channel",
"crossbeam-utils",
"futures-task",
"hdrhistogram",
"humantime",
"prost-types",
"serde",
"serde_json",
"thread_local",
"tokio",
"tokio-stream",
"tonic",
"tracing",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -342,6 +401,34 @@ dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -371,7 +458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
dependencies = [
"cfg-if",
"hashbrown",
"hashbrown 0.14.2",
"lock_api",
"once_cell",
"parking_lot_core",
@ -451,6 +538,16 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flate2"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -624,19 +721,38 @@ dependencies = [
"futures-sink",
"futures-util",
"http",
"indexmap",
"indexmap 2.1.0",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
[[package]]
name = "hdrhistogram"
version = "7.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
dependencies = [
"base64",
"byteorder",
"flate2",
"nom",
"num-traits",
]
[[package]]
name = "heck"
version = "0.4.1"
@ -701,6 +817,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.27"
@ -739,6 +861,18 @@ dependencies = [
"tokio-rustls",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
@ -762,6 +896,16 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.1.0"
@ -769,7 +913,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [
"equivalent",
"hashbrown",
"hashbrown 0.14.2",
]
[[package]]
@ -787,6 +931,15 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
[[package]]
name = "itertools"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.12.0"
@ -847,7 +1000,7 @@ dependencies = [
[[package]]
name = "librqbit"
version = "3.3.0"
version = "4.0.0-beta.0"
dependencies = [
"anyhow",
"axum",
@ -860,7 +1013,7 @@ dependencies = [
"futures",
"hex 0.4.3",
"http",
"itertools",
"itertools 0.12.0",
"librqbit-bencode",
"librqbit-buffers",
"librqbit-clone-to-owned",
@ -913,29 +1066,31 @@ version = "2.2.1"
[[package]]
name = "librqbit-core"
version = "3.0.0"
version = "3.1.0"
dependencies = [
"anyhow",
"hex 0.4.3",
"itertools",
"itertools 0.12.0",
"librqbit-bencode",
"librqbit-buffers",
"librqbit-clone-to-owned",
"parking_lot",
"serde",
"tokio",
"tracing",
"url",
"uuid",
]
[[package]]
name = "librqbit-dht"
version = "3.1.0"
version = "3.2.0"
dependencies = [
"anyhow",
"directories",
"futures",
"hex 0.4.3",
"indexmap",
"indexmap 2.1.0",
"leaky-bucket",
"librqbit-bencode",
"librqbit-clone-to-owned",
@ -952,7 +1107,7 @@ dependencies = [
[[package]]
name = "librqbit-peer-protocol"
version = "3.0.0"
version = "3.1.0"
dependencies = [
"anyhow",
"bincode",
@ -1023,6 +1178,12 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.7.1"
@ -1061,6 +1222,16 @@ dependencies = [
"tempfile",
]
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@ -1279,7 +1450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
dependencies = [
"fixedbitset",
"indexmap",
"indexmap 2.1.0",
]
[[package]]
@ -1335,6 +1506,38 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
dependencies = [
"anyhow",
"itertools 0.11.0",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-types"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
dependencies = [
"prost",
]
[[package]]
name = "quote"
version = "1.0.33"
@ -1503,10 +1706,11 @@ dependencies = [
[[package]]
name = "rqbit"
version = "3.3.0"
version = "4.0.0-beta.0"
dependencies = [
"anyhow",
"clap",
"console-subscriber",
"futures",
"librqbit",
"librqbit-dht",
@ -1888,9 +2092,20 @@ dependencies = [
"pin-project-lite",
"socket2 0.5.5",
"tokio-macros",
"tracing",
"windows-sys",
]
[[package]]
name = "tokio-io-timeout"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-macros"
version = "2.2.0"
@ -1948,6 +2163,33 @@ dependencies = [
"tracing",
]
[[package]]
name = "tonic"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"
dependencies = [
"async-stream",
"async-trait",
"axum",
"base64",
"bytes",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower"
version = "0.4.13"
@ -1956,9 +2198,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand",
"slab",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",

12
Makefile
View file

@ -75,7 +75,7 @@ release-linux-current-target: target/openssl-linux/$(TARGET)/lib/libssl.a
cargo build --profile release-github --target=$(TARGET)
@PHONY: release-linux
release-linux: release-linux-x86_64 release-linux-aarch64 release-linux-armv6
release-linux: release-linux-x86_64 release-linux-aarch64 release-linux-armv6 release-linux-armv7
@PHONY: release-linux-x86_64
release-linux-x86_64:
@ -105,6 +105,16 @@ release-linux-armv6:
LDFLAGS=-latomic \
$(MAKE) release-linux-current-target
# armv7-unknown-linux-gnueabihf
@PHONY: release-linux-armv7
release-linux-armv7:
TARGET=armv7-unknown-linux-gnueabihf \
TARGET_SNAKE_CASE=armv7_unknown_linux_gnueabihf \
TARGET_SNAKE_UPPER_CASE=ARMV7_UNKNOWN_LINUX_GNUEABIHF \
CROSS_COMPILE_PREFIX=armv7-linux-gnueabihf \
OPENSSL_CONFIGURE_NAME=linux-generic32 \
$(MAKE) release-linux-current-target
@PHONY: release-all
release-all: release-windows release-linux release-macos-universal

32
TODO.md
View file

@ -5,17 +5,31 @@
- [x] tracing instead of logging. Debugging peers: RUST_LOG=[{peer=.*}]=debug
test-log for tests
- [x] reopen read only is bugged
- [ ] initializing/checking
- [ ] blocks the whole process. Need to break it up. On slower devices (rpi) just hangs for a good while
- [ ] checking torrents should be visible right away
- [x] initializing/checking
- [x] blocks the whole process. Need to break it up. On slower devices (rpi) just hangs for a good while
- [x] checking torrents should be visible right away
- [ ] server persistence
- [ ] it would be nice to restart the server and keep the state
- [ ] torrent actions
- [ ] pause/unpause
- [ ] remove including from disk
- [x] torrent actions
- [x] pause/unpause
- [x] remove including from disk
- [ ] DHT
- [ ] for torrents with a few seeds might be cool to re-query DHT once in a while
- [ ] it's sending many requests now way too fast, locks up Mac OS UI annoyingly
- [ ] for torrents with a few seeds might be cool to re-query DHT once in a while.
- [x] it's sending many requests now way too fast, locks up Mac OS UI annoyingly
someday:
- [ ] cancellation from the client-side for the lib (i.e. stop the torrent manager)
- [x] cancellation from the client-side for the lib (i.e. stop the torrent manager)
refactor:
- [x] where are peers stored
- [x] http api pause/unpause etc
- [x] when a live torrent fails writing to disk, it should transition to error state
- [x] something is wrong when unpausing - can't finish. Recalculate needed/have from chunk tracker.
- [x] silence this: WARN torrent{id=0}:external_peer_adder: librqbit::spawn_utils: finished with error: no longer live
- [x] start from error state should be possible from UI
- [ ] if the torrent was completed, not need to re-check it
- [x] checking is very slow on raspberry
checked. nothing much can be done here. Even if raspberry's own libssl.so is used it's still super slow (sha1)
- [ ] .rqbit-session.json file has 0 bytes when disk full. I guess fs::rename does this when disk is full? at least on linux

4
crates/dht/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "librqbit-dht"
version = "3.1.0"
version = "3.2.0"
edition = "2021"
description = "DHT implementation, used in rqbit torrent client."
license = "Apache-2.0"
@ -33,7 +33,7 @@ indexmap = "2"
directories = "5"
clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
librqbit-core = {path="../librqbit_core", version = "3.0.0"}
librqbit-core = {path="../librqbit_core", version = "3.1.0"}
[dev-dependencies]
tracing-subscriber = "0.3"

2
crates/dht/examples/dht.rs
View file

@ -17,7 +17,7 @@ async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt::init();
let dht = Dht::new().await.context("error initializing DHT")?;
let mut stream = dht.get_peers(info_hash).await?;
let mut stream = dht.get_peers(info_hash)?;
let stats_printer = async {
loop {

56
crates/dht/src/dht.rs
View file

@ -18,7 +18,7 @@ use bencode::ByteString;
use futures::{stream::FuturesUnordered, Stream, StreamExt};
use indexmap::IndexSet;
use leaky_bucket::RateLimiter;
use librqbit_core::{id20::Id20, peer_id::generate_peer_id};
use librqbit_core::{id20::Id20, peer_id::generate_peer_id, spawn_utils::spawn};
use parking_lot::RwLock;
use rand::Rng;
use serde::Serialize;
@ -27,7 +27,7 @@ use tokio::{
sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
};
use tokio_stream::wrappers::{errors::BroadcastStreamRecvError, BroadcastStream};
use tracing::{debug, info, trace, warn};
use tracing::{debug, debug_span, error_span, info, trace, warn, Instrument};
#[derive(Debug, Serialize)]
pub struct DhtStats {
@ -513,7 +513,7 @@ impl DhtWorker {
bootstrap_addrs: &[String],
) -> anyhow::Result<()> {
let (out_tx, mut out_rx) = channel(1);
let framer = run_framer(&self.socket, in_rx, out_tx);
let framer = run_framer(&self.socket, in_rx, out_tx).instrument(debug_span!("dht_framer"));
let bootstrap = async {
let mut futs = FuturesUnordered::new();
@ -521,34 +521,40 @@ impl DhtWorker {
for addr in bootstrap_addrs.iter() {
let this = &self;
let in_tx = &in_tx;
futs.push(async move {
match tokio::net::lookup_host(addr).await {
Ok(addrs) => {
for addr in addrs {
let request = this
.state
.write()
.create_request(Request::FindNode(this.peer_id), addr);
in_tx.send((request, addr))?;
futs.push(
async move {
match tokio::net::lookup_host(addr).await {
Ok(addrs) => {
for addr in addrs {
let request = this
.state
.write()
.create_request(Request::FindNode(this.peer_id), addr);
in_tx.send((request, addr))?;
}
}
Err(e) => {
warn!("error looking up {}: {}", addr, e);
return Err(e.into());
}
}
Err(e) => warn!("error looking up {}: {}", addr, e),
Ok::<_, anyhow::Error>(())
}
Ok::<_, anyhow::Error>(())
});
.instrument(error_span!("dht_bootstrap", addr = addr)),
);
}
let mut successes = 0;
while let Some(resp) = futs.next().await {
match resp {
Ok(_) => successes += 1,
Err(e) => warn!("error in one of the bootstrappers: {}", e),
if resp.is_ok() {
successes += 1
}
}
if successes == 0 {
anyhow::bail!("bootstrapping did not succeed")
}
Ok(())
};
}
.instrument(debug_span!("dht_bootstrapper"));
let mut bootstrap_done = false;
let response_reader = {
@ -563,7 +569,8 @@ impl DhtWorker {
"closed response reader, nowhere to send results to, DHT closed"
))
}
};
}
.instrument(debug_span!("dht_responese_reader"));
tokio::pin!(framer);
tokio::pin!(bootstrap);
@ -676,7 +683,7 @@ impl Dht {
listen_addr,
)));
tokio::spawn({
spawn(error_span!("dht"), {
let state = state.clone();
async move {
let worker = DhtWorker {
@ -684,13 +691,14 @@ impl Dht {
peer_id,
state,
};
let result = worker.start(in_tx, in_rx, &bootstrap_addrs).await;
warn!("DHT worker finished with {:?}", result);
worker.start(in_tx, in_rx, &bootstrap_addrs).await?;
Ok(())
}
});
Ok(Dht { state })
}
pub async fn get_peers(
pub fn get_peers(
&self,
info_hash: Id20,
) -> anyhow::Result<impl Stream<Item = SocketAddr> + Unpin> {

6
crates/dht/src/persistence.rs
View file

@ -1,5 +1,6 @@
// TODO: this now stores only the routing table, but we also need AT LEAST the same socket address...
use librqbit_core::spawn_utils::spawn;
use serde::{Deserialize, Serialize};
use std::fs::OpenOptions;
use std::io::{BufReader, BufWriter};
@ -8,8 +9,7 @@ use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::Context;
use tokio::spawn;
use tracing::{debug, error, info, trace, warn};
use tracing::{debug, error, error_span, info, trace, warn};
use crate::dht::{Dht, DhtConfig};
use crate::routing_table::RoutingTable;
@ -110,7 +110,7 @@ impl PersistentDht {
};
let dht = Dht::with_config(dht_config).await?;
spawn({
spawn(error_span!("dht_persistence"), {
let dht = dht.clone();
let dump_interval = config
.dump_interval

10
crates/librqbit/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "librqbit"
version = "3.3.0"
version = "4.0.0-beta.0"
authors = ["Igor Katson <igor.katson@gmail.com>"]
edition = "2021"
description = "The main library used by rqbit torrent client. The binary is just a small wrapper on top of it."
@ -24,11 +24,11 @@ rust-tls = ["reqwest/rustls-tls"]
[dependencies]
bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
buffers = {path = "../buffers", package="librqbit-buffers", version = "2.2.1"}
librqbit-core = {path = "../librqbit_core", version = "3.0.0"}
librqbit-core = {path = "../librqbit_core", version = "3.1.0"}
clone_to_owned = {path = "../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.0.0"}
peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.1.0"}
sha1w = {path = "../sha1w", default-features=false, package="librqbit-sha1-wrapper", version="2.2.1"}
dht = {path = "../dht", package="librqbit-dht", version="3.1.0"}
dht = {path = "../dht", package="librqbit-dht", version="3.2.0"}
tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
axum = {version = "0.6"}
@ -64,4 +64,4 @@ dashmap = "5.5.3"
[dev-dependencies]
futures = {version = "0.3"}
tracing-subscriber = "0.3"
tracing-subscriber = "0.3"

20
crates/librqbit/examples/ubuntu.rs
View file

@ -16,6 +16,10 @@ const MAGNET_LINK: &str = "magnet:?xt=urn:btih:cab507494d02ebb1178b38f2e9d7be299
#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
// Output logs to console.
match std::env::var("RUST_LOG") {
Ok(_) => {}
Err(_) => std::env::set_var("RUST_LOG", "info"),
}
tracing_subscriber::fmt::init();
let output_dir = std::env::args()
@ -32,31 +36,29 @@ async fn main() -> Result<(), anyhow::Error> {
.add_torrent(
AddTorrent::from_url(MAGNET_LINK),
Some(AddTorrentOptions {
// Set this to true to allow writing on top of existing files.
// If the file is partially downloaded, librqbit will only download the
// missing pieces.
//
// Otherwise it will throw an error that the file exists.
overwrite: false,
// Allow writing on top of existing files.
overwrite: true,
..Default::default()
}),
)
.await
.context("error adding torrent")?
{
AddTorrentResponse::Added(handle) => handle,
AddTorrentResponse::Added(_, handle) => handle,
// For a brand new session other variants won't happen.
_ => unreachable!(),
};
info!("Details: {:?}", &handle.info().info);
// Print stats periodically.
tokio::spawn({
let handle = handle.clone();
async move {
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let stats = handle.torrent_state().stats_snapshot();
info!("stats: {stats:?}");
let stats = handle.stats();
info!("{stats:}");
}
}
});

33
crates/librqbit/src/chunk_tracker.rs
View file

@ -1,6 +1,6 @@
use librqbit_core::lengths::{ChunkInfo, Lengths, ValidPieceIndex};
use peer_binary_protocol::Piece;
use tracing::{debug, info};
use tracing::{debug, trace};
use crate::type_aliases::BF;
@ -75,9 +75,11 @@ impl ChunkTracker {
priority_piece_ids,
}
}
pub fn get_needed_pieces(&self) -> &BF {
&self.needed_pieces
pub fn get_lengths(&self) -> &Lengths {
&self.lengths
}
pub fn get_have_pieces(&self) -> &BF {
&self.have
}
@ -85,6 +87,16 @@ impl ChunkTracker {
self.needed_pieces.set(index.get() as usize, false)
}
pub fn calc_have_bytes(&self) -> u64 {
self.have
.iter_ones()
.filter_map(|piece_id| {
let piece_id = self.lengths.validate_piece_index(piece_id as u32)?;
Some(self.lengths.piece_length(piece_id) as u64)
})
.sum()
}
pub fn iter_needed_pieces(&self) -> impl Iterator<Item = usize> + '_ {
self.priority_piece_ids
.iter()
@ -117,7 +129,7 @@ impl ChunkTracker {
}
pub fn mark_piece_broken(&mut self, index: ValidPieceIndex) -> bool {
info!("remarking piece={} as broken", index);
debug!("remarking piece={} as broken", index);
self.needed_pieces.set(index.get() as usize, true);
self.chunk_status
.get_mut(self.lengths.chunk_range(index))
@ -132,13 +144,6 @@ impl ChunkTracker {
self.have.set(idx.get() as usize, true);
}
pub fn is_chunk_downloaded(&self, chunk: &ChunkInfo) -> bool {
*self
.chunk_status
.get(chunk.absolute_index as usize)
.unwrap()
}
pub fn is_chunk_ready_to_upload(&self, chunk: &ChunkInfo) -> bool {
self.have
.get(chunk.piece_index.get() as usize)
@ -165,9 +170,11 @@ impl ChunkTracker {
return Some(ChunkMarkingResult::PreviouslyCompleted);
}
chunk_range.set(chunk_info.chunk_index as usize, true);
debug!(
trace!(
"piece={}, chunk_info={:?}, bits={:?}",
piece.index, chunk_info, chunk_range,
piece.index,
chunk_info,
chunk_range,
);
if chunk_range.all() {

2
crates/librqbit/src/dht_utils.rs
View file

@ -107,7 +107,7 @@ mod tests {
let info_hash = Id20::from_str("cf3ea75e2ebbd30e0da6e6e215e2226bf35f2e33").unwrap();
let dht = Dht::new().await.unwrap();
let peer_rx = dht.get_peers(info_hash).await.unwrap();
let peer_rx = dht.get_peers(info_hash).unwrap();
let peer_id = generate_peer_id();
match read_metainfo_from_peer_receiver(peer_id, info_hash, peer_rx, None).await {
ReadMetainfoResult::Found { info, .. } => dbg!(info),

13
crates/librqbit/src/file_ops.rs
View file

@ -2,7 +2,10 @@ use std::{
fs::File,
io::{Read, Seek, SeekFrom, Write},
marker::PhantomData,
sync::Arc,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
};
use anyhow::Context;
@ -11,14 +14,14 @@ use librqbit_core::{
lengths::{ChunkInfo, Lengths, ValidPieceIndex},
torrent_metainfo::{FileIteratorName, TorrentMetaV1Info},
};
use tracing::{debug, trace, warn};
use parking_lot::Mutex;
use peer_binary_protocol::Piece;
use sha1w::ISha1;
use tracing::{debug, trace, warn};
use crate::type_aliases::{PeerHandle, BF};
pub struct InitialCheckResults {
pub(crate) struct InitialCheckResults {
pub needed_pieces: BF,
pub have_pieces: BF,
pub have_bytes: u64,
@ -43,7 +46,7 @@ pub fn update_hash_from_file<Sha1: ISha1>(
Ok(())
}
pub struct FileOps<'a, Sha1> {
pub(crate) struct FileOps<'a, Sha1> {
torrent: &'a TorrentMetaV1Info<ByteString>,
files: &'a [Arc<Mutex<File>>],
lengths: &'a Lengths,
@ -67,6 +70,7 @@ impl<'a, Sha1Impl: ISha1> FileOps<'a, Sha1Impl> {
pub fn initial_check(
&self,
only_files: Option<&[usize]>,
progress: &AtomicU64,
) -> anyhow::Result<InitialCheckResults> {
let mut needed_pieces = BF::from_vec(vec![0u8; self.lengths.piece_bitfield_bytes()]);
let mut have_pieces = BF::from_vec(vec![0u8; self.lengths.piece_bitfield_bytes()]);
@ -125,6 +129,7 @@ impl<'a, Sha1Impl: ISha1> FileOps<'a, Sha1Impl> {
let mut piece_remaining = piece_info.len as usize;
let mut some_files_broken = false;
let mut at_least_one_file_required = current_file.full_file_required;
progress.fetch_add(piece_info.len as u64, Ordering::Relaxed);
while piece_remaining > 0 {
let mut to_read_in_file =

304
crates/librqbit/src/http_api.rs
View file

@ -2,29 +2,28 @@ use anyhow::Context;
use axum::body::Bytes;
use axum::extract::{Path, Query, State};
use axum::response::IntoResponse;
use axum::routing::get;
use axum::routing::{get, post};
use buffers::ByteString;
use dht::{Dht, DhtStats};
use dht::DhtStats;
use http::StatusCode;
use itertools::Itertools;
use librqbit_core::id20::Id20;
use librqbit_core::torrent_metainfo::TorrentMetaV1Info;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::mpsc::UnboundedSender;
use tracing::{info, warn};
use axum::Router;
use crate::http_api_error::{ApiError, ApiErrorExt};
use crate::peer_state::PeerStatsFilter;
use crate::session::{
AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session,
AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session, TorrentId,
};
use crate::torrent_manager::TorrentManagerHandle;
use crate::torrent_state::StatsSnapshot;
use crate::torrent_state::peer::stats::snapshot::{PeerStatsFilter, PeerStatsSnapshot};
use crate::torrent_state::stats::{LiveStats, TorrentStats};
use crate::torrent_state::ManagedTorrentHandle;
// Public API
#[derive(Clone)]
@ -33,16 +32,17 @@ pub struct HttpApi {
}
impl HttpApi {
pub fn new(session: Arc<Session>) -> Self {
pub fn new(session: Arc<Session>, rust_log_reload_tx: Option<UnboundedSender<String>>) -> Self {
Self {
inner: Arc::new(ApiInternal::new(session)),
inner: Arc::new(ApiInternal::new(session, rust_log_reload_tx)),
}
}
pub fn add_torrent_handle(&self, handle: TorrentManagerHandle) -> usize {
self.inner.add_torrent_handle(handle)
}
pub async fn make_http_api_and_run(self, addr: SocketAddr) -> anyhow::Result<()> {
pub async fn make_http_api_and_run(
self,
addr: SocketAddr,
read_only: bool,
) -> anyhow::Result<()> {
let state = self.inner;
async fn api_root() -> impl IntoResponse {
@ -54,11 +54,14 @@ impl HttpApi {
"GET /torrents": "List torrents (default torrent is 0)",
"GET /torrents/{index}": "Torrent details",
"GET /torrents/{index}/haves": "The bitfield of have pieces",
"GET /torrents/{index}/stats": "Torrent stats",
"GET /torrents/{index}/stats/v1": "Torrent stats",
"GET /torrents/{index}/peer_stats": "Per peer stats",
// This is kind of not secure as it just reads any local file that it has access to,
// or any URL, but whatever, ok for our purposes / threat model.
"POST /torrents/{index}/pause": "Pause torrent",
"POST /torrents/{index}/start": "Resume torrent",
"POST /torrents/{index}/forget": "Forget about the torrent, keep the files",
"POST /torrents/{index}/delete": "Forget about the torrent, remove the files",
"POST /torrents": "Add a torrent here. magnet: or http:// or a local file.",
"POST /rust_log": "Set RUST_LOG to this post launch (for debugging)",
"GET /web/": "Web UI",
},
"server": "rqbit",
@ -104,11 +107,18 @@ impl HttpApi {
state.api_dump_haves(idx)
}
async fn torrent_stats(
async fn torrent_stats_v0(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_stats(idx).map(axum::Json)
state.api_stats_v0(idx).map(axum::Json)
}
async fn torrent_stats_v1(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_stats_v1(idx).map(axum::Json)
}
async fn peer_stats(
@ -119,17 +129,62 @@ impl HttpApi {
state.api_peer_stats(idx, filter).map(axum::Json)
}
#[allow(unused_mut)]
async fn torrent_action_pause(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_torrent_action_pause(idx).map(axum::Json)
}
async fn torrent_action_start(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_torrent_action_start(idx).map(axum::Json)
}
async fn torrent_action_forget(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_torrent_action_forget(idx).map(axum::Json)
}
async fn torrent_action_delete(
State(state): State<ApiState>,
Path(idx): Path<usize>,
) -> Result<impl IntoResponse> {
state.api_torrent_action_delete(idx).map(axum::Json)
}
async fn set_rust_log(
State(state): State<ApiState>,
new_value: String,
) -> Result<impl IntoResponse> {
state.api_set_rust_log(new_value).map(axum::Json)
}
let mut app = Router::new()
.route("/", get(api_root))
.route("/rust_log", post(set_rust_log))
.route("/dht/stats", get(dht_stats))
.route("/dht/table", get(dht_table))
.route("/torrents", get(torrents_list).post(torrents_post))
.route("/torrents", get(torrents_list))
.route("/torrents/:id", get(torrent_details))
.route("/torrents/:id/haves", get(torrent_haves))
.route("/torrents/:id/stats", get(torrent_stats))
.route("/torrents/:id/stats", get(torrent_stats_v0))
.route("/torrents/:id/stats/v1", get(torrent_stats_v1))
.route("/torrents/:id/peer_stats", get(peer_stats));
if !read_only {
app = app
.route("/torrents", post(torrents_post))
.route("/torrents/:id/pause", post(torrent_action_pause))
.route("/torrents/:id/start", post(torrent_action_start))
.route("/torrents/:id/forget", post(torrent_action_forget))
.route("/torrents/:id/delete", post(torrent_action_delete));
}
#[cfg(feature = "webui")]
{
let webui_router = Router::new()
@ -184,27 +239,6 @@ impl HttpApi {
type Result<T> = std::result::Result<T, ApiError>;
#[derive(Serialize)]
struct Speed {
mbps: f64,
human_readable: String,
}
impl Speed {
fn new(mbps: f64) -> Self {
Self {
mbps,
human_readable: format!("{mbps:.2} MiB/s"),
}
}
}
impl From<f64> for Speed {
fn from(mbps: f64) -> Self {
Self::new(mbps)
}
}
#[derive(Serialize)]
struct TorrentListResponseItem {
id: usize,
@ -223,41 +257,15 @@ pub struct TorrentDetailsResponseFile {
pub included: bool,
}
#[derive(Default, Serialize)]
struct EmptyJsonResponse {}
#[derive(Serialize, Deserialize)]
pub struct TorrentDetailsResponse {
pub info_hash: String,
pub files: Vec<TorrentDetailsResponseFile>,
}
struct DurationWithHumanReadable(Duration);
impl Serialize for DurationWithHumanReadable {
fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
struct Tmp {
duration: Duration,
human_readable: String,
}
Tmp {
duration: self.0,
human_readable: format!("{:?}", self.0),
}
.serialize(serializer)
}
}
#[derive(Serialize)]
struct StatsResponse {
snapshot: StatsSnapshot,
average_piece_download_time: Option<Duration>,
download_speed: Speed,
all_time_download_speed: Speed,
time_remaining: Option<DurationWithHumanReadable>,
}
#[derive(Serialize, Deserialize)]
pub struct ApiAddTorrentResponse {
pub id: Option<usize>,
@ -330,69 +338,94 @@ impl TorrentAddQueryParams {
}
// Private HTTP API internals. Agnostic of web framework.
pub struct ApiInternal {
dht: Option<Dht>,
startup_time: Instant,
torrent_managers: RwLock<Vec<TorrentManagerHandle>>,
struct ApiInternal {
session: Arc<Session>,
rust_log_reload_tx: Option<UnboundedSender<String>>,
}
type ApiState = Arc<ApiInternal>;
impl ApiInternal {
pub fn new(session: Arc<Session>) -> Self {
pub fn new(session: Arc<Session>, rust_log_reload_tx: Option<UnboundedSender<String>>) -> Self {
Self {
dht: session.get_dht(),
startup_time: Instant::now(),
torrent_managers: RwLock::new(Vec::new()),
session,
rust_log_reload_tx,
}
}
fn add_torrent_handle(&self, handle: TorrentManagerHandle) -> usize {
let mut g = self.torrent_managers.write();
let idx = g.len();
g.push(handle);
idx
}
fn mgr_handle(&self, idx: usize) -> Result<TorrentManagerHandle> {
self.torrent_managers
.read()
fn mgr_handle(&self, idx: TorrentId) -> Result<ManagedTorrentHandle> {
self.session
.get(idx)
.cloned()
.ok_or(ApiError::torrent_not_found(idx))
}
fn api_torrent_list(&self) -> TorrentListResponse {
TorrentListResponse {
torrents: self
.torrent_managers
.read()
.iter()
.enumerate()
let items = self.session.with_torrents(|torrents| {
torrents
.map(|(id, mgr)| TorrentListResponseItem {
id,
info_hash: mgr.torrent_state().info_hash().as_string(),
info_hash: mgr.info().info_hash.as_string(),
})
.collect(),
}
.collect()
});
TorrentListResponse { torrents: items }
}
fn api_torrent_details(&self, idx: usize) -> Result<TorrentDetailsResponse> {
fn api_torrent_details(&self, idx: TorrentId) -> Result<TorrentDetailsResponse> {
let handle = self.mgr_handle(idx)?;
let info_hash = handle.torrent_state().info_hash();
let info_hash = handle.info().info_hash;
let only_files = handle.only_files();
make_torrent_details(&info_hash, handle.torrent_state().info(), only_files)
make_torrent_details(&info_hash, &handle.info().info, only_files.as_deref())
}
fn api_peer_stats(
&self,
idx: usize,
filter: PeerStatsFilter,
) -> Result<crate::peer_state::PeerStatsSnapshot> {
fn api_peer_stats(&self, idx: TorrentId, filter: PeerStatsFilter) -> Result<PeerStatsSnapshot> {
let handle = self.mgr_handle(idx)?;
Ok(handle.torrent_state().per_peer_stats_snapshot(filter))
Ok(handle
.live()
.context("not live")?
.per_peer_stats_snapshot(filter))
}
fn api_torrent_action_pause(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
let handle = self.mgr_handle(idx)?;
handle
.pause()
.context("error pausing torrent")
.with_error_status_code(StatusCode::BAD_REQUEST)?;
Ok(Default::default())
}
fn api_torrent_action_start(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
let handle = self.mgr_handle(idx)?;
self.session
.unpause(&handle)
.context("error unpausing torrent")
.with_error_status_code(StatusCode::BAD_REQUEST)?;
Ok(Default::default())
}
fn api_torrent_action_forget(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
self.session
.delete(idx, false)
.context("error forgetting torrent")?;
Ok(Default::default())
}
fn api_torrent_action_delete(&self, idx: TorrentId) -> Result<EmptyJsonResponse> {
self.session
.delete(idx, true)
.context("error deleting torrent with files")?;
Ok(Default::default())
}
fn api_set_rust_log(&self, new_value: String) -> Result<EmptyJsonResponse> {
let tx = self
.rust_log_reload_tx
.as_ref()
.context("rust_log_reload_tx was not set")?;
tx.send(new_value)
.context("noone is listening to RUST_LOG changes")?;
Ok(Default::default())
}
pub async fn api_add_torrent(
@ -407,11 +440,12 @@ impl ApiInternal {
.context("error adding torrent")
.with_error_status_code(StatusCode::BAD_REQUEST)?
{
AddTorrentResponse::AlreadyManaged(managed) => {
AddTorrentResponse::AlreadyManaged(id, managed) => {
return Err(anyhow::anyhow!(
"{:?} is already managed, downloaded to {:?}",
managed.info_hash,
managed.output_folder
"{:?} is already managed, id={}, downloaded to {:?}",
managed.info_hash(),
id,
&managed.info().out_dir
))
.with_error_status_code(StatusCode::CONFLICT);
}
@ -424,14 +458,13 @@ impl ApiInternal {
details: make_torrent_details(&info_hash, &info, only_files.as_deref())
.context("error making torrent details")?,
},
AddTorrentResponse::Added(handle) => {
AddTorrentResponse::Added(id, handle) => {
let details = make_torrent_details(
&handle.torrent_state().info_hash(),
handle.torrent_state().info(),
handle.only_files(),
&handle.info_hash(),
&handle.info().info,
handle.only_files().as_deref(),
)
.context("error making torrent details")?;
let id = self.add_torrent_handle(handle);
ApiAddTorrentResponse {
id: Some(id),
details,
@ -442,45 +475,32 @@ impl ApiInternal {
}
fn api_dht_stats(&self) -> Result<DhtStats> {
self.dht
self.session
.get_dht()
.as_ref()
.map(|d| d.stats())
.ok_or(ApiError::dht_disabled())
}
fn api_dht_table(&self) -> Result<impl Serialize> {
let dht = self.dht.as_ref().ok_or(ApiError::dht_disabled())?;
let dht = self.session.get_dht().ok_or(ApiError::dht_disabled())?;
Ok(dht.with_routing_table(|r| r.clone()))
}
fn api_stats(&self, idx: usize) -> Result<StatsResponse> {
fn api_stats_v0(&self, idx: TorrentId) -> Result<LiveStats> {
let mgr = self.mgr_handle(idx)?;
let snapshot = mgr.torrent_state().stats_snapshot();
let estimator = mgr.speed_estimator();
let live = mgr.live().context("torrent not live")?;
Ok(LiveStats::from(&*live))
}
// Poor mans download speed computation
let elapsed = self.startup_time.elapsed();
let downloaded_bytes = snapshot.downloaded_and_checked_bytes;
let downloaded_mb = downloaded_bytes as f64 / 1024f64 / 1024f64;
Ok(StatsResponse {
average_piece_download_time: snapshot.average_piece_download_time(),
snapshot,
all_time_download_speed: (downloaded_mb / elapsed.as_secs_f64()).into(),
download_speed: estimator.download_mbps().into(),
time_remaining: estimator.time_remaining().map(DurationWithHumanReadable),
})
fn api_stats_v1(&self, idx: TorrentId) -> Result<TorrentStats> {
let mgr = self.mgr_handle(idx)?;
Ok(mgr.stats())
}
fn api_dump_haves(&self, idx: usize) -> Result<String> {
let mgr = self.mgr_handle(idx)?;
Ok(format!(
"{:?}",
mgr.torrent_state()
.lock_read("api_dump_haves")
.chunks
.get_have_pieces(),
))
Ok(mgr.with_chunk_tracker(|chunks| format!("{:?}", chunks.get_have_pieces()))?)
}
}

9
crates/librqbit/src/http_api_error.rs
View file

@ -19,6 +19,15 @@ impl ApiError {
}
}
#[allow(dead_code)]
pub fn not_implemented(msg: &str) -> Self {
Self {
status: Some(StatusCode::INTERNAL_SERVER_ERROR),
kind: ApiErrorKind::Other(anyhow::anyhow!("{}", msg)),
plaintext: false,
}
}
pub const fn dht_disabled() -> Self {
Self {
status: Some(StatusCode::NOT_FOUND),

2
crates/librqbit/src/lib.rs
View file

@ -6,10 +6,8 @@ pub mod http_api_client;
mod http_api_error;
pub mod peer_connection;
pub mod peer_info_reader;
pub mod peer_state;
pub mod session;
pub mod spawn_utils;
pub mod torrent_manager;
pub mod torrent_state;
pub mod tracker_comms;
pub mod type_aliases;

16
crates/librqbit/src/peer_connection.rs
View file

@ -20,7 +20,7 @@ use crate::spawn_utils::BlockingSpawner;
pub trait PeerConnectionHandler {
fn on_connected(&self, _connection_time: Duration) {}
fn get_have_bytes(&self) -> u64;
fn serialize_bitfield_message_to_buf(&self, buf: &mut Vec<u8>) -> Option<usize>;
fn serialize_bitfield_message_to_buf(&self, buf: &mut Vec<u8>) -> anyhow::Result<usize>;
fn on_handshake(&self, handshake: Handshake) -> anyhow::Result<()>;
fn on_extended_handshake(
&self,
@ -204,15 +204,13 @@ impl<H: PeerConnectionHandler> PeerConnection<H> {
.unwrap_or_else(|| Duration::from_secs(120));
if self.handler.get_have_bytes() > 0 {
if let Some(len) = self
let len = self
.handler
.serialize_bitfield_message_to_buf(&mut write_buf)
{
with_timeout(rwtimeout, write_half.write_all(&write_buf[..len]))
.await
.context("error writing bitfield to peer")?;
debug!("sent bitfield");
}
.serialize_bitfield_message_to_buf(&mut write_buf)?;
with_timeout(rwtimeout, write_half.write_all(&write_buf[..len]))
.await
.context("error writing bitfield to peer")?;
debug!("sent bitfield");
}
loop {

4
crates/librqbit/src/peer_info_reader/mod.rs
View file

@ -141,8 +141,8 @@ impl PeerConnectionHandler for Handler {
0
}
fn serialize_bitfield_message_to_buf(&self, _buf: &mut Vec<u8>) -> Option<usize> {
None
fn serialize_bitfield_message_to_buf(&self, _buf: &mut Vec<u8>) -> anyhow::Result<usize> {
Ok(0)
}
fn on_handshake(&self, handshake: Handshake) -> anyhow::Result<()> {

349
crates/librqbit/src/peer_state.rs
View file

@ -1,349 +0,0 @@
use std::collections::HashSet;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
use librqbit_core::id20::Id20;
use librqbit_core::lengths::{ChunkInfo, ValidPieceIndex};
use serde::Serialize;
use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
use crate::peer_connection::WriterRequest;
use crate::type_aliases::BF;
#[derive(Debug, Hash, PartialEq, Eq)]
pub struct InflightRequest {
pub piece: ValidPieceIndex,
pub chunk: u32,
}
impl From<&ChunkInfo> for InflightRequest {
fn from(c: &ChunkInfo) -> Self {
Self {
piece: c.piece_index,
chunk: c.chunk_index,
}
}
}
// TODO: Arc can be removed probably, as UnboundedSender should be clone + it can be downgraded to weak.
pub type PeerRx = UnboundedReceiver<WriterRequest>;
pub type PeerTx = UnboundedSender<WriterRequest>;
pub trait SendMany {
fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()>;
}
impl SendMany for PeerTx {
fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()> {
requests
.into_iter()
.try_for_each(|r| self.send(r))
.context("peer dropped")
}
}
#[derive(Default, Debug)]
pub struct PeerCounters {
pub fetched_bytes: AtomicU64,
pub total_time_connecting_ms: AtomicU64,
pub connection_attempts: AtomicU32,
pub connections: AtomicU32,
pub errors: AtomicU32,
pub fetched_chunks: AtomicU32,
pub downloaded_and_checked_pieces: AtomicU32,
pub downloaded_and_checked_bytes: AtomicU64,
}
#[derive(Debug)]
pub struct PeerStats {
pub counters: Arc<PeerCounters>,
pub backoff: ExponentialBackoff,
}
impl Default for PeerStats {
fn default() -> Self {
Self {
counters: Arc::new(Default::default()),
backoff: ExponentialBackoffBuilder::new()
.with_initial_interval(Duration::from_secs(10))
.with_multiplier(6.)
.with_max_interval(Duration::from_secs(3600))
.with_max_elapsed_time(Some(Duration::from_secs(86400)))
.build(),
}
}
}
#[derive(Debug, Default)]
pub struct Peer {
pub state: PeerStateNoMut,
pub stats: PeerStats,
}
#[derive(Debug, Default, Serialize)]
pub struct AggregatePeerStatsAtomic {
pub queued: AtomicU32,
pub connecting: AtomicU32,
pub live: AtomicU32,
pub seen: AtomicU32,
pub dead: AtomicU32,
pub not_needed: AtomicU32,
}
pub fn atomic_inc(c: &AtomicU32) -> u32 {
c.fetch_add(1, Ordering::Relaxed)
}
pub fn atomic_dec(c: &AtomicU32) -> u32 {
c.fetch_sub(1, Ordering::Relaxed)
}
impl AggregatePeerStatsAtomic {
pub fn counter(&self, state: &PeerState) -> &AtomicU32 {
match state {
PeerState::Connecting(_) => &self.connecting,
PeerState::Live(_) => &self.live,
PeerState::Queued => &self.queued,
PeerState::Dead => &self.dead,
PeerState::NotNeeded => &self.not_needed,
}
}
pub fn inc(&self, state: &PeerState) {
atomic_inc(self.counter(state));
}
pub fn dec(&self, state: &PeerState) {
atomic_dec(self.counter(state));
}
pub fn incdec(&self, old: &PeerState, new: &PeerState) {
self.dec(old);
self.inc(new);
}
}
#[derive(Debug, Default)]
pub enum PeerState {
#[default]
// Will be tried to be connected as soon as possible.
Queued,
Connecting(PeerTx),
Live(LivePeerState),
// There was an error, and it's waiting for exponential backoff.
Dead,
// We don't need to do anything with the peer any longer.
// The peer has the full torrent, and we have the full torrent, so no need
// to keep talking to it.
NotNeeded,
}
impl std::fmt::Display for PeerState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.name())
}
}
impl PeerState {
pub fn name(&self) -> &'static str {
match self {
PeerState::Queued => "queued",
PeerState::Connecting(_) => "connecting",
PeerState::Live(_) => "live",
PeerState::Dead => "dead",
PeerState::NotNeeded => "not needed",
}
}
pub fn take_live_no_counters(self) -> Option<LivePeerState> {
match self {
PeerState::Live(l) => Some(l),
_ => None,
}
}
}
#[derive(Debug, Default)]
pub struct PeerStateNoMut(PeerState);
impl PeerStateNoMut {
pub fn get(&self) -> &PeerState {
&self.0
}
pub fn take(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
self.set(Default::default(), counters)
}
pub fn set(&mut self, new: PeerState, counters: &AggregatePeerStatsAtomic) -> PeerState {
counters.incdec(&self.0, &new);
std::mem::replace(&mut self.0, new)
}
pub fn get_live(&self) -> Option<&LivePeerState> {
match &self.0 {
PeerState::Live(l) => Some(l),
_ => None,
}
}
pub fn get_live_mut(&mut self) -> Option<&mut LivePeerState> {
match &mut self.0 {
PeerState::Live(l) => Some(l),
_ => None,
}
}
pub fn queued_to_connecting(
&mut self,
counters: &AggregatePeerStatsAtomic,
) -> Option<(PeerRx, PeerTx)> {
if let PeerState::Queued = &self.0 {
let (tx, rx) = unbounded_channel();
let tx_2 = tx.clone();
self.set(PeerState::Connecting(tx), counters);
Some((rx, tx_2))
} else {
None
}
}
pub fn connecting_to_live(
&mut self,
peer_id: Id20,
counters: &AggregatePeerStatsAtomic,
) -> Option<&mut LivePeerState> {
if let PeerState::Connecting(_) = &self.0 {
let tx = match self.take(counters) {
PeerState::Connecting(tx) => tx,
_ => unreachable!(),
};
self.set(PeerState::Live(LivePeerState::new(peer_id, tx)), counters);
self.get_live_mut()
} else {
None
}
}
pub fn to_dead(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
self.set(PeerState::Dead, counters)
}
pub fn to_not_needed(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
self.set(PeerState::NotNeeded, counters)
}
}
#[derive(Debug)]
pub struct LivePeerState {
pub peer_id: Id20,
pub peer_interested: bool,
// This is used to track the pieces the peer has.
pub bitfield: BF,
// When the peer sends us data this is used to track if we asked for it.
pub inflight_requests: HashSet<InflightRequest>,
// The main channel to send requests to peer.
pub tx: PeerTx,
}
impl LivePeerState {
pub fn new(peer_id: Id20, tx: PeerTx) -> Self {
LivePeerState {
peer_id,
peer_interested: false,
bitfield: BF::new(),
inflight_requests: Default::default(),
tx,
}
}
pub fn has_full_torrent(&self, total_pieces: usize) -> bool {
self.bitfield
.get(0..total_pieces)
.map_or(false, |s| s.all())
}
}
mod peer_stats_snapshot {
use std::{collections::HashMap, sync::atomic::Ordering};
use serde::{Deserialize, Serialize};
use crate::peer_state::PeerState;
#[derive(Serialize, Deserialize)]
pub struct PeerCounters {
pub fetched_bytes: u64,
pub total_time_connecting_ms: u64,
pub connection_attempts: u32,
pub connections: u32,
pub errors: u32,
pub fetched_chunks: u32,
pub downloaded_and_checked_pieces: u32,
}
#[derive(Serialize, Deserialize)]
pub struct PeerStats {
pub counters: PeerCounters,
pub state: &'static str,
}
impl From<&crate::peer_state::PeerCounters> for PeerCounters {
fn from(counters: &crate::peer_state::PeerCounters) -> Self {
Self {
fetched_bytes: counters.fetched_bytes.load(Ordering::Relaxed),
total_time_connecting_ms: counters.total_time_connecting_ms.load(Ordering::Relaxed),
connection_attempts: counters.connection_attempts.load(Ordering::Relaxed),
connections: counters.connections.load(Ordering::Relaxed),
errors: counters.errors.load(Ordering::Relaxed),
fetched_chunks: counters.fetched_chunks.load(Ordering::Relaxed),
downloaded_and_checked_pieces: counters
.downloaded_and_checked_pieces
.load(Ordering::Relaxed),
}
}
}
impl From<&crate::peer_state::Peer> for PeerStats {
fn from(peer: &crate::peer_state::Peer) -> Self {
Self {
counters: peer.stats.counters.as_ref().into(),
state: peer.state.get().name(),
}
}
}
#[derive(Serialize)]
pub struct PeerStatsSnapshot {
pub peers: HashMap<String, PeerStats>,
}
#[derive(Clone, Copy, Default, Deserialize)]
pub enum PeerStatsFilterState {
All,
#[default]
Live,
}
impl PeerStatsFilterState {
pub fn matches(&self, s: &PeerState) -> bool {
match (self, s) {
(Self::All, _) => true,
(Self::Live, PeerState::Live(_)) => true,
_ => false,
}
}
}
#[derive(Default, Deserialize)]
pub struct PeerStatsFilter {
pub state: PeerStatsFilterState,
}
}
pub use peer_stats_snapshot::{PeerStatsFilter, PeerStatsSnapshot};

386
crates/librqbit/src/session.rs
View file

@ -1,4 +1,13 @@
use std::{borrow::Cow, io::Read, net::SocketAddr, path::PathBuf, time::Duration};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
io::{BufReader, BufWriter, Read},
net::SocketAddr,
path::PathBuf,
str::FromStr,
sync::Arc,
time::Duration,
};
use anyhow::{bail, Context};
use buffers::ByteString;
@ -10,64 +19,78 @@ use librqbit_core::{
};
use parking_lot::RwLock;
use reqwest::Url;
use serde::{Deserialize, Serialize};
use tokio_stream::StreamExt;
use tracing::{debug, info, span, warn, Level};
use tracing::{debug, error, error_span, info, warn};
use crate::{
dht_utils::{read_metainfo_from_peer_receiver, ReadMetainfoResult},
peer_connection::PeerConnectionOptions,
spawn_utils::{spawn, BlockingSpawner},
torrent_manager::{TorrentManagerBuilder, TorrentManagerHandle},
torrent_state::{ManagedTorrentBuilder, ManagedTorrentHandle, ManagedTorrentState},
};
pub const SUPPORTED_SCHEMES: [&str; 3] = ["http:", "https:", "magnet:"];
#[derive(Clone)]
pub enum ManagedTorrentState {
Initializing,
Running(TorrentManagerHandle),
}
#[derive(Clone)]
pub struct ManagedTorrent {
pub info_hash: Id20,
pub output_folder: PathBuf,
pub state: ManagedTorrentState,
}
impl PartialEq for ManagedTorrent {
fn eq(&self, other: &Self) -> bool {
self.info_hash == other.info_hash && self.output_folder == other.output_folder
}
}
pub type TorrentId = usize;
#[derive(Default)]
pub struct SessionLocked {
torrents: Vec<ManagedTorrent>,
pub struct SessionDatabase {
next_id: usize,
torrents: HashMap<usize, ManagedTorrentHandle>,
}
enum SessionLockedAddTorrentResult {
AlreadyManaged(ManagedTorrent),
Added(usize),
}
impl SessionLocked {
fn add_torrent(&mut self, torrent: ManagedTorrent) -> SessionLockedAddTorrentResult {
if let Some(handle) = self.torrents.iter().find(|t| **t == torrent) {
return SessionLockedAddTorrentResult::AlreadyManaged(handle.clone());
}
let idx = self.torrents.len();
self.torrents.push(torrent);
SessionLockedAddTorrentResult::Added(idx)
impl SessionDatabase {
fn add_torrent(&mut self, torrent: ManagedTorrentHandle) -> TorrentId {
let idx = self.next_id;
self.torrents.insert(idx, torrent);
self.next_id += 1;
idx
}
fn serialize(&self) -> SerializedSessionDatabase {
SerializedSessionDatabase {
torrents: self
.torrents
.values()
.map(|torrent| SerializedTorrent {
trackers: torrent
.info()
.trackers
.iter()
.map(|u| u.to_string())
.collect(),
info_hash: torrent.info_hash().as_string(),
only_files: torrent.only_files.clone(),
is_paused: torrent.with_state(|s| matches!(s, ManagedTorrentState::Paused(_))),
output_folder: torrent.info().out_dir.clone(),
})
.collect(),
}
}
}
#[derive(Serialize, Deserialize)]
struct SerializedTorrent {
info_hash: String,
trackers: HashSet<String>,
output_folder: PathBuf,
only_files: Option<Vec<usize>>,
is_paused: bool,
}
#[derive(Serialize, Deserialize)]
struct SerializedSessionDatabase {
torrents: Vec<SerializedTorrent>,
}
pub struct Session {
peer_id: Id20,
dht: Option<Dht>,
persistence_filename: PathBuf,
peer_opts: PeerConnectionOptions,
spawner: BlockingSpawner,
locked: RwLock<SessionLocked>,
db: RwLock<SessionDatabase>,
output_folder: PathBuf,
}
@ -107,6 +130,7 @@ fn compute_only_files<ByteBuf: AsRef<[u8]>>(
#[derive(Default, Clone)]
pub struct AddTorrentOptions {
pub paused: bool,
pub only_files_regex: Option<String>,
pub only_files: Option<Vec<usize>>,
pub overwrite: bool,
@ -124,9 +148,9 @@ pub struct ListOnlyResponse {
}
pub enum AddTorrentResponse {
AlreadyManaged(ManagedTorrent),
AlreadyManaged(TorrentId, ManagedTorrentHandle),
ListOnly(ListOnlyResponse),
Added(TorrentManagerHandle),
Added(TorrentId, ManagedTorrentHandle),
}
pub fn read_local_file_including_stdin(filename: &str) -> anyhow::Result<Vec<u8>> {
@ -185,20 +209,26 @@ impl<'a> AddTorrent<'a> {
pub struct SessionOptions {
pub disable_dht: bool,
pub disable_dht_persistence: bool,
pub persistence: bool,
// Will default to output_folder/.rqbit-session.json
pub persistence_filename: Option<PathBuf>,
pub dht_config: Option<PersistentDhtConfig>,
pub peer_id: Option<Id20>,
pub peer_opts: Option<PeerConnectionOptions>,
}
impl Session {
pub async fn new(output_folder: PathBuf, spawner: BlockingSpawner) -> anyhow::Result<Self> {
pub async fn new(
output_folder: PathBuf,
spawner: BlockingSpawner,
) -> anyhow::Result<Arc<Self>> {
Self::new_with_opts(output_folder, spawner, SessionOptions::default()).await
}
pub async fn new_with_opts(
output_folder: PathBuf,
spawner: BlockingSpawner,
opts: SessionOptions,
) -> anyhow::Result<Self> {
) -> anyhow::Result<Arc<Self>> {
let peer_id = opts.peer_id.unwrap_or_else(generate_peer_id);
let dht = if opts.disable_dht {
None
@ -212,31 +242,147 @@ impl Session {
Some(dht)
};
let peer_opts = opts.peer_opts.unwrap_or_default();
Ok(Self {
let persistence_filename = opts
.persistence_filename
.unwrap_or_else(|| output_folder.join(".rqbit-session.json"));
let session = Arc::new(Self {
persistence_filename,
peer_id,
dht,
peer_opts,
spawner,
output_folder,
locked: RwLock::new(SessionLocked::default()),
})
db: RwLock::new(Default::default()),
});
if opts.persistence {
if let Some(parent) = session.persistence_filename.parent() {
std::fs::create_dir_all(parent).with_context(|| {
format!("couldn't create directory {:?} for session storage", parent)
})?;
}
let session = session.clone();
spawn(
"session persistene",
error_span!("session_persistence"),
async move {
// Populate initial from the state filename
if let Err(e) = session.populate_from_stored().await {
error!("could not populate session from stored file: {:?}", e);
}
let session = Arc::downgrade(&session);
loop {
tokio::time::sleep(Duration::from_secs(10)).await;
let session = match session.upgrade() {
Some(s) => s,
None => break,
};
if let Err(e) = session.dump_to_disk() {
error!("error dumping session to disk: {:?}", e);
}
}
Ok(())
},
);
}
Ok(session)
}
pub fn get_dht(&self) -> Option<Dht> {
self.dht.clone()
pub fn get_dht(&self) -> Option<&Dht> {
self.dht.as_ref()
}
pub fn with_torrents<F>(&self, callback: F)
where
F: Fn(&[ManagedTorrent]),
{
callback(&self.locked.read().torrents)
async fn populate_from_stored(self: &Arc<Self>) -> anyhow::Result<()> {
let mut rdr = match std::fs::File::open(&self.persistence_filename) {
Ok(f) => BufReader::new(f),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => {
return Err(e).context(format!(
"error opening session file {:?}",
self.persistence_filename
))
}
};
let db: SerializedSessionDatabase =
serde_json::from_reader(&mut rdr).context("error deserializing session database")?;
let mut futures = Vec::new();
for storrent in db.torrents.into_iter() {
let magnet = Magnet {
info_hash: Id20::from_str(&storrent.info_hash)
.context("error deserializing info_hash")?,
trackers: storrent.trackers.into_iter().collect(),
};
futures.push({
let session = self.clone();
async move {
session
.add_torrent(
AddTorrent::Url(Cow::Owned(magnet.to_string())),
Some(AddTorrentOptions {
paused: storrent.is_paused,
output_folder: Some(
storrent
.output_folder
.to_str()
.context("broken path")?
.to_owned(),
),
only_files: storrent.only_files,
overwrite: true,
..Default::default()
}),
)
.await
.map_err(|e| {
error!("error adding torrent from stored session: {:?}", e);
e
})
}
});
}
futures::future::join_all(futures).await;
Ok(())
}
fn dump_to_disk(&self) -> anyhow::Result<()> {
let tmp_filename = format!("{}.tmp", self.persistence_filename.to_str().unwrap());
let mut tmp = BufWriter::new(
std::fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&tmp_filename)
.with_context(|| format!("error opening {:?}", tmp_filename))?,
);
let serialized = self.db.read().serialize();
serde_json::to_writer(&mut tmp, &serialized).context("error serializing")?;
drop(tmp);
std::fs::rename(&tmp_filename, &self.persistence_filename)
.context("error renaming persistence file")?;
debug!("wrote persistence to {:?}", &self.persistence_filename);
Ok(())
}
pub fn with_torrents<R>(
&self,
callback: impl Fn(&mut dyn Iterator<Item = (TorrentId, &ManagedTorrentHandle)>) -> R,
) -> R {
callback(&mut self.db.read().torrents.iter().map(|(id, t)| (*id, t)))
}
pub async fn add_torrent(
&self,
add: impl Into<AddTorrent<'_>>,
opts: Option<AddTorrentOptions>,
) -> anyhow::Result<AddTorrentResponse> {
// Magnet links are different in that we first need to discover the metadata.
let span = error_span!("add_torrent");
let _ = span.enter();
let opts = opts.unwrap_or_default();
let (info_hash, info, dht_rx, trackers, initial_peers) = match add.into() {
@ -250,8 +396,7 @@ impl Session {
.dht
.as_ref()
.context("magnet links without DHT are not supported")?
.get_peers(info_hash)
.await?;
.get_peers(info_hash)?;
let trackers = trackers
.into_iter()
@ -264,6 +409,7 @@ impl Session {
})
.collect();
debug!("querying DHT for {:?}", info_hash);
let (info, dht_rx, initial_peers) = match read_metainfo_from_peer_receiver(
self.peer_id,
info_hash,
@ -277,6 +423,7 @@ impl Session {
anyhow::bail!("DHT died, no way to discover torrent metainfo")
}
};
debug!("received result from DHT: {:?}", info);
(info_hash, info, Some(dht_rx), trackers, initial_peers)
}
other => {
@ -300,7 +447,7 @@ impl Session {
let dht_rx = match self.dht.as_ref() {
Some(dht) => {
debug!("reading peers for {:?} from DHT", torrent.info_hash);
Some(dht.get_peers(torrent.info_hash).await?)
Some(dht.get_peers(torrent.info_hash)?)
}
None => None,
};
@ -404,24 +551,13 @@ impl Session {
.unwrap_or_else(|| self.output_folder.clone())
.join(sub_folder);
let managed_torrent = ManagedTorrent {
info_hash,
output_folder: output_folder.clone(),
state: ManagedTorrentState::Initializing,
};
match self.locked.write().add_torrent(managed_torrent) {
SessionLockedAddTorrentResult::AlreadyManaged(managed) => {
return Ok(AddTorrentResponse::AlreadyManaged(managed))
}
SessionLockedAddTorrentResult::Added(_) => {}
}
let mut builder = TorrentManagerBuilder::new(info, info_hash, output_folder.clone());
let mut builder = ManagedTorrentBuilder::new(info, info_hash, output_folder.clone());
builder
.overwrite(opts.overwrite)
.spawner(self.spawner)
.peer_id(self.peer_id);
.peer_id(self.peer_id)
.trackers(trackers);
if let Some(only_files) = only_files {
builder.only_files(only_files);
}
@ -437,61 +573,79 @@ impl Session {
builder.peer_read_write_timeout(t);
}
let handle = match builder
.start_manager()
.context("error starting torrent manager")
{
Ok(handle) => {
let mut g = self.locked.write();
let m = g
.torrents
.iter_mut()
.find(|t| t.info_hash == info_hash && t.output_folder == output_folder)
.unwrap();
m.state = ManagedTorrentState::Running(handle.clone());
handle
}
Err(error) => {
let mut g = self.locked.write();
let idx = g
.torrents
.iter()
.position(|t| t.info_hash == info_hash && t.output_folder == output_folder)
.unwrap();
g.torrents.remove(idx);
return Err(error);
let (managed_torrent, id) = {
let mut g = self.db.write();
if let Some((id, handle)) = g.torrents.iter().find(|(_, t)| t.info_hash() == info_hash)
{
return Ok(AddTorrentResponse::AlreadyManaged(*id, handle.clone()));
}
let next_id = g.torrents.len();
let managed_torrent =
builder.build(error_span!(parent: None, "torrent", id = next_id))?;
let id = g.add_torrent(managed_torrent.clone());
(managed_torrent, id)
};
{
let mut g = self.locked.write();
let m = g
.torrents
.iter_mut()
.find(|t| t.info_hash == info_hash && t.output_folder == output_folder)
.unwrap();
m.state = ManagedTorrentState::Running(handle.clone());
let span = managed_torrent.info.span.clone();
let _ = span.enter();
managed_torrent
.start(initial_peers, dht_peer_rx, opts.paused)
.context("error starting torrent")?;
}
for url in trackers {
handle.add_tracker(url);
}
for peer in initial_peers {
handle.add_peer(peer);
}
Ok(AddTorrentResponse::Added(id, managed_torrent))
}
if let Some(mut dht_peer_rx) = dht_peer_rx {
spawn(span!(Level::INFO, "dht_peer_adder"), {
let handle = handle.clone();
async move {
while let Some(peer) = dht_peer_rx.next().await {
handle.add_peer(peer);
pub fn get(&self, id: TorrentId) -> Option<ManagedTorrentHandle> {
self.db.read().torrents.get(&id).cloned()
}
pub fn delete(&self, id: TorrentId, delete_files: bool) -> anyhow::Result<()> {
let removed = self
.db
.write()
.torrents
.remove(&id)
.with_context(|| format!("torrent with id {} did not exist", id))?;
let paused = removed
.with_state_mut(|s| {
let paused = match s.take() {
ManagedTorrentState::Paused(p) => p,
ManagedTorrentState::Live(l) => l.pause()?,
_ => return Ok(None),
};
Ok::<_, anyhow::Error>(Some(paused))
})
.context("error pausing torrent");
match (paused, delete_files) {
(Err(e), true) => Err(e).context("torrent deleted, but could not delete files"),
(Err(e), false) => {
warn!("could not delete torrent files: {:?}", e);
Ok(())
}
(Ok(Some(paused)), true) => {
drop(paused.files);
for file in paused.filenames {
if let Err(e) = std::fs::remove_file(&file) {
warn!("could not delete file {:?}: {:?}", file, e);
}
warn!("dht was closed");
Ok(())
}
});
Ok(())
}
_ => Ok(()),
}
}
Ok(AddTorrentResponse::Added(handle))
pub fn unpause(&self, handle: &ManagedTorrentHandle) -> anyhow::Result<()> {
let peer_rx = self
.dht
.as_ref()
.map(|dht| dht.get_peers(handle.info_hash()))
.transpose()?;
handle.start(Default::default(), peer_rx, false)?;
Ok(())
}
}

19
crates/librqbit/src/spawn_utils.rs
View file

@ -1,22 +1,9 @@
use tracing::{debug, error, trace, Instrument};
pub fn spawn(
_name: &str,
span: tracing::Span,
fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
) {
let fut = async move {
trace!("started");
match fut.await {
Ok(_) => {
debug!("finished");
}
Err(e) => {
error!("{:#}", e)
}
}
}
.instrument(span.or_current());
tokio::spawn(fut);
) -> tokio::task::JoinHandle<()> {
librqbit_core::spawn_utils::spawn(span, fut)
}
#[derive(Clone, Copy, Debug)]

380
crates/librqbit/src/torrent_manager.rs
View file

@ -1,380 +0,0 @@
use std::{
collections::HashSet,
fs::{File, OpenOptions},
net::SocketAddr,
path::{Path, PathBuf},
sync::Arc,
time::{Duration, Instant},
};
use anyhow::Context;
use bencode::from_bytes;
use buffers::ByteString;
use librqbit_core::{
id20::Id20, lengths::Lengths, peer_id::generate_peer_id, speed_estimator::SpeedEstimator,
torrent_metainfo::TorrentMetaV1Info,
};
use parking_lot::Mutex;
use reqwest::Url;
use sha1w::Sha1;
use size_format::SizeFormatterBinary as SF;
use tracing::{debug, info, span, warn, Level};
use crate::{
chunk_tracker::ChunkTracker,
file_ops::FileOps,
spawn_utils::{spawn, BlockingSpawner},
torrent_state::{TorrentState, TorrentStateOptions},
tracker_comms::{TrackerError, TrackerRequest, TrackerRequestEvent, TrackerResponse},
};
#[derive(Default)]
struct TorrentManagerOptions {
force_tracker_interval: Option<Duration>,
peer_connect_timeout: Option<Duration>,
peer_read_write_timeout: Option<Duration>,
only_files: Option<Vec<usize>>,
peer_id: Option<Id20>,
overwrite: bool,
}
pub struct TorrentManagerBuilder {
info: TorrentMetaV1Info<ByteString>,
info_hash: Id20,
output_folder: PathBuf,
options: TorrentManagerOptions,
spawner: Option<BlockingSpawner>,
}
impl TorrentManagerBuilder {
pub fn new<P: AsRef<Path>>(
info: TorrentMetaV1Info<ByteString>,
info_hash: Id20,
output_folder: P,
) -> Self {
Self {
info,
info_hash,
output_folder: output_folder.as_ref().into(),
spawner: None,
options: TorrentManagerOptions::default(),
}
}
pub fn only_files(&mut self, only_files: Vec<usize>) -> &mut Self {
self.options.only_files = Some(only_files);
self
}
pub fn overwrite(&mut self, overwrite: bool) -> &mut Self {
self.options.overwrite = overwrite;
self
}
pub fn force_tracker_interval(&mut self, force_tracker_interval: Duration) -> &mut Self {
self.options.force_tracker_interval = Some(force_tracker_interval);
self
}
pub fn spawner(&mut self, spawner: BlockingSpawner) -> &mut Self {
self.spawner = Some(spawner);
self
}
pub fn peer_id(&mut self, peer_id: Id20) -> &mut Self {
self.options.peer_id = Some(peer_id);
self
}
pub fn peer_connect_timeout(&mut self, timeout: Duration) -> &mut Self {
self.options.peer_connect_timeout = Some(timeout);
self
}
pub fn peer_read_write_timeout(&mut self, timeout: Duration) -> &mut Self {
self.options.peer_read_write_timeout = Some(timeout);
self
}
pub fn start_manager(self) -> anyhow::Result<TorrentManagerHandle> {
TorrentManager::start(
self.info,
self.info_hash,
self.output_folder,
self.spawner.unwrap_or_else(|| BlockingSpawner::new(true)),
Some(self.options),
)
}
}
#[derive(Clone)]
pub struct TorrentManagerHandle {
manager: Arc<TorrentManager>,
}
impl TorrentManagerHandle {
pub fn add_tracker(&self, url: Url) -> bool {
let mgr = self.manager.clone();
if mgr.trackers.lock().insert(url.clone()) {
spawn(
span!(Level::ERROR, "tracker_monitor", url = url.to_string()),
async move { mgr.single_tracker_monitor(url).await },
);
true
} else {
false
}
}
pub fn only_files(&self) -> Option<&[usize]> {
self.manager.options.only_files.as_deref()
}
pub fn add_peer(&self, addr: SocketAddr) -> bool {
self.manager.state.add_peer_if_not_seen(addr)
}
pub fn torrent_state(&self) -> &TorrentState {
&self.manager.state
}
pub fn speed_estimator(&self) -> &Arc<SpeedEstimator> {
&self.manager.speed_estimator
}
pub async fn cancel(&self) -> anyhow::Result<()> {
todo!()
}
pub async fn wait_until_completed(&self) -> anyhow::Result<()> {
self.manager.state.wait_until_completed().await;
Ok(())
}
}
struct TorrentManager {
state: Arc<TorrentState>,
#[allow(dead_code)]
speed_estimator: Arc<SpeedEstimator>,
trackers: Mutex<HashSet<Url>>,
options: TorrentManagerOptions,
}
fn make_lengths<ByteBuf: AsRef<[u8]>>(
torrent: &TorrentMetaV1Info<ByteBuf>,
) -> anyhow::Result<Lengths> {
let total_length = torrent.iter_file_lengths()?.sum();
Lengths::new(total_length, torrent.piece_length, None)
}
fn ensure_file_length(file: &File, length: u64) -> anyhow::Result<()> {
Ok(file.set_len(length)?)
}
impl TorrentManager {
fn start<P: AsRef<Path>>(
info: TorrentMetaV1Info<ByteString>,
info_hash: Id20,
out: P,
spawner: BlockingSpawner,
options: Option<TorrentManagerOptions>,
) -> anyhow::Result<TorrentManagerHandle> {
let options = options.unwrap_or_default();
let (files, filenames) = {
let mut files =
Vec::<Arc<Mutex<File>>>::with_capacity(info.iter_file_lengths()?.count());
let mut filenames = Vec::new();
for (path_bits, _) in info.iter_filenames_and_lengths()? {
let mut full_path = out.as_ref().to_owned();
let relative_path = path_bits
.to_pathbuf()
.context("error converting file to path")?;
full_path.push(relative_path);
std::fs::create_dir_all(full_path.parent().unwrap())?;
let file = if options.overwrite {
OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(&full_path)?
} else {
// TODO: create_new does not seem to work with read(true), so calling this twice.
OpenOptions::new()
.create_new(true)
.write(true)
.open(&full_path)
.with_context(|| format!("error creating {:?}", &full_path))?;
OpenOptions::new().read(true).write(true).open(&full_path)?
};
filenames.push(full_path);
files.push(Arc::new(Mutex::new(file)))
}
(files, filenames)
};
let peer_id = options.peer_id.unwrap_or_else(generate_peer_id);
let lengths = make_lengths(&info).context("unable to compute Lengths from torrent")?;
debug!("computed lengths: {:?}", &lengths);
info!("Doing initial checksum validation, this might take a while...");
let initial_check_results = spawner.spawn_block_in_place(|| {
FileOps::<Sha1>::new(&info, &files, &lengths)
.initial_check(options.only_files.as_deref())
})?;
info!(
"Initial check results: have {}, needed {}",
SF::new(initial_check_results.have_bytes),
SF::new(initial_check_results.needed_bytes)
);
spawner.spawn_block_in_place(|| {
for (idx, (file, (name, length))) in files
.iter()
.zip(info.iter_filenames_and_lengths().unwrap())
.enumerate()
{
if options
.only_files
.as_ref()
.map(|v| !v.contains(&idx))
.unwrap_or(false)
{
continue;
}
let now = Instant::now();
if let Err(err) = ensure_file_length(&file.lock(), length) {
warn!(
"Error setting length for file {:?} to {}: {:#?}",
name, length, err
);
} else {
debug!(
"Set length for file {:?} to {} in {:?}",
name,
SF::new(length),
now.elapsed()
);
}
}
});
let chunk_tracker = ChunkTracker::new(
initial_check_results.needed_pieces,
initial_check_results.have_pieces,
lengths,
);
#[allow(clippy::needless_update)]
let state_options = TorrentStateOptions {
peer_connect_timeout: options.peer_connect_timeout,
peer_read_write_timeout: options.peer_read_write_timeout,
..Default::default()
};
let state = TorrentState::new(
info,
info_hash,
peer_id,
files,
filenames,
chunk_tracker,
lengths,
initial_check_results.have_bytes,
initial_check_results.needed_bytes,
spawner,
Some(state_options),
);
let estimator = Arc::new(SpeedEstimator::new(5));
let mgr = Arc::new(Self {
state,
speed_estimator: estimator.clone(),
trackers: Mutex::new(HashSet::new()),
options,
});
spawn(span!(Level::ERROR, "speed_estimator_updater"), {
let state = mgr.state.clone();
async move {
loop {
let stats = state.stats_snapshot();
let fetched = stats.fetched_bytes;
let needed = state.initially_needed();
// fetched can be too high in theory, so for safety make sure that it doesn't wrap around u64.
let remaining = needed
.wrapping_sub(fetched)
.min(needed - stats.downloaded_and_checked_bytes);
estimator.add_snapshot(fetched, remaining, Instant::now());
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
});
Ok(mgr.into_handle())
}
fn into_handle(self: Arc<Self>) -> TorrentManagerHandle {
TorrentManagerHandle { manager: self }
}
async fn tracker_one_request(&self, tracker_url: Url) -> anyhow::Result<u64> {
let response: reqwest::Response = reqwest::get(tracker_url).await?;
if !response.status().is_success() {
anyhow::bail!("tracker responded with {:?}", response.status());
}
let bytes = response.bytes().await?;
if let Ok(error) = from_bytes::<TrackerError>(&bytes) {
anyhow::bail!(
"tracker returned failure. Failure reason: {}",
error.failure_reason
)
};
let response = from_bytes::<TrackerResponse>(&bytes)?;
for peer in response.peers.iter_sockaddrs() {
self.state.add_peer_if_not_seen(peer);
}
Ok(response.interval)
}
async fn single_tracker_monitor(&self, mut tracker_url: Url) -> anyhow::Result<()> {
let mut event = Some(TrackerRequestEvent::Started);
loop {
let request = TrackerRequest {
info_hash: self.state.info_hash(),
peer_id: self.state.peer_id(),
port: 6778,
uploaded: self.state.get_uploaded_bytes(),
downloaded: self.state.get_downloaded_bytes(),
left: self.state.get_left_to_download_bytes(),
compact: true,
no_peer_id: false,
event,
ip: None,
numwant: None,
key: None,
trackerid: None,
};
let request_query = request.as_querystring();
tracker_url.set_query(Some(&request_query));
match self.tracker_one_request(tracker_url.clone()).await {
Ok(interval) => {
event = None;
let interval = self
.options
.force_tracker_interval
.unwrap_or_else(|| Duration::from_secs(interval));
debug!(
"sleeping for {:?} after calling tracker {}",
interval,
tracker_url.host().unwrap()
);
tokio::time::sleep(interval).await;
}
Err(e) => {
debug!("error calling the tracker {}: {:#}", tracker_url, e);
tokio::time::sleep(Duration::from_secs(60)).await;
}
};
}
}
}

140
crates/librqbit/src/torrent_state/initializing.rs Normal file
View file

@ -0,0 +1,140 @@
use std::{
fs::{File, OpenOptions},
sync::{atomic::AtomicU64, Arc},
time::Instant,
};
use anyhow::Context;
use parking_lot::Mutex;
use sha1w::Sha1;
use size_format::SizeFormatterBinary as SF;
use tracing::{debug, info, warn};
use crate::{chunk_tracker::ChunkTracker, file_ops::FileOps};
use super::{paused::TorrentStatePaused, ManagedTorrentInfo};
fn ensure_file_length(file: &File, length: u64) -> anyhow::Result<()> {
Ok(file.set_len(length)?)
}
pub struct TorrentStateInitializing {
pub(crate) meta: Arc<ManagedTorrentInfo>,
pub(crate) only_files: Option<Vec<usize>>,
pub(crate) checked_bytes: AtomicU64,
}
impl TorrentStateInitializing {
pub fn new(meta: Arc<ManagedTorrentInfo>, only_files: Option<Vec<usize>>) -> Self {
Self {
meta,
only_files,
checked_bytes: AtomicU64::new(0),
}
}
pub fn get_checked_bytes(&self) -> u64 {
self.checked_bytes
.load(std::sync::atomic::Ordering::Relaxed)
}
pub async fn check(&self) -> anyhow::Result<TorrentStatePaused> {
let (files, filenames) = {
let mut files =
Vec::<Arc<Mutex<File>>>::with_capacity(self.meta.info.iter_file_lengths()?.count());
let mut filenames = Vec::new();
for (path_bits, _) in self.meta.info.iter_filenames_and_lengths()? {
let mut full_path = self.meta.out_dir.clone();
let relative_path = path_bits
.to_pathbuf()
.context("error converting file to path")?;
full_path.push(relative_path);
std::fs::create_dir_all(full_path.parent().unwrap())?;
let file = if self.meta.options.overwrite {
OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(&full_path)
.with_context(|| {
format!("error opening {full_path:?} in read/write mode")
})?
} else {
// TODO: create_new does not seem to work with read(true), so calling this twice.
OpenOptions::new()
.create_new(true)
.write(true)
.open(&full_path)
.with_context(|| format!("error creating {:?}", &full_path))?;
OpenOptions::new().read(true).write(true).open(&full_path)?
};
filenames.push(full_path);
files.push(Arc::new(Mutex::new(file)))
}
(files, filenames)
};
debug!("computed lengths: {:?}", &self.meta.lengths);
info!("Doing initial checksum validation, this might take a while...");
let initial_check_results = self.meta.spawner.spawn_block_in_place(|| {
FileOps::<Sha1>::new(&self.meta.info, &files, &self.meta.lengths)
.initial_check(self.only_files.as_deref(), &self.checked_bytes)
})?;
info!(
"Initial check results: have {}, needed {}",
SF::new(initial_check_results.have_bytes),
SF::new(initial_check_results.needed_bytes)
);
self.meta.spawner.spawn_block_in_place(|| {
for (idx, (file, (name, length))) in files
.iter()
.zip(self.meta.info.iter_filenames_and_lengths().unwrap())
.enumerate()
{
if self
.only_files
.as_ref()
.map(|v| !v.contains(&idx))
.unwrap_or(false)
{
continue;
}
let now = Instant::now();
if let Err(err) = ensure_file_length(&file.lock(), length) {
warn!(
"Error setting length for file {:?} to {}: {:#?}",
name, length, err
);
} else {
debug!(
"Set length for file {:?} to {} in {:?}",
name,
SF::new(length),
now.elapsed()
);
}
}
});
let chunk_tracker = ChunkTracker::new(
initial_check_results.needed_pieces,
initial_check_results.have_pieces,
self.meta.lengths,
);
let paused = TorrentStatePaused {
info: self.meta.clone(),
files,
filenames,
chunk_tracker,
have_bytes: initial_check_results.have_bytes,
};
Ok(paused)
}
}

788
crates/librqbit/src/torrent_state.rs → crates/librqbit/src/torrent_state/live/mod.rs
View file

File diff suppressed because it is too large Load diff

187
crates/librqbit/src/torrent_state/live/peer/mod.rs Normal file
View file

@ -0,0 +1,187 @@
pub mod stats;
use std::collections::HashSet;
use anyhow::Context;
use librqbit_core::id20::Id20;
use librqbit_core::lengths::{ChunkInfo, ValidPieceIndex};
use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
use crate::peer_connection::WriterRequest;
use crate::type_aliases::BF;
use super::peers::stats::atomic::AggregatePeerStatsAtomic;
#[derive(Debug, Hash, PartialEq, Eq)]
pub(crate) struct InflightRequest {
pub piece: ValidPieceIndex,
pub chunk: u32,
}
impl From<&ChunkInfo> for InflightRequest {
fn from(c: &ChunkInfo) -> Self {
Self {
piece: c.piece_index,
chunk: c.chunk_index,
}
}
}
// TODO: Arc can be removed probably, as UnboundedSender should be clone + it can be downgraded to weak.
pub(crate) type PeerRx = UnboundedReceiver<WriterRequest>;
pub(crate) type PeerTx = UnboundedSender<WriterRequest>;
pub trait SendMany {
fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()>;
}
impl SendMany for PeerTx {
fn send_many(&self, requests: impl IntoIterator<Item = WriterRequest>) -> anyhow::Result<()> {
requests
.into_iter()
.try_for_each(|r| self.send(r))
.context("peer dropped")
}
}
#[derive(Debug, Default)]
pub(crate) struct Peer {
pub state: PeerStateNoMut,
pub stats: stats::atomic::PeerStats,
}
#[derive(Debug, Default)]
pub(crate) enum PeerState {
#[default]
// Will be tried to be connected as soon as possible.
Queued,
Connecting(PeerTx),
Live(LivePeerState),
// There was an error, and it's waiting for exponential backoff.
Dead,
// We don't need to do anything with the peer any longer.
// The peer has the full torrent, and we have the full torrent, so no need
// to keep talking to it.
NotNeeded,
}
impl std::fmt::Display for PeerState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.name())
}
}
impl PeerState {
pub fn name(&self) -> &'static str {
match self {
PeerState::Queued => "queued",
PeerState::Connecting(_) => "connecting",
PeerState::Live(_) => "live",
PeerState::Dead => "dead",
PeerState::NotNeeded => "not needed",
}
}
pub fn take_live_no_counters(self) -> Option<LivePeerState> {
match self {
PeerState::Live(l) => Some(l),
_ => None,
}
}
}
#[derive(Debug, Default)]
pub(crate) struct PeerStateNoMut(PeerState);
impl PeerStateNoMut {
pub fn get(&self) -> &PeerState {
&self.0
}
pub fn take(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
self.set(Default::default(), counters)
}
pub fn set(&mut self, new: PeerState, counters: &AggregatePeerStatsAtomic) -> PeerState {
counters.incdec(&self.0, &new);
std::mem::replace(&mut self.0, new)
}
pub fn get_live_mut(&mut self) -> Option<&mut LivePeerState> {
match &mut self.0 {
PeerState::Live(l) => Some(l),
_ => None,
}
}
pub fn queued_to_connecting(
&mut self,
counters: &AggregatePeerStatsAtomic,
) -> Option<(PeerRx, PeerTx)> {
if let PeerState::Queued = &self.0 {
let (tx, rx) = unbounded_channel();
let tx_2 = tx.clone();
self.set(PeerState::Connecting(tx), counters);
Some((rx, tx_2))
} else {
None
}
}
pub fn connecting_to_live(
&mut self,
peer_id: Id20,
counters: &AggregatePeerStatsAtomic,
) -> Option<&mut LivePeerState> {
if let PeerState::Connecting(_) = &self.0 {
let tx = match self.take(counters) {
PeerState::Connecting(tx) => tx,
_ => unreachable!(),
};
self.set(PeerState::Live(LivePeerState::new(peer_id, tx)), counters);
self.get_live_mut()
} else {
None
}
}
pub fn set_not_needed(&mut self, counters: &AggregatePeerStatsAtomic) -> PeerState {
self.set(PeerState::NotNeeded, counters)
}
}
#[derive(Debug)]
pub(crate) struct LivePeerState {
#[allow(dead_code)]
peer_id: Id20,
pub peer_interested: bool,
// This is used to track the pieces the peer has.
pub bitfield: BF,
// When the peer sends us data this is used to track if we asked for it.
pub inflight_requests: HashSet<InflightRequest>,
// The main channel to send requests to peer.
pub tx: PeerTx,
}
impl LivePeerState {
pub fn new(peer_id: Id20, tx: PeerTx) -> Self {
LivePeerState {
peer_id,
peer_interested: false,
bitfield: BF::new(),
inflight_requests: Default::default(),
tx,
}
}
pub fn has_full_torrent(&self, total_pieces: usize) -> bool {
self.bitfield
.get(0..total_pieces)
.map_or(false, |s| s.all())
}
}

41
crates/librqbit/src/torrent_state/live/peer/stats/atomic.rs Normal file
View file

@ -0,0 +1,41 @@
use std::{
sync::{
atomic::{AtomicU32, AtomicU64},
Arc,
},
time::Duration,
};
use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
#[derive(Default, Debug)]
pub(crate) struct PeerCountersAtomic {
pub fetched_bytes: AtomicU64,
pub total_time_connecting_ms: AtomicU64,
pub connection_attempts: AtomicU32,
pub connections: AtomicU32,
pub errors: AtomicU32,
pub fetched_chunks: AtomicU32,
pub downloaded_and_checked_pieces: AtomicU32,
pub downloaded_and_checked_bytes: AtomicU64,
}
#[derive(Debug)]
pub(crate) struct PeerStats {
pub counters: Arc<PeerCountersAtomic>,
pub backoff: ExponentialBackoff,
}
impl Default for PeerStats {
fn default() -> Self {
Self {
counters: Arc::new(Default::default()),
backoff: ExponentialBackoffBuilder::new()
.with_initial_interval(Duration::from_secs(10))
.with_multiplier(6.)
.with_max_interval(Duration::from_secs(3600))
.with_max_elapsed_time(Some(Duration::from_secs(86400)))
.build(),
}
}
}

2
crates/librqbit/src/torrent_state/live/peer/stats/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod atomic;
pub mod snapshot;

70
crates/librqbit/src/torrent_state/live/peer/stats/snapshot.rs Normal file
View file

@ -0,0 +1,70 @@
use std::{collections::HashMap, sync::atomic::Ordering};
use serde::{Deserialize, Serialize};
use crate::torrent_state::live::peer::{Peer, PeerState};
#[derive(Serialize, Deserialize)]
pub struct PeerCounters {
pub fetched_bytes: u64,
pub total_time_connecting_ms: u64,
pub connection_attempts: u32,
pub connections: u32,
pub errors: u32,
pub fetched_chunks: u32,
pub downloaded_and_checked_pieces: u32,
}
#[derive(Serialize, Deserialize)]
pub struct PeerStats {
pub counters: PeerCounters,
pub state: &'static str,
}
impl From<&super::atomic::PeerCountersAtomic> for PeerCounters {
fn from(counters: &super::atomic::PeerCountersAtomic) -> Self {
Self {
fetched_bytes: counters.fetched_bytes.load(Ordering::Relaxed),
total_time_connecting_ms: counters.total_time_connecting_ms.load(Ordering::Relaxed),
connection_attempts: counters.connection_attempts.load(Ordering::Relaxed),
connections: counters.connections.load(Ordering::Relaxed),
errors: counters.errors.load(Ordering::Relaxed),
fetched_chunks: counters.fetched_chunks.load(Ordering::Relaxed),
downloaded_and_checked_pieces: counters
.downloaded_and_checked_pieces
.load(Ordering::Relaxed),
}
}
}
impl From<&Peer> for PeerStats {
fn from(peer: &Peer) -> Self {
Self {
counters: peer.stats.counters.as_ref().into(),
state: peer.state.get().name(),
}
}
}
#[derive(Serialize)]
pub struct PeerStatsSnapshot {
pub peers: HashMap<String, PeerStats>,
}
#[derive(Clone, Copy, Default, Deserialize)]
pub enum PeerStatsFilterState {
All,
#[default]
Live,
}
impl PeerStatsFilterState {
pub(crate) fn matches(&self, s: &PeerState) -> bool {
matches!((self, s), (Self::All, _) | (Self::Live, PeerState::Live(_)))
}
}
#[derive(Default, Deserialize)]
pub struct PeerStatsFilter {
pub state: PeerStatsFilterState,
}

107
crates/librqbit/src/torrent_state/live/peers/mod.rs Normal file
View file

@ -0,0 +1,107 @@
use std::net::SocketAddr;
use anyhow::Context;
use backoff::backoff::Backoff;
use dashmap::DashMap;
use crate::{
torrent_state::utils::{atomic_inc, TimedExistence},
type_aliases::{PeerHandle, BF},
};
use self::stats::{atomic::AggregatePeerStatsAtomic, snapshot::AggregatePeerStats};
use super::peer::{LivePeerState, Peer, PeerRx, PeerState, PeerTx};
pub mod stats;
#[derive(Default)]
pub(crate) struct PeerStates {
pub stats: AggregatePeerStatsAtomic,
pub states: DashMap<PeerHandle, Peer>,
}
impl PeerStates {
pub fn stats(&self) -> AggregatePeerStats {
AggregatePeerStats::from(&self.stats)
}
pub fn add_if_not_seen(&self, addr: SocketAddr) -> Option<PeerHandle> {
use dashmap::mapref::entry::Entry;
match self.states.entry(addr) {
Entry::Occupied(_) => None,
Entry::Vacant(vac) => {
vac.insert(Default::default());
atomic_inc(&self.stats.queued);
atomic_inc(&self.stats.seen);
Some(addr)
}
}
}
pub fn with_peer<R>(&self, addr: PeerHandle, f: impl FnOnce(&Peer) -> R) -> Option<R> {
self.states.get(&addr).map(|e| f(e.value()))
}
pub fn with_peer_mut<R>(
&self,
addr: PeerHandle,
reason: &'static str,
f: impl FnOnce(&mut Peer) -> R,
) -> Option<R> {
use crate::torrent_state::utils::timeit;
timeit(reason, || self.states.get_mut(&addr))
.map(|e| f(TimedExistence::new(e, reason).value_mut()))
}
pub fn with_live_mut<R>(
&self,
addr: PeerHandle,
reason: &'static str,
f: impl FnOnce(&mut LivePeerState) -> R,
) -> Option<R> {
self.with_peer_mut(addr, reason, |peer| peer.state.get_live_mut().map(f))
.flatten()
}
pub fn drop_peer(&self, handle: PeerHandle) -> Option<Peer> {
let p = self.states.remove(&handle).map(|r| r.1)?;
self.stats.dec(p.state.get());
Some(p)
}
pub fn mark_peer_interested(&self, handle: PeerHandle, is_interested: bool) -> Option<bool> {
self.with_live_mut(handle, "mark_peer_interested", |live| {
let prev = live.peer_interested;
live.peer_interested = is_interested;
prev
})
}
pub fn update_bitfield_from_vec(&self, handle: PeerHandle, bitfield: Vec<u8>) -> Option<()> {
self.with_live_mut(handle, "update_bitfield_from_vec", |live| {
live.bitfield = BF::from_vec(bitfield);
})
}
pub fn mark_peer_connecting(&self, h: PeerHandle) -> anyhow::Result<(PeerRx, PeerTx)> {
let rx = self
.with_peer_mut(h, "mark_peer_connecting", |peer| {
peer.state
.queued_to_connecting(&self.stats)
.context("invalid peer state")
})
.context("peer not found in states")??;
Ok(rx)
}
pub fn reset_peer_backoff(&self, handle: PeerHandle) {
self.with_peer_mut(handle, "reset_peer_backoff", |p| {
p.stats.backoff.reset();
});
}
pub fn mark_peer_not_needed(&self, handle: PeerHandle) -> Option<PeerState> {
let prev = self.with_peer_mut(handle, "mark_peer_not_needed", |peer| {
peer.state.set_not_needed(&self.stats)
})?;
Some(prev)
}
}

43
crates/librqbit/src/torrent_state/live/peers/stats/atomic.rs Normal file
View file

@ -0,0 +1,43 @@
use std::sync::atomic::AtomicU32;
use serde::Serialize;
use crate::torrent_state::{
live::peer::PeerState,
utils::{atomic_dec, atomic_inc},
};
#[derive(Debug, Default, Serialize)]
pub(crate) struct AggregatePeerStatsAtomic {
pub queued: AtomicU32,
pub connecting: AtomicU32,
pub live: AtomicU32,
pub seen: AtomicU32,
pub dead: AtomicU32,
pub not_needed: AtomicU32,
}
impl AggregatePeerStatsAtomic {
pub fn counter(&self, state: &PeerState) -> &AtomicU32 {
match state {
PeerState::Connecting(_) => &self.connecting,
PeerState::Live(_) => &self.live,
PeerState::Queued => &self.queued,
PeerState::Dead => &self.dead,
PeerState::NotNeeded => &self.not_needed,
}
}
pub fn inc(&self, state: &PeerState) {
atomic_inc(self.counter(state));
}
pub fn dec(&self, state: &PeerState) {
atomic_dec(self.counter(state));
}
pub fn incdec(&self, old: &PeerState, new: &PeerState) {
self.dec(old);
self.inc(new);
}
}

2
crates/librqbit/src/torrent_state/live/peers/stats/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod atomic;
pub mod snapshot;

29
crates/librqbit/src/torrent_state/live/peers/stats/snapshot.rs Normal file
View file

@ -0,0 +1,29 @@
use std::sync::atomic::Ordering;
use serde::Serialize;
use super::atomic::AggregatePeerStatsAtomic;
#[derive(Debug, Default, Serialize, PartialEq, Eq)]
pub struct AggregatePeerStats {
pub queued: usize,
pub connecting: usize,
pub live: usize,
pub seen: usize,
pub dead: usize,
pub not_needed: usize,
}
impl<'a> From<&'a AggregatePeerStatsAtomic> for AggregatePeerStats {
fn from(s: &'a AggregatePeerStatsAtomic) -> Self {
let ordering = Ordering::Relaxed;
Self {
queued: s.queued.load(ordering) as usize,
connecting: s.connecting.load(ordering) as usize,
live: s.live.load(ordering) as usize,
seen: s.seen.load(ordering) as usize,
dead: s.dead.load(ordering) as usize,
not_needed: s.not_needed.load(ordering) as usize,
}
}
}

25
crates/librqbit/src/torrent_state/live/stats/atomic.rs Normal file
View file

@ -0,0 +1,25 @@
use std::{
sync::atomic::{AtomicU64, Ordering},
time::Duration,
};
#[derive(Default, Debug)]
pub struct AtomicStats {
pub have_bytes: AtomicU64,
pub downloaded_and_checked_bytes: AtomicU64,
pub downloaded_and_checked_pieces: AtomicU64,
pub uploaded_bytes: AtomicU64,
pub fetched_bytes: AtomicU64,
pub total_piece_download_ms: AtomicU64,
}
impl AtomicStats {
pub fn average_piece_download_time(&self) -> Option<Duration> {
let d = self.downloaded_and_checked_pieces.load(Ordering::Acquire);
let t = self.total_piece_download_ms.load(Ordering::Acquire);
if d == 0 {
return None;
}
Some(Duration::from_secs_f64(t as f64 / d as f64 / 1000f64))
}
}

2
crates/librqbit/src/torrent_state/live/stats/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod atomic;
pub mod snapshot;

30
crates/librqbit/src/torrent_state/live/stats/snapshot.rs Normal file
View file

@ -0,0 +1,30 @@
use std::time::Duration;
use serde::Serialize;
use crate::torrent_state::live::peers::stats::snapshot::AggregatePeerStats;
#[derive(Debug, Serialize, Default)]
pub struct StatsSnapshot {
pub have_bytes: u64,
pub downloaded_and_checked_bytes: u64,
pub downloaded_and_checked_pieces: u64,
pub fetched_bytes: u64,
pub uploaded_bytes: u64,
pub initially_needed_bytes: u64,
pub remaining_bytes: u64,
pub total_bytes: u64,
pub total_piece_download_ms: u64,
pub peer_stats: AggregatePeerStats,
}
impl StatsSnapshot {
pub fn average_piece_download_time(&self) -> Option<Duration> {
let d = self.downloaded_and_checked_pieces;
let t = self.total_piece_download_ms;
if d == 0 {
return None;
}
Some(Duration::from_secs_f64(t as f64 / d as f64 / 1000f64))
}
}

490
crates/librqbit/src/torrent_state/mod.rs Normal file
View file

@ -0,0 +1,490 @@
pub mod initializing;
pub mod live;
pub mod paused;
pub mod stats;
pub mod utils;
use std::collections::HashSet;
use std::net::SocketAddr;
use std::path::Path;
use std::path::PathBuf;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
use anyhow::bail;
use anyhow::Context;
use buffers::ByteString;
use librqbit_core::id20::Id20;
use librqbit_core::lengths::Lengths;
use librqbit_core::peer_id::generate_peer_id;
use librqbit_core::torrent_metainfo::TorrentMetaV1Info;
pub use live::*;
use parking_lot::RwLock;
use tokio_stream::StreamExt;
use tracing::debug;
use tracing::error;
use tracing::error_span;
use tracing::warn;
use url::Url;
use crate::chunk_tracker::ChunkTracker;
use crate::spawn_utils::spawn;
use crate::spawn_utils::BlockingSpawner;
use crate::torrent_state::stats::LiveStats;
use initializing::TorrentStateInitializing;
use self::paused::TorrentStatePaused;
use self::stats::TorrentStats;
pub enum ManagedTorrentState {
Initializing(Arc<TorrentStateInitializing>),
Paused(TorrentStatePaused),
Live(Arc<TorrentStateLive>),
Error(anyhow::Error),
// This is used when swapping between states, outside world should never see it.
None,
}
impl ManagedTorrentState {
fn assert_paused(self) -> TorrentStatePaused {
match self {
Self::Paused(paused) => paused,
_ => panic!("Expected paused state"),
}
}
pub(crate) fn take(&mut self) -> Self {
std::mem::replace(self, Self::None)
}
}
pub(crate) struct ManagedTorrentLocked {
pub state: ManagedTorrentState,
}
#[derive(Default)]
pub(crate) struct ManagedTorrentOptions {
pub force_tracker_interval: Option<Duration>,
pub peer_connect_timeout: Option<Duration>,
pub peer_read_write_timeout: Option<Duration>,
pub overwrite: bool,
}
pub struct ManagedTorrentInfo {
pub info: TorrentMetaV1Info<ByteString>,
pub info_hash: Id20,
pub out_dir: PathBuf,
pub spawner: BlockingSpawner,
pub trackers: HashSet<Url>,
pub peer_id: Id20,
pub lengths: Lengths,
pub span: tracing::Span,
pub(crate) options: ManagedTorrentOptions,
}
pub struct ManagedTorrent {
pub info: Arc<ManagedTorrentInfo>,
pub(crate) only_files: Option<Vec<usize>>,
locked: RwLock<ManagedTorrentLocked>,
}
impl ManagedTorrent {
pub fn info(&self) -> &ManagedTorrentInfo {
&self.info
}
pub fn get_total_bytes(&self) -> u64 {
self.info.lengths.total_length()
}
pub fn info_hash(&self) -> Id20 {
self.info.info_hash
}
pub fn only_files(&self) -> Option<Vec<usize>> {
self.only_files.clone()
}
pub fn with_state<R>(&self, f: impl FnOnce(&ManagedTorrentState) -> R) -> R {
f(&self.locked.read().state)
}
pub(crate) fn with_state_mut<R>(&self, f: impl FnOnce(&mut ManagedTorrentState) -> R) -> R {
f(&mut self.locked.write().state)
}
pub fn with_chunk_tracker<R>(&self, f: impl FnOnce(&ChunkTracker) -> R) -> anyhow::Result<R> {
let g = self.locked.read();
match &g.state {
ManagedTorrentState::Paused(p) => Ok(f(&p.chunk_tracker)),
ManagedTorrentState::Live(l) => Ok(f(l
.lock_read("chunk_tracker")
.get_chunks()
.context("error getting chunks")?)),
_ => bail!("no chunk tracker, torrent neither paused nor live"),
}
}
pub fn live(&self) -> Option<Arc<TorrentStateLive>> {
let g = self.locked.read();
match &g.state {
ManagedTorrentState::Live(live) => Some(live.clone()),
_ => None,
}
}
fn stop_with_error(&self, error: anyhow::Error) {
let mut g = self.locked.write();
match g.state.take() {
ManagedTorrentState::Live(live) => {
if let Err(err) = live.pause() {
warn!(
"error pausing live torrent during fatal error handling: {:?}",
err
);
}
}
ManagedTorrentState::Error(e) => {
warn!("bug: torrent already was in error state when trying to stop it. Previous error was: {:?}", e);
}
ManagedTorrentState::None => {
warn!("bug: torrent encountered in None state during fatal error handling")
}
_ => {}
};
g.state = ManagedTorrentState::Error(error)
}
pub fn start(
self: &Arc<Self>,
initial_peers: Vec<SocketAddr>,
peer_rx: Option<impl StreamExt<Item = SocketAddr> + Unpin + Send + Sync + 'static>,
start_paused: bool,
) -> anyhow::Result<()> {
let mut g = self.locked.write();
let spawn_fatal_errors_receiver =
|state: &Arc<Self>, rx: tokio::sync::oneshot::Receiver<anyhow::Error>| {
let span = state.info.span.clone();
let state = Arc::downgrade(state);
spawn(
"fatal_errors_receiver",
error_span!(parent: span, "fatal_errors_receiver"),
async move {
let e = match rx.await {
Ok(e) => e,
Err(_) => return Ok(()),
};
if let Some(state) = state.upgrade() {
state.stop_with_error(e);
} else {
warn!("tried to stop the torrent with error, but it's couldn't upgrade the arc");
}
Ok(())
},
);
};
fn spawn_peer_adder(
live: &Arc<TorrentStateLive>,
initial_peers: Vec<SocketAddr>,
peer_rx: Option<impl StreamExt<Item = SocketAddr> + Unpin + Send + Sync + 'static>,
) {
let span = live.meta().span.clone();
let live = Arc::downgrade(live);
spawn(
"external_peer_adder",
error_span!(parent: span, "external_peer_adder"),
async move {
{
let live: Arc<TorrentStateLive> =
live.upgrade().context("no longer live")?;
for peer in initial_peers {
live.add_peer_if_not_seen(peer).context("torrent closed")?;
}
}
if let Some(mut peer_rx) = peer_rx {
while let Some(peer) = peer_rx.next().await {
let live = match live.upgrade() {
Some(live) => live,
None => return Ok(()),
};
live.add_peer_if_not_seen(peer).context("torrent closed")?;
}
} else {
error!("peer rx is not set");
}
Ok(())
},
);
}
match &g.state {
ManagedTorrentState::Live(_) => {
bail!("torrent is already live");
}
ManagedTorrentState::Initializing(init) => {
let init = init.clone();
drop(g);
let t = self.clone();
let span = self.info().span.clone();
spawn(
"initialize_and_start",
error_span!(parent: span.clone(), "initialize_and_start"),
async move {
match init.check().await {
Ok(paused) => {
let mut g = t.locked.write();
if let ManagedTorrentState::Initializing(_) = &g.state {
} else {
debug!("no need to start torrent anymore, as it switched state from initilizing");
return Ok(());
}
if start_paused {
g.state = ManagedTorrentState::Paused(paused);
return Ok(());
}
let (tx, rx) = tokio::sync::oneshot::channel();
let live = TorrentStateLive::new(paused, tx);
g.state = ManagedTorrentState::Live(live.clone());
spawn_fatal_errors_receiver(&t, rx);
spawn_peer_adder(&live, initial_peers, peer_rx);
Ok(())
}
Err(err) => {
let result = anyhow::anyhow!("{:?}", err);
t.locked.write().state = ManagedTorrentState::Error(err);
Err(result)
}
}
},
);
Ok(())
}
ManagedTorrentState::Paused(_) => {
let paused = g.state.take().assert_paused();
let (tx, rx) = tokio::sync::oneshot::channel();
let live = TorrentStateLive::new(paused, tx);
g.state = ManagedTorrentState::Live(live.clone());
spawn_fatal_errors_receiver(self, rx);
spawn_peer_adder(&live, initial_peers, peer_rx);
Ok(())
}
ManagedTorrentState::Error(_) => {
let initializing = Arc::new(TorrentStateInitializing::new(
self.info.clone(),
self.only_files.clone(),
));
g.state = ManagedTorrentState::Initializing(initializing.clone());
drop(g);
// Recurse.
self.start(initial_peers, peer_rx, start_paused)
}
ManagedTorrentState::None => bail!("bug: torrent is in empty state"),
}
}
pub fn pause(&self) -> anyhow::Result<()> {
let mut g = self.locked.write();
match &g.state {
ManagedTorrentState::Live(live) => {
let paused = live.pause()?;
g.state = ManagedTorrentState::Paused(paused);
Ok(())
}
ManagedTorrentState::Initializing(_) => {
bail!("torrent is initializing, can't pause");
}
ManagedTorrentState::Paused(_) => {
bail!("torrent is already paused");
}
ManagedTorrentState::Error(_) => {
bail!("can't pause torrent in error state")
}
ManagedTorrentState::None => bail!("bug: torrent is in empty state"),
}
}
pub fn stats(&self) -> TorrentStats {
let mut resp = TorrentStats {
total_bytes: self.info().lengths.total_length(),
state: "",
error: None,
progress_bytes: 0,
finished: false,
live: None,
};
self.with_state(|s| {
match s {
ManagedTorrentState::Initializing(i) => {
resp.state = "initializing";
resp.progress_bytes = i.checked_bytes.load(Ordering::Relaxed);
}
ManagedTorrentState::Paused(p) => {
resp.state = "paused";
resp.progress_bytes = p.have_bytes;
resp.finished = p.have_bytes == resp.total_bytes;
}
ManagedTorrentState::Live(l) => {
resp.state = "live";
let live_stats = LiveStats::from(l.as_ref());
resp.progress_bytes = live_stats.snapshot.have_bytes;
resp.finished = resp.progress_bytes == resp.total_bytes;
resp.live = Some(live_stats);
}
ManagedTorrentState::Error(e) => {
resp.state = "error";
resp.error = Some(format!("{:?}", e))
}
ManagedTorrentState::None => {
resp.state = "error";
resp.error = Some("bug: torrent in broken \"None\" state".to_string());
}
}
resp
})
}
pub async fn wait_until_completed(&self) -> anyhow::Result<()> {
// TODO: rewrite, this polling is horrible
let live = loop {
let live = self.with_state(|s| match s {
ManagedTorrentState::Initializing(_) | ManagedTorrentState::Paused(_) => Ok(None),
ManagedTorrentState::Live(l) => Ok(Some(l.clone())),
ManagedTorrentState::Error(e) => bail!("{:?}", e),
ManagedTorrentState::None => bail!("bug: torrent state is None"),
})?;
if let Some(live) = live {
break live;
}
tokio::time::sleep(Duration::from_secs(1)).await;
};
live.wait_until_completed().await;
Ok(())
}
}
pub struct ManagedTorrentBuilder {
info: TorrentMetaV1Info<ByteString>,
info_hash: Id20,
output_folder: PathBuf,
force_tracker_interval: Option<Duration>,
peer_connect_timeout: Option<Duration>,
peer_read_write_timeout: Option<Duration>,
only_files: Option<Vec<usize>>,
trackers: Vec<Url>,
peer_id: Option<Id20>,
overwrite: bool,
spawner: Option<BlockingSpawner>,
}
impl ManagedTorrentBuilder {
pub fn new<P: AsRef<Path>>(
info: TorrentMetaV1Info<ByteString>,
info_hash: Id20,
output_folder: P,
) -> Self {
Self {
info,
info_hash,
output_folder: output_folder.as_ref().into(),
spawner: None,
force_tracker_interval: None,
peer_connect_timeout: None,
peer_read_write_timeout: None,
only_files: None,
trackers: Default::default(),
peer_id: None,
overwrite: false,
}
}
pub fn only_files(&mut self, only_files: Vec<usize>) -> &mut Self {
self.only_files = Some(only_files);
self
}
pub fn trackers(&mut self, trackers: Vec<Url>) -> &mut Self {
self.trackers = trackers;
self
}
pub fn overwrite(&mut self, overwrite: bool) -> &mut Self {
self.overwrite = overwrite;
self
}
pub fn force_tracker_interval(&mut self, force_tracker_interval: Duration) -> &mut Self {
self.force_tracker_interval = Some(force_tracker_interval);
self
}
pub fn spawner(&mut self, spawner: BlockingSpawner) -> &mut Self {
self.spawner = Some(spawner);
self
}
pub fn peer_id(&mut self, peer_id: Id20) -> &mut Self {
self.peer_id = Some(peer_id);
self
}
pub fn peer_connect_timeout(&mut self, timeout: Duration) -> &mut Self {
self.peer_connect_timeout = Some(timeout);
self
}
pub fn peer_read_write_timeout(&mut self, timeout: Duration) -> &mut Self {
self.peer_read_write_timeout = Some(timeout);
self
}
pub(crate) fn build(self, span: tracing::Span) -> anyhow::Result<ManagedTorrentHandle> {
let lengths = Lengths::from_torrent(&self.info)?;
let info = Arc::new(ManagedTorrentInfo {
span,
info: self.info,
info_hash: self.info_hash,
out_dir: self.output_folder,
trackers: self.trackers.into_iter().collect(),
spawner: self.spawner.unwrap_or_default(),
peer_id: self.peer_id.unwrap_or_else(generate_peer_id),
lengths,
options: ManagedTorrentOptions {
force_tracker_interval: self.force_tracker_interval,
peer_connect_timeout: self.peer_connect_timeout,
peer_read_write_timeout: self.peer_read_write_timeout,
overwrite: self.overwrite,
},
});
let initializing = Arc::new(TorrentStateInitializing::new(
info.clone(),
self.only_files.clone(),
));
Ok(Arc::new(ManagedTorrent {
only_files: self.only_files,
locked: RwLock::new(ManagedTorrentLocked {
state: ManagedTorrentState::Initializing(initializing),
}),
info,
}))
}
}
pub type ManagedTorrentHandle = Arc<ManagedTorrent>;

24
crates/librqbit/src/torrent_state/paused.rs Normal file
View file

@ -0,0 +1,24 @@
use std::{fs::File, path::PathBuf, sync::Arc};
use parking_lot::Mutex;
use crate::chunk_tracker::ChunkTracker;
use super::ManagedTorrentInfo;
pub struct TorrentStatePaused {
pub(crate) info: Arc<ManagedTorrentInfo>,
pub(crate) files: Vec<Arc<Mutex<File>>>,
pub(crate) filenames: Vec<PathBuf>,
pub(crate) chunk_tracker: ChunkTracker,
pub(crate) have_bytes: u64,
}
// impl TorrentStatePaused {
// pub fn get_have_bytes(&self) -> u64 {
// self.have_bytes
// }
// pub fn get_needed_bytes(&self) -> u64 {
// self.needed_bytes
// }
// }

198
crates/librqbit/src/torrent_state/stats.rs Normal file
View file

@ -0,0 +1,198 @@
use std::time::Duration;
use serde::Serialize;
use super::{live::stats::snapshot::StatsSnapshot, TorrentStateLive};
use size_format::SizeFormatterBinary as SF;
#[derive(Serialize, Default, Debug)]
pub struct LiveStats {
pub snapshot: StatsSnapshot,
pub average_piece_download_time: Option<Duration>,
pub download_speed: Speed,
pub time_remaining: Option<DurationWithHumanReadable>,
}
impl std::fmt::Display for LiveStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "down speed: {}", self.download_speed)?;
if let Some(time_remaining) = &self.time_remaining {
write!(f, " eta: {time_remaining}")?;
}
Ok(())
}
}
impl From<&TorrentStateLive> for LiveStats {
fn from(live: &TorrentStateLive) -> Self {
let snapshot = live.stats_snapshot();
let estimator = live.speed_estimator();
Self {
average_piece_download_time: snapshot.average_piece_download_time(),
snapshot,
download_speed: estimator.download_mbps().into(),
time_remaining: estimator.time_remaining().map(DurationWithHumanReadable),
}
}
}
#[derive(Serialize, Debug)]
pub struct TorrentStats {
pub state: &'static str,
pub error: Option<String>,
pub progress_bytes: u64,
pub total_bytes: u64,
pub finished: bool,
pub live: Option<LiveStats>,
}
impl std::fmt::Display for TorrentStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: ", self.state)?;
if let Some(error) = &self.error {
return write!(f, "{error}");
}
write!(
f,
"{} ({})",
self.progress_percent_human_readable(),
self.progress_bytes_human_readable()
)?;
if let Some(live) = &self.live {
write!(f, " [{live}]")?;
}
Ok(())
}
}
impl TorrentStats {
pub fn progress_percent_human_readable(&self) -> impl std::fmt::Display {
struct Percents {
progress: u64,
total: u64,
}
impl std::fmt::Display for Percents {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.total == 0 {
return write!(f, "N/A");
}
let pct = self.progress as f64 / self.total as f64 * 100f64;
write!(f, "{pct:.2}%")
}
}
Percents {
progress: self.progress_bytes,
total: self.total_bytes,
}
}
pub fn progress_bytes_human_readable(&self) -> impl std::fmt::Display {
struct Progress {
progress: u64,
total: u64,
}
impl std::fmt::Display for Progress {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} / {}", SF::new(self.progress), SF::new(self.total))
}
}
Progress {
progress: self.progress_bytes,
total: self.total_bytes,
}
}
}
fn format_seconds_to_time(seconds: u64, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let hours = seconds / 3600;
let minutes = (seconds % 3600) / 60;
let seconds = seconds % 60;
if hours > 0 {
write!(f, "{}h {}m", hours, minutes)
} else if minutes > 0 {
write!(f, "{}m {}s", minutes, seconds)
} else {
write!(f, "{}s", seconds)
}
}
pub struct DurationWithHumanReadable(Duration);
impl core::fmt::Display for DurationWithHumanReadable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> core::fmt::Result {
format_seconds_to_time(self.0.as_secs(), f)
}
}
impl core::fmt::Debug for DurationWithHumanReadable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self)
}
}
impl Serialize for DurationWithHumanReadable {
fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
struct Tmp {
duration: Duration,
human_readable: String,
}
Tmp {
duration: self.0,
human_readable: format!("{}", self),
}
.serialize(serializer)
}
}
#[derive(Default)]
pub struct Speed {
pub mbps: f64,
}
impl Speed {
fn new(mbps: f64) -> Self {
Self { mbps }
}
}
impl From<f64> for Speed {
fn from(mbps: f64) -> Self {
Self::new(mbps)
}
}
impl core::fmt::Display for Speed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:.2} MiB/s", self.mbps)
}
}
impl core::fmt::Debug for Speed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self)
}
}
impl Serialize for Speed {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
struct Tmp {
mbps: f64,
human_readable: String,
}
Tmp {
mbps: self.mbps,
human_readable: format!("{}", self),
}
.serialize(serializer)
}
}

108
crates/librqbit/src/torrent_state/utils.rs Normal file
View file

@ -0,0 +1,108 @@
use std::sync::atomic::{AtomicU32, Ordering};
pub fn atomic_inc(c: &AtomicU32) -> u32 {
c.fetch_add(1, Ordering::Relaxed)
}
pub fn atomic_dec(c: &AtomicU32) -> u32 {
c.fetch_sub(1, Ordering::Relaxed)
}
// Used during debugging to see if some locks take too long.
#[cfg(not(feature = "timed_existence"))]
mod timed_existence {
use std::ops::{Deref, DerefMut};
pub struct TimedExistence<T>(T);
impl<T> TimedExistence<T> {
#[inline(always)]
pub fn new(object: T, _reason: &'static str) -> Self {
Self(object)
}
}
impl<T> Deref for TimedExistence<T> {
type Target = T;
#[inline(always)]
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> DerefMut for TimedExistence<T> {
#[inline(always)]
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
#[inline(always)]
pub fn timeit<R>(_n: impl std::fmt::Display, f: impl FnOnce() -> R) -> R {
f()
}
}
#[cfg(feature = "timed_existence")]
mod timed_existence {
use std::ops::{Deref, DerefMut};
use std::time::{Duration, Instant};
use tracing::warn;
const MAX: Duration = Duration::from_millis(1);
// Prints if the object exists for too long.
// This is used to track long-lived locks for debugging.
pub struct TimedExistence<T> {
object: T,
reason: &'static str,
started: Instant,
}
impl<T> TimedExistence<T> {
pub fn new(object: T, reason: &'static str) -> Self {
Self {
object,
reason,
started: Instant::now(),
}
}
}
impl<T> Drop for TimedExistence<T> {
fn drop(&mut self) {
let elapsed = self.started.elapsed();
let reason = self.reason;
if elapsed > MAX {
warn!("elapsed on lock {reason:?}: {elapsed:?}")
}
}
}
impl<T> Deref for TimedExistence<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.object
}
}
impl<T> DerefMut for TimedExistence<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.object
}
}
pub fn timeit<R>(name: impl std::fmt::Display, f: impl FnOnce() -> R) -> R {
let now = Instant::now();
let r = f();
let elapsed = now.elapsed();
if elapsed > MAX {
warn!("elapsed on \"{name:}\": {elapsed:?}")
}
r
}
}
pub use timed_existence::{timeit, TimedExistence};

20
crates/librqbit/webui/dist/app.js vendored
View file

File diff suppressed because one or more lines are too long

2
crates/librqbit/webui/dist/index.html vendored
View file

@ -8,6 +8,8 @@
<!-- Include Bootstrap CSS -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"
integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css"
integrity="sha384-4LISF5TTJX/fLmGSxO53rV4miRxdg84mZsxmO8Rx5jGtp/LbrixFETvWa5a6sESd" crossorigin="anonymous">
<script type="module" crossorigin src="app.js"></script>
</head>

2
crates/librqbit/webui/index.html
View file

@ -8,6 +8,8 @@
<!-- Include Bootstrap CSS -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css"
integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css"
integrity="sha384-4LISF5TTJX/fLmGSxO53rV4miRxdg84mZsxmO8Rx5jGtp/LbrixFETvWa5a6sESd" crossorigin="anonymous">
</head>
<body>

34
crates/librqbit/webui/src/api.ts
View file

@ -29,7 +29,7 @@ export interface ListTorrentsResponse {
}
// Interface for the Torrent Stats API response
export interface TorrentStats {
export interface LiveTorrentStats {
snapshot: {
have_bytes: number;
downloaded_and_checked_bytes: number;
@ -69,6 +69,20 @@ export interface TorrentStats {
} | null;
}
export const STATE_INITIALIZING = 'initializing';
export const STATE_PAUSED = 'paused';
export const STATE_LIVE = 'live';
export const STATE_ERROR = 'error';
export interface TorrentStats {
state: 'initializing' | 'paused' | 'live' | 'error',
error: string | null,
progress_bytes: number,
finished: boolean,
total_bytes: number,
live: LiveTorrentStats | null;
}
export interface ErrorDetails {
id?: number,
@ -129,7 +143,7 @@ export const API = {
return makeRequest('GET', `/torrents/${index}`);
},
getTorrentStats: (index: number): Promise<TorrentStats> => {
return makeRequest('GET', `/torrents/${index}/stats`);
return makeRequest('GET', `/torrents/${index}/stats/v1`);
},
uploadTorrent: (data: string | File, opts?: {
@ -144,5 +158,21 @@ export const API = {
url += `&only_files=${opts.selectedFiles.join(',')}`;
}
return makeRequest('POST', url, data)
},
pause: (index: number): Promise<void> => {
return makeRequest('POST', `/torrents/${index}/pause`);
},
start: (index: number): Promise<void> => {
return makeRequest('POST', `/torrents/${index}/start`);
},
forget: (index: number): Promise<void> => {
return makeRequest('POST', `/torrents/${index}/forget`);
},
delete: (index: number): Promise<void> => {
return makeRequest('POST', `/torrents/${index}/delete`);
}
}

251
crates/librqbit/webui/src/index.tsx
View file

@ -1,7 +1,7 @@
import { StrictMode, createContext, useContext, useEffect, useRef, useState } from 'react';
import { MouseEventHandler, StrictMode, createContext, useContext, useEffect, useRef, useState } from 'react';
import ReactDOM from 'react-dom/client';
import { ProgressBar, Button, Container, Row, Col, Alert, Modal, Form, Spinner, Table } from 'react-bootstrap';
import { AddTorrentResponse, TorrentDetails, TorrentFile, TorrentId, TorrentStats, ErrorDetails, API } from './api';
import { AddTorrentResponse, TorrentDetails, TorrentFile, TorrentId, TorrentStats, ErrorDetails, API, STATE_INITIALIZING, STATE_LIVE, STATE_PAUSED, STATE_ERROR } from './api';
interface Error {
text: string,
@ -14,33 +14,214 @@ interface ContextType {
}
const AppContext = createContext<ContextType>(null);
const RefreshTorrentStatsContext = createContext<{ refresh: () => void }>(null);
const IconButton: React.FC<{
className: string,
onClick: () => void,
disabled?: boolean,
color?: string,
}> = ({ className, onClick, disabled, color }) => {
const onClickStopPropagation = (e) => {
e.stopPropagation();
if (disabled) {
return;
}
onClick();
}
return <a className={`bi ${className} p-1`} onClick={onClickStopPropagation} href='#'></a>
}
const DeleteTorrentModal = ({ id, show, onHide }) => {
if (!show) {
return null;
}
const [deleteFiles, setDeleteFiles] = useState(false);
const [error, setError] = useState<Error>(null);
const [deleting, setDeleting] = useState(false);
const ctx = useContext(AppContext);
const close = () => {
setDeleteFiles(false);
setError(null);
setDeleting(false);
onHide();
}
const deleteTorrent = () => {
setDeleting(true);
const call = deleteFiles ? API.delete : API.forget;
call(id).then(() => {
ctx.refreshTorrents();
close();
}).catch((e) => {
setError({
text: `Error deleting torrent id=${id}`,
details: e,
});
setDeleting(false);
})
}
return <Modal show={show} onHide={close}>
<Modal.Header closeButton>
Delete torrent
</Modal.Header>
<Modal.Body>
<Form>
<Form.Group controlId='delete-torrent'>
<Form.Check
type="checkbox"
label='Also delete files'
checked={deleteFiles}
onChange={() => setDeleteFiles(!deleteFiles)}>
</Form.Check>
</Form.Group>
</Form>
{error && <ErrorComponent error={error} />}
</Modal.Body>
<Modal.Footer>
{deleting && <Spinner />}
<Button variant="primary" onClick={deleteTorrent} disabled={deleting}>
OK
</Button>
<Button variant="secondary" onClick={close}>
Cancel
</Button>
</Modal.Footer>
</Modal>
}
const TorrentActions: React.FC<{
id: number, statsResponse: TorrentStats
}> = ({ id, statsResponse }) => {
let state = statsResponse.state;
let [disabled, setDisabled] = useState<boolean>(false);
let [deleting, setDeleting] = useState<boolean>(false);
let refreshCtx = useContext(RefreshTorrentStatsContext);
const canPause = state == 'live';
const canUnpause = state == 'paused' || state == 'error';
const ctx = useContext(AppContext);
const unpause = () => {
setDisabled(true);
API.start(id).then(() => { refreshCtx.refresh() }, (e) => {
ctx.setCloseableError({
text: `Error starting torrent id=${id}`,
details: e,
});
}).finally(() => setDisabled(false))
};
const pause = () => {
setDisabled(true);
API.pause(id).then(() => { refreshCtx.refresh() }, (e) => {
ctx.setCloseableError({
text: `Error pausing torrent id=${id}`,
details: e,
});
}).finally(() => setDisabled(false))
};
const startDeleting = () => {
setDisabled(true);
setDeleting(true);
}
const cancelDeleting = () => {
setDisabled(false);
setDeleting(false);
}
return <Row>
<Col>
{canUnpause && <IconButton className="bi-play-circle" onClick={unpause} disabled={disabled} color='success' />}
{canPause && <IconButton className="bi-pause-circle" onClick={pause} disabled={disabled} />}
<IconButton className="bi-x-circle" onClick={startDeleting} disabled={disabled} color='danger' />
<DeleteTorrentModal id={id} show={deleting} onHide={cancelDeleting} />
</Col>
</Row>
}
const TorrentRow: React.FC<{
id: number, detailsResponse: TorrentDetails, statsResponse: TorrentStats
}> = ({ id, detailsResponse, statsResponse }) => {
const totalBytes = statsResponse?.snapshot?.total_bytes ?? 1;
const downloadedBytes = statsResponse?.snapshot?.have_bytes ?? 0;
const finished = totalBytes == downloadedBytes;
const downloadPercentage = (downloadedBytes / totalBytes) * 100;
const state = statsResponse?.state ?? "";
const error = statsResponse?.error;
const totalBytes = statsResponse?.total_bytes ?? 1;
const progressBytes = statsResponse?.progress_bytes ?? 0;
const finished = statsResponse?.finished || false;
const progressPercentage = error ? 100 : (progressBytes / totalBytes) * 100;
const isAnimated = (state == STATE_INITIALIZING || state == STATE_LIVE) && !finished;
const progressLabel = error ? 'Error' : `${progressPercentage.toFixed(2)}%`;
const progressBarVariant = error ? 'danger' : finished ? 'success' : state == STATE_INITIALIZING ? 'warning' : 'primary';
const formatPeersString = () => {
let peer_stats = statsResponse?.live?.snapshot.peer_stats;
if (!peer_stats) {
return '';
}
return `${peer_stats.live} / ${peer_stats.seen}`;
}
const formatDownloadSpeed = () => {
if (finished) {
return 'Completed';
}
switch (state) {
case STATE_PAUSED: return 'Paused';
case STATE_INITIALIZING: return 'Checking files';
case STATE_ERROR: return 'Error';
}
return statsResponse.live?.download_speed.human_readable ?? "N/A";
}
let classNames = [];
if (error) {
classNames.push('bg-warning');
} else {
if (id % 2 == 0) {
classNames.push('bg-light');
}
}
return (
<Row className={`${id % 2 == 0 ? 'bg-light' : ''}`}>
<Column size={4} label="Name">
<Row className={classNames.join(' ')}>
<Column size={3} label="Name">
{detailsResponse ?
<div className='text-truncate'>
{getLargestFileName(detailsResponse)}
</div>
<>
<div className='text-truncate'>
{getLargestFileName(detailsResponse)}
</div>
{error && <p className='text-danger'><strong>Error:</strong> {error}</p>}
</>
: <Spinner />}
</Column>
{statsResponse ?
<>
<Column label="Size">{`${formatBytes(totalBytes)} `}</Column>
<Column size={2} label="Progress">
<ProgressBar now={downloadPercentage} label={`${downloadPercentage.toFixed(2)}% `} animated={!finished} />
<Column size={2} label={state == STATE_PAUSED ? 'Progress' : 'Progress'}>
<ProgressBar
now={progressPercentage}
label={progressLabel}
animated={isAnimated}
variant={progressBarVariant} />
</Column>
<Column size={2} label="Down Speed">{statsResponse.download_speed.human_readable}</Column>
<Column size={2} label="Down Speed">{formatDownloadSpeed()}</Column>
<Column label="ETA">{getCompletionETA(statsResponse)}</Column>
<Column size={2} label="Peers">{`${statsResponse.snapshot.peer_stats.live} / ${statsResponse.snapshot.peer_stats.seen}`}</Column >
<Column size={2} label="Peers">{formatPeersString()}</Column >
<Column label="Actions">
<TorrentActions id={id} statsResponse={statsResponse} />
</Column>
</>
: <Column label="Loading stats" size={8}><Spinner /></Column>
}
@ -63,6 +244,11 @@ const Column: React.FC<{
const Torrent = ({ id, torrent }) => {
const [detailsResponse, updateDetailsResponse] = useState<TorrentDetails>(null);
const [statsResponse, updateStatsResponse] = useState<TorrentStats>(null);
const [forceStatsRefresh, setForceStatsRefresh] = useState(0);
const forceStatsRefreshCallback = () => {
setForceStatsRefresh(forceStatsRefresh + 1);
}
// Update details once.
useEffect(() => {
@ -76,18 +262,29 @@ const Torrent = ({ id, torrent }) => {
// Update stats once then forever.
useEffect(() => customSetInterval((async () => {
const errorInterval = 10000;
const liveInterval = 500;
const finishedInterval = 5000;
const liveInterval = 1000;
const finishedInterval = 10000;
const nonLiveInterval = 10000;
return API.getTorrentStats(torrent.id).then((stats) => {
updateStatsResponse(stats);
return torrentIsDone(stats) ? finishedInterval : liveInterval;
return stats;
}).then((stats) => {
if (stats.finished) {
return finishedInterval;
}
if (stats.state == STATE_INITIALIZING || stats.state == STATE_LIVE) {
return liveInterval;
}
return nonLiveInterval;
}, (e) => {
return errorInterval;
});
}), 0), []);
}), 0), [forceStatsRefresh]);
return <TorrentRow id={id} detailsResponse={detailsResponse} statsResponse={statsResponse} />
return <RefreshTorrentStatsContext.Provider value={{ refresh: forceStatsRefreshCallback }}>
<TorrentRow id={id} detailsResponse={detailsResponse} statsResponse={statsResponse} />
</RefreshTorrentStatsContext.Provider >
}
const TorrentsList = (props: { torrents: Array<TorrentId>, loading: boolean }) => {
@ -143,7 +340,7 @@ const Root = () => {
return <AppContext.Provider value={context}>
<div className='text-center'>
<h1 className="mt-3 mb-4">rqbit web 0.0.1-alpha</h1>
<h1 className="mt-3 mb-4">rqbit web 4.0.0-beta.0</h1>
<RootContent
closeableError={closeableError}
otherError={otherError}
@ -199,7 +396,7 @@ const UploadButton = ({ buttonText, onClick, data, resetData, variant }) => {
const response = await API.uploadTorrent(data, { listOnly: true });
setFileList(response.details.files);
} catch (e) {
setFileListError({ text: 'Error listing torrent', details: e });
setFileListError({ text: 'Error uploading torrent', details: e });
} finally {
setLoading(false);
}
@ -373,10 +570,6 @@ const RootContent = (props: { closeableError: ErrorDetails, otherError: ErrorDet
</Container>
};
function torrentIsDone(stats: TorrentStats): boolean {
return stats.snapshot.have_bytes == stats.snapshot.total_bytes;
}
function formatBytes(bytes: number): string {
if (bytes === 0) return '0 Bytes';
@ -398,11 +591,11 @@ function getLargestFileName(torrentDetails: TorrentDetails): string {
}
function getCompletionETA(stats: TorrentStats): string {
if (stats.time_remaining && stats.time_remaining.duration) {
return formatSecondsToTime(stats.time_remaining.duration.secs);
} else {
let duration = stats?.live?.time_remaining?.duration?.secs;
if (duration == null) {
return 'N/A';
}
return formatSecondsToTime(duration);
}
function formatSecondsToTime(seconds: number): string {

1
crates/librqbit/webui/vite.config.ts
View file

@ -1,5 +1,4 @@
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react'
// https://vitejs.dev/config/
export default defineConfig({

4
crates/librqbit_core/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "librqbit-core"
version = "3.0.0"
version = "3.1.0"
edition = "2021"
description = "Important utilities used throughout librqbit useful for working with torrents."
license = "Apache-2.0"
@ -17,6 +17,8 @@ sha1-openssl = ["bencode/sha1-openssl"]
sha1-rust = ["bencode/sha1-rust"]
[dependencies]
tracing = "0.1.40"
tokio = "1"
hex = "0.4"
anyhow = "1"
url = "2"

9
crates/librqbit_core/src/lengths.rs
View file

@ -1,4 +1,4 @@
use crate::constants::CHUNK_SIZE;
use crate::{constants::CHUNK_SIZE, torrent_metainfo::TorrentMetaV1Info};
const fn is_power_of_two(x: u64) -> bool {
(x != 0) && ((x & (x - 1)) == 0)
@ -61,6 +61,13 @@ impl ValidPieceIndex {
}
impl Lengths {
pub fn from_torrent<ByteBuf: AsRef<[u8]>>(
torrent: &TorrentMetaV1Info<ByteBuf>,
) -> anyhow::Result<Lengths> {
let total_length = torrent.iter_file_lengths()?.sum();
Lengths::new(total_length, torrent.piece_length, None)
}
pub fn new(
total_length: u64,
piece_length: u32,

1
crates/librqbit_core/src/lib.rs
View file

@ -3,5 +3,6 @@ pub mod id20;
pub mod lengths;
pub mod magnet;
pub mod peer_id;
pub mod spawn_utils;
pub mod speed_estimator;
pub mod torrent_metainfo;

11
crates/librqbit_core/src/magnet.rs
View file

@ -41,6 +41,17 @@ impl Magnet {
}
}
impl std::fmt::Display for Magnet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"magnet:?xt=urn:btih:{}&tr={}",
self.info_hash.as_string(),
self.trackers.join("&tr=")
)
}
}
#[cfg(test)]
mod tests {
#[test]

20
crates/librqbit_core/src/spawn_utils.rs Normal file
View file

@ -0,0 +1,20 @@
use tracing::{debug, error, trace, Instrument};
pub fn spawn(
span: tracing::Span,
fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
) -> tokio::task::JoinHandle<()> {
let fut = async move {
trace!("started");
match fut.await {
Ok(_) => {
debug!("finished");
}
Err(e) => {
error!("finished with error: {:#}", e)
}
}
}
.instrument(span);
tokio::task::spawn(fut)
}

4
crates/peer_binary_protocol/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "librqbit-peer-protocol"
version = "3.0.0"
version = "3.1.0"
edition = "2021"
description = "Protocol for working with torrent peers. Used in rqbit torrent client."
license = "Apache-2.0"
@ -23,6 +23,6 @@ byteorder = "1"
buffers = {path="../buffers", package="librqbit-buffers", version = "2.2.1"}
bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
librqbit-core = {path="../librqbit_core", version = "3.0.0"}
librqbit-core = {path="../librqbit_core", version = "3.1.0"}
bitvec = "1"
anyhow = "1"

8
crates/rqbit/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "rqbit"
version = "3.3.0"
version = "4.0.0-beta.0"
authors = ["Igor Katson <igor.katson@gmail.com>"]
edition = "2021"
description = "A bittorrent command line client and server."
@ -13,6 +13,7 @@ readme = "README.md"
[features]
default = ["sha1-system", "default-tls", "webui"]
tokio-console = ["console-subscriber", "tokio/tracing"]
webui = ["librqbit/webui"]
timed_existence = ["librqbit/timed_existence"]
sha1-system = ["librqbit/sha1-system"]
@ -22,9 +23,10 @@ default-tls = ["librqbit/default-tls"]
rust-tls = ["librqbit/rust-tls"]
[dependencies]
librqbit = {path="../librqbit", default-features=false, version = "3.3.0"}
dht = {path="../dht", package="librqbit-dht", version="3.1.0"}
librqbit = {path="../librqbit", default-features=false, version = "4.0.0-beta.0"}
dht = {path="../dht", package="librqbit-dht", version="3.2.0"}
tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
console-subscriber = {version = "0.2", optional = true}
anyhow = "1"
clap = {version = "4", features = ["derive", "deprecated"]}
tracing = "0.1"

281
crates/rqbit/src/main.rs
View file

@ -7,14 +7,14 @@ use librqbit::{
http_api_client,
peer_connection::PeerConnectionOptions,
session::{
AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, ManagedTorrentState,
Session, SessionOptions,
AddTorrent, AddTorrentOptions, AddTorrentResponse, ListOnlyResponse, Session,
SessionOptions,
},
spawn_utils::{spawn, BlockingSpawner},
torrent_state::timeit,
torrent_state::ManagedTorrentState,
};
use size_format::SizeFormatterBinary as SF;
use tracing::{error, info, span, warn, Level};
use tracing::{error, error_span, info, trace_span, warn};
#[derive(Debug, Clone, Copy, ValueEnum)]
enum LogLevel {
@ -77,6 +77,13 @@ struct Opts {
struct ServerStartOptions {
/// The output folder to write to. If not exists, it will be created.
output_folder: String,
#[arg(
long = "disable-persistence",
help = "Disable server persistence. It will not read or write its state to disk."
)]
disable_persistence: bool,
#[arg(long = "persistence-filename")]
persistence_filename: Option<String>,
}
#[derive(Parser)]
@ -134,31 +141,84 @@ enum SubCommand {
Download(DownloadOpts),
}
fn init_logging(opts: &Opts) {
if std::env::var_os("RUST_LOG").is_none() {
match opts.log_level.as_ref() {
Some(level) => {
let level_str = match level {
LogLevel::Trace => "trace",
LogLevel::Debug => "debug",
LogLevel::Info => "info",
LogLevel::Warn => "warn",
LogLevel::Error => "error",
};
std::env::set_var("RUST_LOG", level_str);
}
None => {
std::env::set_var("RUST_LOG", "info");
}
};
}
// Iint logging and make a channel to send new RUST_LOG values to.
fn init_logging(opts: &Opts) -> tokio::sync::mpsc::UnboundedSender<String> {
let default_rust_log = match opts.log_level.as_ref() {
Some(level) => match level {
LogLevel::Trace => "trace",
LogLevel::Debug => "debug",
LogLevel::Info => "info",
LogLevel::Warn => "warn",
LogLevel::Error => "error",
},
None => "info",
};
let stderr_filter = match std::env::var("RUST_LOG").ok() {
Some(rust_log) => EnvFilter::builder()
.parse(rust_log)
.expect("can't parse RUST_LOG"),
None => EnvFilter::builder()
.parse(default_rust_log)
.expect("can't parse default_rust_log"),
};
let (stderr_filter, reload_stderr_filter) =
tracing_subscriber::reload::Layer::new(stderr_filter);
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
tracing_subscriber::registry()
.with(fmt::layer())
.with(EnvFilter::from_default_env())
.init();
#[cfg(feature = "tokio-console")]
{
let (console_layer, server) = console_subscriber::Builder::default()
.with_default_env()
.build();
tracing_subscriber::registry()
.with(fmt::layer().with_filter(stderr_filter))
.with(console_layer)
.init();
spawn(
"console_subscriber server",
error_span!("console_subscriber server"),
async move {
server
.serve()
.await
.map_err(|e| anyhow::anyhow!("{:#?}", e))
.context("error running console subscriber server")
},
);
}
#[cfg(not(feature = "tokio-console"))]
{
tracing_subscriber::registry()
.with(fmt::layer())
.with(stderr_filter)
.init();
}
let (reload_tx, mut reload_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
spawn(
"fmt_filter_reloader",
error_span!("fmt_filter_reloader"),
async move {
while let Some(rust_log) = reload_rx.recv().await {
let stderr_env_filter = match EnvFilter::builder().parse(&rust_log) {
Ok(f) => f,
Err(e) => {
eprintln!("can't parse env filter {:?}: {:#?}", rust_log, e);
continue;
}
};
eprintln!("setting RUST_LOG to {:?}", rust_log);
let _ = reload_stderr_filter.reload(stderr_env_filter);
}
Ok(())
},
);
reload_tx
}
fn _start_deadlock_detector_thread() {
@ -188,9 +248,6 @@ fn _start_deadlock_detector_thread() {
fn main() -> anyhow::Result<()> {
let opts = Opts::parse();
init_logging(&opts);
// start_deadlock_detector_thread();
let (mut rt_builder, spawner) = match opts.single_thread_runtime {
true => (
tokio::runtime::Builder::new_current_thread(),
@ -223,10 +280,14 @@ fn main() -> anyhow::Result<()> {
}
async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()> {
let sopts = SessionOptions {
let logging_reload_tx = init_logging(&opts);
let mut sopts = SessionOptions {
disable_dht: opts.disable_dht,
disable_dht_persistence: opts.disable_dht_persistence,
dht_config: None,
persistence: false,
persistence_filename: None,
peer_id: None,
peer_opts: Some(PeerConnectionOptions {
connect_timeout: Some(opts.peer_connect_timeout),
@ -238,39 +299,49 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
let stats_printer = |session: Arc<Session>| async move {
loop {
session.with_torrents(|torrents| {
for (idx, torrent) in torrents.iter().enumerate() {
match &torrent.state {
ManagedTorrentState::Initializing => {
info!("[{}] initializing", idx);
},
ManagedTorrentState::Running(handle) => {
let stats = timeit("stats_snapshot", || handle.torrent_state().stats_snapshot());
let speed = handle.speed_estimator();
let total = stats.total_bytes;
let progress = stats.total_bytes - stats.remaining_bytes;
let downloaded_pct = if stats.remaining_bytes == 0 {
100f64
} else {
(progress as f64 / total as f64) * 100f64
};
info!(
"[{}]: {:.2}% ({:.2}), down speed {:.2} MiB/s, fetched {}, remaining {:.2} of {:.2}, uploaded {:.2}, peers: {{live: {}, connecting: {}, queued: {}, seen: {}, dead: {}}}",
idx,
downloaded_pct,
SF::new(progress),
speed.download_mbps(),
SF::new(stats.fetched_bytes),
SF::new(stats.remaining_bytes),
SF::new(total),
SF::new(stats.uploaded_bytes),
stats.peer_stats.live,
stats.peer_stats.connecting,
stats.peer_stats.queued,
stats.peer_stats.seen,
stats.peer_stats.dead,
);
},
}
for (idx, torrent) in torrents {
let live = torrent.with_state(|s| {
match s {
ManagedTorrentState::Initializing(i) => {
let total = torrent.get_total_bytes();
let progress = i.get_checked_bytes();
let pct = (progress as f64 / total as f64) * 100f64;
info!("[{}] initializing {:.2}%", idx, pct)
},
ManagedTorrentState::Live(h) => return Some(h.clone()),
_ => {},
};
None
});
let handle = match live {
Some(live) => live,
None => continue
};
let stats = handle.stats_snapshot();
let speed = handle.speed_estimator();
let total = stats.total_bytes;
let progress = stats.total_bytes - stats.remaining_bytes;
let downloaded_pct = if stats.remaining_bytes == 0 {
100f64
} else {
(progress as f64 / total as f64) * 100f64
};
info!(
"[{}]: {:.2}% ({:.2}), down speed {:.2} MiB/s, fetched {}, remaining {:.2} of {:.2}, uploaded {:.2}, peers: {{live: {}, connecting: {}, queued: {}, seen: {}, dead: {}}}",
idx,
downloaded_pct,
SF::new(progress),
speed.download_mbps(),
SF::new(stats.fetched_bytes),
SF::new(stats.remaining_bytes),
SF::new(total),
SF::new(stats.uploaded_bytes),
stats.peer_stats.live,
stats.peer_stats.connecting,
stats.peer_stats.queued,
stats.peer_stats.seen,
stats.peer_stats.dead,
);
}
});
tokio::time::sleep(Duration::from_secs(1)).await;
@ -280,23 +351,25 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
match &opts.subcommand {
SubCommand::Server(server_opts) => match &server_opts.subcommand {
ServerSubcommand::Start(start_opts) => {
let session = Arc::new(
Session::new_with_opts(
PathBuf::from(&start_opts.output_folder),
spawner,
sopts,
)
.await
.context("error initializing rqbit session")?,
);
sopts.persistence = !start_opts.disable_persistence;
sopts.persistence_filename =
start_opts.persistence_filename.clone().map(PathBuf::from);
let session = Session::new_with_opts(
PathBuf::from(&start_opts.output_folder),
spawner,
sopts,
)
.await
.context("error initializing rqbit session")?;
spawn(
span!(Level::TRACE, "stats_printer"),
"stats_printer",
trace_span!("stats_printer"),
stats_printer(session.clone()),
);
let http_api = HttpApi::new(session);
let http_api = HttpApi::new(session, Some(logging_reload_tx));
let http_api_listen_addr = opts.http_api_listen_addr;
http_api
.make_http_api_and_run(http_api_listen_addr)
.make_http_api_and_run(http_api_listen_addr, false)
.await
.context("error starting HTTP API")
}
@ -353,30 +426,32 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
}
Ok(())
} else {
let session = Arc::new(
Session::new_with_opts(
download_opts
.output_folder
.as_ref()
.map(PathBuf::from)
.context(
"output_folder is required if can't connect to an existing server",
)?,
spawner,
sopts,
)
.await
.context("error initializing rqbit session")?,
);
let session = Session::new_with_opts(
download_opts
.output_folder
.as_ref()
.map(PathBuf::from)
.context(
"output_folder is required if can't connect to an existing server",
)?,
spawner,
sopts,
)
.await
.context("error initializing rqbit session")?;
spawn(
span!(Level::TRACE, "stats_printer"),
"stats_printer",
trace_span!("stats_printer"),
stats_printer(session.clone()),
);
let http_api = HttpApi::new(session.clone());
let http_api = HttpApi::new(session.clone(), Some(logging_reload_tx));
let http_api_listen_addr = opts.http_api_listen_addr;
spawn(
span!(Level::ERROR, "http_api"),
http_api.clone().make_http_api_and_run(http_api_listen_addr),
"http_api",
error_span!("http_api"),
http_api
.clone()
.make_http_api_and_run(http_api_listen_addr, true),
);
let mut added = false;
@ -392,10 +467,12 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
.await
{
Ok(v) => match v {
AddTorrentResponse::AlreadyManaged(handle) => {
AddTorrentResponse::AlreadyManaged(id, handle) => {
info!(
"torrent {:?} is already managed, downloaded to {:?}",
handle.info_hash, handle.output_folder
"torrent {:?} is already managed, id={}, downloaded to {:?}",
handle.info_hash(),
id,
handle.info().out_dir
);
continue;
}
@ -420,7 +497,7 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
}
continue;
}
AddTorrentResponse::Added(handle) => {
AddTorrentResponse::Added(_, handle) => {
added = true;
handle
}
@ -431,7 +508,6 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
}
};
http_api.add_torrent_handle(handle.clone());
handles.push(handle);
}
@ -441,8 +517,11 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
if download_opts.exit_on_finish {
let results = futures::future::join_all(
handles.iter().map(|h| h.wait_until_completed()),
);
results.await;
)
.await;
if results.iter().any(|r| r.is_err()) {
anyhow::bail!("some downloads failed")
}
info!("All downloads completed, exiting");
Ok(())
} else {