leyoda/rqbit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

DHT instrumentation

Browse source
This commit is contained in:
Igor Katson 2023-11-25 15:15:16 +00:00
parent 6f113c5137
commit d8fdb94305
No known key found for this signature in database
GPG key ID: B4EC22B66D61A3F5
11 changed files with 76 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -1000,7 +1000,7 @@ dependencies = [
[[package]] [[package]]
name = "librqbit" name = "librqbit"
version = "3.3.0" version = "4.0.0-beta.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"axum", "axum",
@ -1066,7 +1066,7 @@ version = "2.2.1"
[[package]] [[package]]
name = "librqbit-core" name = "librqbit-core"
version = "3.0.0" version = "3.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"hex 0.4.3", "hex 0.4.3",
@ -1076,13 +1076,15 @@ dependencies = [
"librqbit-clone-to-owned", "librqbit-clone-to-owned",
"parking_lot", "parking_lot",
"serde", "serde",
"tokio",
"tracing",
"url", "url",
"uuid", "uuid",
] ]
[[package]] [[package]]
name = "librqbit-dht" name = "librqbit-dht"
version = "3.1.0" version = "3.2.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"directories", "directories",
@ -1105,7 +1107,7 @@ dependencies = [
[[package]] [[package]]
name = "librqbit-peer-protocol" name = "librqbit-peer-protocol"
version = "3.0.0" version = "3.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bincode", "bincode",
@ -1704,7 +1706,7 @@ dependencies = [
[[package]] [[package]]
name = "rqbit" name = "rqbit"
version = "3.3.0" version = "4.0.0-beta.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",

4
crates/dht/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "librqbit-dht" name = "librqbit-dht"
version = "3.1.0" version = "3.2.0"
edition = "2021" edition = "2021"
description = "DHT implementation, used in rqbit torrent client." description = "DHT implementation, used in rqbit torrent client."
license = "Apache-2.0" license = "Apache-2.0"
@ -33,7 +33,7 @@ indexmap = "2"
directories = "5" directories = "5"
clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"} clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
librqbit-core = {path="../librqbit_core", version = "3.0.0"} librqbit-core = {path="../librqbit_core", version = "3.1.0"}
[dev-dependencies] [dev-dependencies]
tracing-subscriber = "0.3" tracing-subscriber = "0.3"

53
crates/dht/src/dht.rs
View file

@ -18,7 +18,7 @@ use bencode::ByteString;
use futures::{stream::FuturesUnordered, Stream, StreamExt}; use futures::{stream::FuturesUnordered, Stream, StreamExt};
use indexmap::IndexSet; use indexmap::IndexSet;
use leaky_bucket::RateLimiter; use leaky_bucket::RateLimiter;
use librqbit_core::{id20::Id20, peer_id::generate_peer_id}; use librqbit_core::{id20::Id20, peer_id::generate_peer_id, spawn_utils::spawn};
use parking_lot::RwLock; use parking_lot::RwLock;
use rand::Rng; use rand::Rng;
use serde::Serialize; use serde::Serialize;
@ -27,7 +27,7 @@ use tokio::{
sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender}, sync::mpsc::{channel, unbounded_channel, Sender, UnboundedReceiver, UnboundedSender},
}; };
use tokio_stream::wrappers::{errors::BroadcastStreamRecvError, BroadcastStream}; use tokio_stream::wrappers::{errors::BroadcastStreamRecvError, BroadcastStream};
use tracing::{debug, info, trace, warn}; use tracing::{debug, debug_span, error_span, info, trace, warn, Instrument};
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
pub struct DhtStats { pub struct DhtStats {
@ -513,7 +513,7 @@ impl DhtWorker {
bootstrap_addrs: &[String], bootstrap_addrs: &[String],
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let (out_tx, mut out_rx) = channel(1); let (out_tx, mut out_rx) = channel(1);
let framer = run_framer(&self.socket, in_rx, out_tx); let framer = run_framer(&self.socket, in_rx, out_tx).instrument(debug_span!("dht_framer"));
let bootstrap = async { let bootstrap = async {
let mut futs = FuturesUnordered::new(); let mut futs = FuturesUnordered::new();
@ -521,34 +521,40 @@ impl DhtWorker {
for addr in bootstrap_addrs.iter() { for addr in bootstrap_addrs.iter() {
let this = &self; let this = &self;
let in_tx = &in_tx; let in_tx = &in_tx;
futs.push(async move { futs.push(
match tokio::net::lookup_host(addr).await { async move {
Ok(addrs) => { match tokio::net::lookup_host(addr).await {
for addr in addrs { Ok(addrs) => {
let request = this for addr in addrs {
.state let request = this
.write() .state
.create_request(Request::FindNode(this.peer_id), addr); .write()
in_tx.send((request, addr))?; .create_request(Request::FindNode(this.peer_id), addr);
in_tx.send((request, addr))?;
}
}
Err(e) => {
warn!("error looking up {}: {}", addr, e);
return Err(e.into());
} }
} }
Err(e) => warn!("error looking up {}: {}", addr, e), Ok::<_, anyhow::Error>(())
} }
Ok::<_, anyhow::Error>(()) .instrument(error_span!("dht_bootstrap", addr = addr)),
}); );
} }
let mut successes = 0; let mut successes = 0;
while let Some(resp) = futs.next().await { while let Some(resp) = futs.next().await {
match resp { if resp.is_ok() {
Ok(_) => successes += 1, successes += 1
Err(e) => warn!("error in one of the bootstrappers: {}", e),
} }
} }
if successes == 0 { if successes == 0 {
anyhow::bail!("bootstrapping did not succeed") anyhow::bail!("bootstrapping did not succeed")
} }
Ok(()) Ok(())
}; }
.instrument(debug_span!("dht_bootstrapper"));
let mut bootstrap_done = false; let mut bootstrap_done = false;
let response_reader = { let response_reader = {
@ -563,7 +569,8 @@ impl DhtWorker {
"closed response reader, nowhere to send results to, DHT closed" "closed response reader, nowhere to send results to, DHT closed"
)) ))
} }
}; }
.instrument(debug_span!("dht_responese_reader"));
tokio::pin!(framer); tokio::pin!(framer);
tokio::pin!(bootstrap); tokio::pin!(bootstrap);
@ -676,7 +683,7 @@ impl Dht {
listen_addr, listen_addr,
))); )));
tokio::spawn({ spawn(error_span!("dht"), {
let state = state.clone(); let state = state.clone();
async move { async move {
let worker = DhtWorker { let worker = DhtWorker {
@ -684,8 +691,8 @@ impl Dht {
peer_id, peer_id,
state, state,
}; };
let result = worker.start(in_tx, in_rx, &bootstrap_addrs).await; worker.start(in_tx, in_rx, &bootstrap_addrs).await?;
warn!("DHT worker finished with {:?}", result); Ok(())
} }
}); });
Ok(Dht { state }) Ok(Dht { state })

6
crates/dht/src/persistence.rs
View file

@ -1,5 +1,6 @@
// TODO: this now stores only the routing table, but we also need AT LEAST the same socket address... // TODO: this now stores only the routing table, but we also need AT LEAST the same socket address...
use librqbit_core::spawn_utils::spawn;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::io::{BufReader, BufWriter}; use std::io::{BufReader, BufWriter};
@ -8,8 +9,7 @@ use std::path::{Path, PathBuf};
use std::time::Duration; use std::time::Duration;
use anyhow::Context; use anyhow::Context;
use tokio::spawn; use tracing::{debug, error, error_span, info, trace, warn};
use tracing::{debug, error, info, trace, warn};
use crate::dht::{Dht, DhtConfig}; use crate::dht::{Dht, DhtConfig};
use crate::routing_table::RoutingTable; use crate::routing_table::RoutingTable;
@ -110,7 +110,7 @@ impl PersistentDht {
}; };
let dht = Dht::with_config(dht_config).await?; let dht = Dht::with_config(dht_config).await?;
spawn({ spawn(error_span!("dht_persistence"), {
let dht = dht.clone(); let dht = dht.clone();
let dump_interval = config let dump_interval = config
.dump_interval .dump_interval

8
crates/librqbit/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "librqbit" name = "librqbit"
version = "3.3.0" version = "4.0.0-beta.0"
authors = ["Igor Katson <igor.katson@gmail.com>"] authors = ["Igor Katson <igor.katson@gmail.com>"]
edition = "2021" edition = "2021"
description = "The main library used by rqbit torrent client. The binary is just a small wrapper on top of it." description = "The main library used by rqbit torrent client. The binary is just a small wrapper on top of it."
@ -24,11 +24,11 @@ rust-tls = ["reqwest/rustls-tls"]
[dependencies] [dependencies]
bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"} bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
buffers = {path = "../buffers", package="librqbit-buffers", version = "2.2.1"} buffers = {path = "../buffers", package="librqbit-buffers", version = "2.2.1"}
librqbit-core = {path = "../librqbit_core", version = "3.0.0"} librqbit-core = {path = "../librqbit_core", version = "3.1.0"}
clone_to_owned = {path = "../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"} clone_to_owned = {path = "../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.0.0"} peer_binary_protocol = {path = "../peer_binary_protocol", package="librqbit-peer-protocol", version = "3.1.0"}
sha1w = {path = "../sha1w", default-features=false, package="librqbit-sha1-wrapper", version="2.2.1"} sha1w = {path = "../sha1w", default-features=false, package="librqbit-sha1-wrapper", version="2.2.1"}
dht = {path = "../dht", package="librqbit-dht", version="3.1.0"} dht = {path = "../dht", package="librqbit-dht", version="3.2.0"}
tokio = {version = "1", features = ["macros", "rt-multi-thread"]} tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
axum = {version = "0.6"} axum = {version = "0.6"}

16
crates/librqbit/src/spawn_utils.rs
View file

@ -1,23 +1,9 @@
use tracing::{debug, trace, warn, Instrument};
pub fn spawn( pub fn spawn(
_name: &str, _name: &str,
span: tracing::Span, span: tracing::Span,
fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static, fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
) -> tokio::task::JoinHandle<()> { ) -> tokio::task::JoinHandle<()> {
let fut = async move { librqbit_core::spawn_utils::spawn(span, fut)
trace!("started");
match fut.await {
Ok(_) => {
debug!("finished");
}
Err(e) => {
warn!("finished with error: {:#}", e)
}
}
}
.instrument(span.or_current());
tokio::task::spawn(fut)
} }
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]

4
crates/librqbit_core/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "librqbit-core" name = "librqbit-core"
version = "3.0.0" version = "3.1.0"
edition = "2021" edition = "2021"
description = "Important utilities used throughout librqbit useful for working with torrents." description = "Important utilities used throughout librqbit useful for working with torrents."
license = "Apache-2.0" license = "Apache-2.0"
@ -17,6 +17,8 @@ sha1-openssl = ["bencode/sha1-openssl"]
sha1-rust = ["bencode/sha1-rust"] sha1-rust = ["bencode/sha1-rust"]
[dependencies] [dependencies]
tracing = "0.1.40"
tokio = "1"
hex = "0.4" hex = "0.4"
anyhow = "1" anyhow = "1"
url = "2" url = "2"

1
crates/librqbit_core/src/lib.rs
View file

@ -3,5 +3,6 @@ pub mod id20;
pub mod lengths; pub mod lengths;
pub mod magnet; pub mod magnet;
pub mod peer_id; pub mod peer_id;
pub mod spawn_utils;
pub mod speed_estimator; pub mod speed_estimator;
pub mod torrent_metainfo; pub mod torrent_metainfo;

20
crates/librqbit_core/src/spawn_utils.rs Normal file
View file

@ -0,0 +1,20 @@
use tracing::{debug, error, trace, Instrument};
pub fn spawn(
span: tracing::Span,
fut: impl std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
) -> tokio::task::JoinHandle<()> {
let fut = async move {
trace!("started");
match fut.await {
Ok(_) => {
debug!("finished");
}
Err(e) => {
error!("finished with error: {:#}", e)
}
}
}
.instrument(span);
tokio::task::spawn(fut)
}

4
crates/peer_binary_protocol/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "librqbit-peer-protocol" name = "librqbit-peer-protocol"
version = "3.0.0" version = "3.1.0"
edition = "2021" edition = "2021"
description = "Protocol for working with torrent peers. Used in rqbit torrent client." description = "Protocol for working with torrent peers. Used in rqbit torrent client."
license = "Apache-2.0" license = "Apache-2.0"
@ -23,6 +23,6 @@ byteorder = "1"
buffers = {path="../buffers", package="librqbit-buffers", version = "2.2.1"} buffers = {path="../buffers", package="librqbit-buffers", version = "2.2.1"}
bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"} bencode = {path = "../bencode", default-features=false, package="librqbit-bencode", version="2.2.1"}
clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"} clone_to_owned = {path="../clone_to_owned", package="librqbit-clone-to-owned", version = "2.2.1"}
librqbit-core = {path="../librqbit_core", version = "3.0.0"} librqbit-core = {path="../librqbit_core", version = "3.1.0"}
bitvec = "1" bitvec = "1"
anyhow = "1" anyhow = "1"

6
crates/rqbit/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "rqbit" name = "rqbit"
version = "3.3.0" version = "4.0.0-beta.0"
authors = ["Igor Katson <igor.katson@gmail.com>"] authors = ["Igor Katson <igor.katson@gmail.com>"]
edition = "2021" edition = "2021"
description = "A bittorrent command line client and server." description = "A bittorrent command line client and server."
@ -23,8 +23,8 @@ default-tls = ["librqbit/default-tls"]
rust-tls = ["librqbit/rust-tls"] rust-tls = ["librqbit/rust-tls"]
[dependencies] [dependencies]
librqbit = {path="../librqbit", default-features=false, version = "3.3.0"} librqbit = {path="../librqbit", default-features=false, version = "4.0.0-beta.0"}
dht = {path="../dht", package="librqbit-dht", version="3.1.0"} dht = {path="../dht", package="librqbit-dht", version="3.2.0"}
tokio = {version = "1", features = ["macros", "rt-multi-thread"]} tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
console-subscriber = {version = "0.2", optional = true} console-subscriber = {version = "0.2", optional = true}
anyhow = "1" anyhow = "1"