Replaced DHT with custom one! Lets see if it works

crates/dht/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-kad = "0.6"
 tokio = {version = "1", features = ["macros", "rt-multi-thread", "net", "sync"]}
 tokio-stream = "0.1"
 serde = {version = "1", features = ["derive"]}

crates/dht/src/bprotocol.rs

@@ -6,13 +6,12 @@ use std::{
 
 use bencode::ByteBuf;
 use clone_to_owned::CloneToOwned;
+use librqbit_core::id20::Id20;
 use serde::{
     de::{IgnoredAny, Unexpected},
     Deserialize, Deserializer, Serialize,
 };
 
-use crate::id20::Id20;
-
 #[derive(Debug)]
 enum MessageType {
     Request,

crates/dht/src/dht.rs

@@ -7,13 +7,12 @@ use crate::{
     bprotocol::{
         self, CompactNodeInfo, FindNodeRequest, GetPeersRequest, Message, MessageKind, Node,
     },
-    id20::Id20,
     routing_table::{InsertResult, RoutingTable},
 };
 use anyhow::Context;
 use bencode::ByteString;
 use futures::{stream::FuturesUnordered, StreamExt};
-use librqbit_core::peer_id::generate_peer_id;
+use librqbit_core::{id20::Id20, peer_id::generate_peer_id};
 use log::{debug, info, trace, warn};
 use parking_lot::Mutex;
 use tokio::{

crates/dht/src/id20.rs

@@ -1,131 +1 @@
-use std::{cmp::Ordering, str::FromStr};
 
-use serde::{Deserialize, Deserializer, Serialize};
-
-#[derive(Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Id20(pub [u8; 20]);
-
-impl FromStr for Id20 {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut out = [0u8; 20];
-        if s.len() != 40 {
-            anyhow::bail!("expected a hex string of length 40")
-        };
-        hex::decode_to_slice(s, &mut out)?;
-        Ok(Id20(out))
-    }
-}
-
-impl std::fmt::Debug for Id20 {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "<")?;
-        for byte in self.0 {
-            write!(f, "{:02x?}", byte)?;
-        }
-        write!(f, ">")?;
-        Ok(())
-    }
-}
-
-impl Serialize for Id20 {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        serializer.serialize_bytes(&self.0)
-    }
-}
-
-impl<'de> Deserialize<'de> for Id20 {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        struct Visitor;
-        impl<'de> serde::de::Visitor<'de> for Visitor {
-            type Value = Id20;
-
-            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-                write!(formatter, "a 20 byte slice")
-            }
-            fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                if v.len() != 20 {
-                    return Err(E::invalid_length(20, &self));
-                }
-                let mut buf = [0u8; 20];
-                buf.copy_from_slice(&v);
-                Ok(Id20(buf))
-            }
-        }
-        deserializer.deserialize_bytes(Visitor {})
-    }
-}
-
-impl Id20 {
-    pub fn distance(&self, other: &Id20) -> Id20 {
-        let mut xor = [0u8; 20];
-        for (idx, (s, o)) in self
-            .0
-            .iter()
-            .copied()
-            .zip(other.0.iter().copied())
-            .enumerate()
-        {
-            xor[idx] = s ^ o;
-        }
-        Id20(xor)
-    }
-    pub fn set_bit(&mut self, bit: u8, value: bool) {
-        let n = &mut self.0[(bit / 8) as usize];
-        if value {
-            *n |= 1 << (7 - bit % 8)
-        } else {
-            let mask = !(1 << (7 - bit % 8));
-            *n &= mask;
-        }
-    }
-    pub fn set_bits_range(&mut self, r: std::ops::Range<u8>, value: bool) {
-        for bit in r {
-            self.set_bit(bit, value)
-        }
-    }
-}
-
-impl Ord for Id20 {
-    fn cmp(&self, other: &Id20) -> Ordering {
-        for (s, o) in self.0.iter().copied().zip(other.0.iter().copied()) {
-            match s.cmp(&o) {
-                Ordering::Less => return Ordering::Less,
-                Ordering::Equal => continue,
-                Ordering::Greater => return Ordering::Greater,
-            }
-        }
-        Ordering::Equal
-    }
-}
-
-impl PartialOrd<Id20> for Id20 {
-    fn partial_cmp(&self, other: &Id20) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::Id20;
-
-    #[test]
-    fn test_set_bit_range() {
-        let mut id = Id20([0u8; 20]);
-        id.set_bits_range(9..17, true);
-        assert_eq!(
-            id,
-            Id20([0, 127, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
-        )
-    }
-}

crates/dht/src/lib.rs

@@ -1,4 +1,6 @@
-pub mod bprotocol;
-pub mod dht;
-pub mod id20;
-pub mod routing_table;
+mod bprotocol;
+mod dht;
+mod routing_table;
+
+pub use dht::Dht;
+pub use librqbit_core::id20::Id20;

crates/dht/src/main.rs

@@ -1,7 +1,7 @@
-use std::{collections::HashSet, str::FromStr, time::Duration};
+use std::{collections::HashSet, str::FromStr};
 
 use anyhow::Context;
-use dht::{dht::Dht, id20::Id20};
+use dht::{Dht, Id20};
 use tokio_stream::StreamExt;
 
 #[tokio::main]

crates/dht/src/routing_table.rs

@@ -3,10 +3,9 @@ use std::{
     time::{Duration, Instant},
 };
 
+use librqbit_core::id20::Id20;
 use log::debug;
 
-use crate::id20::Id20;
-
 #[derive(Debug)]
 enum BucketTreeNode {
     Leaf(Vec<RoutingTableNode>),
@@ -426,9 +425,10 @@ impl RoutingTable {
 mod tests {
     use std::net::SocketAddrV4;
 
+    use librqbit_core::id20::Id20;
     use rand::Rng;
 
-    use crate::{id20::Id20, routing_table::compute_split_start_end};
+    use crate::routing_table::compute_split_start_end;
 
     use super::RoutingTable;
 

crates/librqbit/Cargo.toml

@@ -21,6 +21,7 @@ peer_binary_protocol = {path = "../peer_binary_protocol"}
 sha1w = {path = "../sha1w"}
 
 tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
+tokio-stream = "0.1"
 serde = {version = "1", features=["derive"]}
 serde_json = "1"
 anyhow = "1"

crates/librqbit/src/dht/inforead.rs

@@ -1,36 +1,35 @@
-use std::net::SocketAddr;
+use std::{collections::HashSet, net::SocketAddr};
 
 use buffers::ByteString;
-use futures::{stream::FuturesUnordered, Stream, StreamExt};
+use futures::{stream::FuturesUnordered, StreamExt};
 use librqbit_core::torrent_metainfo::TorrentMetaV1Info;
 use log::debug;
-use tokio::sync::mpsc::UnboundedReceiver;
 
 use crate::peer_info_reader;
 
 #[derive(Debug)]
-pub enum ReadMetainfoResult {
+pub enum ReadMetainfoResult<Rx> {
     Found {
         info: TorrentMetaV1Info<ByteString>,
-        rx: UnboundedReceiver<SocketAddr>,
-        seen: Vec<SocketAddr>,
+        rx: Rx,
+        seen: HashSet<SocketAddr>,
     },
     ChannelClosed {
-        seen: Vec<SocketAddr>,
+        seen: HashSet<SocketAddr>,
     },
 }
 
-pub async fn read_metainfo_from_peer_receiver(
+pub async fn read_metainfo_from_peer_receiver<A: StreamExt<Item = SocketAddr> + Unpin>(
     peer_id: [u8; 20],
     info_hash: [u8; 20],
-    mut addrs: impl StreamExt<Item = SocketAddr> + Unpin,
-) -> ReadMetainfoResult {
-    let mut seen = Vec::<SocketAddr>::new();
+    mut addrs: A,
+) -> ReadMetainfoResult<A> {
+    let mut seen = HashSet::<SocketAddr>::new();
     let first_addr = match addrs.next().await {
         Some(addr) => addr,
         None => return ReadMetainfoResult::ChannelClosed { seen },
     };
-    seen.push(first_addr);
+    seen.insert(first_addr);
 
     let mut unordered = FuturesUnordered::new();
     unordered.push(peer_info_reader::read_metainfo_from_peer(
@@ -42,8 +41,9 @@ pub async fn read_metainfo_from_peer_receiver(
             next_addr = addrs.next() => {
                 match next_addr {
                     Some(addr) => {
-                        seen.push(addr);
-                        unordered.push(peer_info_reader::read_metainfo_from_peer(addr, peer_id, info_hash));
+                        if seen.insert(addr) {
+                            unordered.push(peer_info_reader::read_metainfo_from_peer(addr, peer_id, info_hash));
+                        }
                     },
                     None => return ReadMetainfoResult::ChannelClosed { seen },
                 }
@@ -64,6 +64,7 @@ pub async fn read_metainfo_from_peer_receiver(
 #[cfg(test)]
 mod tests {
     use librqbit_core::{info_hash::decode_info_hash, peer_id::generate_peer_id};
+    use tokio_stream::wrappers::UnboundedReceiverStream;
 
     use crate::dht::jsdht::JsDht;
 
@@ -83,6 +84,13 @@ mod tests {
         let info_hash = decode_info_hash("9905f844e5d8787ecd5e08fb46b2eb0a42c131d7").unwrap();
         let peer_rx = JsDht::new(info_hash).start_peer_discovery().unwrap();
         let peer_id = generate_peer_id();
-        dbg!(read_metainfo_from_peer_receiver(peer_id, info_hash, peer_rx).await);
+        dbg!(
+            read_metainfo_from_peer_receiver(
+                peer_id,
+                info_hash,
+                UnboundedReceiverStream::new(peer_rx)
+            )
+            .await
+        );
     }
 }

crates/librqbit_core/src/id20.rs

@@ -0,0 +1,131 @@
+use std::{cmp::Ordering, str::FromStr};
+
+use serde::{Deserialize, Deserializer, Serialize};
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Id20(pub [u8; 20]);
+
+impl FromStr for Id20 {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut out = [0u8; 20];
+        if s.len() != 40 {
+            anyhow::bail!("expected a hex string of length 40")
+        };
+        hex::decode_to_slice(s, &mut out)?;
+        Ok(Id20(out))
+    }
+}
+
+impl std::fmt::Debug for Id20 {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<")?;
+        for byte in self.0 {
+            write!(f, "{:02x?}", byte)?;
+        }
+        write!(f, ">")?;
+        Ok(())
+    }
+}
+
+impl Serialize for Id20 {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_bytes(&self.0)
+    }
+}
+
+impl<'de> Deserialize<'de> for Id20 {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct Visitor;
+        impl<'de> serde::de::Visitor<'de> for Visitor {
+            type Value = Id20;
+
+            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+                write!(formatter, "a 20 byte slice")
+            }
+            fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                if v.len() != 20 {
+                    return Err(E::invalid_length(20, &self));
+                }
+                let mut buf = [0u8; 20];
+                buf.copy_from_slice(&v);
+                Ok(Id20(buf))
+            }
+        }
+        deserializer.deserialize_bytes(Visitor {})
+    }
+}
+
+impl Id20 {
+    pub fn distance(&self, other: &Id20) -> Id20 {
+        let mut xor = [0u8; 20];
+        for (idx, (s, o)) in self
+            .0
+            .iter()
+            .copied()
+            .zip(other.0.iter().copied())
+            .enumerate()
+        {
+            xor[idx] = s ^ o;
+        }
+        Id20(xor)
+    }
+    pub fn set_bit(&mut self, bit: u8, value: bool) {
+        let n = &mut self.0[(bit / 8) as usize];
+        if value {
+            *n |= 1 << (7 - bit % 8)
+        } else {
+            let mask = !(1 << (7 - bit % 8));
+            *n &= mask;
+        }
+    }
+    pub fn set_bits_range(&mut self, r: std::ops::Range<u8>, value: bool) {
+        for bit in r {
+            self.set_bit(bit, value)
+        }
+    }
+}
+
+impl Ord for Id20 {
+    fn cmp(&self, other: &Id20) -> Ordering {
+        for (s, o) in self.0.iter().copied().zip(other.0.iter().copied()) {
+            match s.cmp(&o) {
+                Ordering::Less => return Ordering::Less,
+                Ordering::Equal => continue,
+                Ordering::Greater => return Ordering::Greater,
+            }
+        }
+        Ordering::Equal
+    }
+}
+
+impl PartialOrd<Id20> for Id20 {
+    fn partial_cmp(&self, other: &Id20) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Id20;
+
+    #[test]
+    fn test_set_bit_range() {
+        let mut id = Id20([0u8; 20]);
+        id.set_bits_range(9..17, true);
+        assert_eq!(
+            id,
+            Id20([0, 127, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
+        )
+    }
+}

crates/librqbit_core/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod constants;
+pub mod id20;
 pub mod info_hash;
 pub mod lengths;
 pub mod magnet;

crates/rqbit/Cargo.toml

@@ -9,6 +9,7 @@ rt-single-thread = []
 
 [dependencies]
 librqbit = {path="../librqbit"}
+dht = {path="../dht"}
 tokio = {version = "1", features = ["macros", "rt-multi-thread"]}
 anyhow = "1"
 clap = "3.0.0-beta.2"
@@ -16,6 +17,7 @@ log = "0.4"
 pretty_env_logger = "0.4"
 reqwest = "0.11"
 regex = "1"
+futures = "0.3"
 
 [dev-dependencies]
 futures = {version = "0.3"}

crates/rqbit/src/main.rs

@@ -2,8 +2,10 @@ use std::{fs::File, io::Read, net::SocketAddr, time::Duration};
 
 use anyhow::Context;
 use clap::Clap;
+use dht::{Dht, Id20};
+use futures::StreamExt;
 use librqbit::{
-    dht::{inforead::read_metainfo_from_peer_receiver, jsdht::JsDht},
+    dht::inforead::read_metainfo_from_peer_receiver,
     generate_peer_id,
     spawn_utils::{spawn, BlockingSpawner},
     torrent_from_bytes,
@@ -12,7 +14,6 @@ use librqbit::{
 };
 use log::{info, warn};
 use reqwest::Url;
-use tokio::sync::mpsc::UnboundedReceiver;
 
 async fn torrent_from_url(url: &str) -> anyhow::Result<TorrentMetaV1Owned> {
     let response = reqwest::get(url)
@@ -168,12 +169,16 @@ fn main() -> anyhow::Result<()> {
 
 async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()> {
     let peer_id = generate_peer_id();
+    let dht = Dht::new(&["dht.transmissionbt.com:6881", "dht.libtorrent.org:25401"])
+        .await
+        .context("error initializing DHT")?;
+
     if opts.torrent_path.starts_with("magnet:") {
         let Magnet {
             info_hash,
             trackers,
         } = Magnet::parse(&opts.torrent_path).context("provided path is not a valid magnet URL")?;
-        let dht_rx = JsDht::new(info_hash).start_peer_discovery()?;
+        let dht_rx = dht.get_peers(Id20(info_hash)).await;
         let (info, dht_rx, initial_peers) =
             match read_metainfo_from_peer_receiver(peer_id, info_hash, dht_rx).await {
                 librqbit::dht::inforead::ReadMetainfoResult::Found { info, rx, seen } => {
@@ -189,7 +194,7 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
             info,
             peer_id,
             dht_rx,
-            initial_peers,
+            initial_peers.into_iter().collect(),
             trackers
                 .into_iter()
                 .filter_map(|url| match reqwest::Url::parse(&url) {
@@ -211,7 +216,7 @@ async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()>
         } else {
             torrent_from_file(&opts.torrent_path)?
         };
-        let dht_rx = JsDht::new(torrent.info_hash).start_peer_discovery()?;
+        let dht_rx = dht.get_peers(Id20(torrent.info_hash)).await;
         let trackers = torrent
             .iter_announce()
             .filter_map(|tracker| {
@@ -251,7 +256,7 @@ async fn main_info(
     info_hash: InfoHash,
     info: TorrentMetaV1Info<ByteString>,
     peer_id: [u8; 20],
-    mut dht_peer_rx: UnboundedReceiver<SocketAddr>,
+    mut dht_peer_rx: impl StreamExt<Item = SocketAddr> + Unpin + Send + Sync + 'static,
     initial_peers: Vec<SocketAddr>,
     trackers: Vec<reqwest::Url>,
     spawner: BlockingSpawner,
@@ -298,7 +303,7 @@ async fn main_info(
     spawn("DHT peer adder", {
         let handle = handle.clone();
         async move {
-            while let Some(peer) = dht_peer_rx.recv().await {
+            while let Some(peer) = dht_peer_rx.next().await {
                 handle.add_peer(peer);
             }
             warn!("dht was closed");